diff --git a/deps/x86/rocksdb_static/librocksdb.a b/deps/x86/rocksdb_static/librocksdb.a
index 0c788dd674..0461fce8e4 100644
Binary files a/deps/x86/rocksdb_static/librocksdb.a and b/deps/x86/rocksdb_static/librocksdb.a differ
diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md
index 2ea144c56a..4f8ccc418b 100644
--- a/docs/en/12-taos-sql/10-function.md
+++ b/docs/en/12-taos-sql/10-function.md
@@ -565,7 +565,7 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal)
 - The uppercase or lowercase of `MONTH`, `MON`, `DAY`, `DY` and formtas that output digits have same effect when used in `to_timestamp`, like `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month` can be replaced by `MONTH`, or `month`. The cases are ignored.
 - If multi times are specified for one component, the previous will be overwritten. Like `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, the output year will be `2022`.
 - To avoid unexpected time zone used during the convertion, it's recommended to put time zone in the ts string, e.g. '2023-10-10 10:10:10+08'. If time zone not specified, default will be that in server or client.
-- The default timestamp if some components are not specified will be: `1970-01-01 00:00:00` with the timezone specified or default to local timezone.
+- The default timestamp if some components are not specified will be: `1970-01-01 00:00:00` with the timezone specified or default to local timezone. Only `DDD` is specified without `DD` is not supported currently, e.g. format 'yyyy-mm-ddd' is not supported, but 'yyyy-mm-dd' is supported.
 - If `AM` or `PM` is specified in formats, the Hour must between `1-12`.
 - In some cases, `to_timestamp` can convert correctly even the format and the timestamp string are not totally matched. Like `to_timetamp('200101/2', 'yyyyMM1/dd')`, the digit `1` in format string are ignored, and the output timestsamp is `2001-01-02 00:00:00`. Spaces and tabs in formats and tiemstamp string are also ignored automatically.
 - The precision of the output timestamp will be the same as the table in SELECT stmt, millisecond will be used if no table is specified. The output of `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')` will be truncated to millisecond precision. If a nano precision table is specified, no truncation will be applied. Like `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`.
diff --git a/docs/en/12-taos-sql/12-distinguished.md b/docs/en/12-taos-sql/12-distinguished.md
index 7f0b8c7769..502b7562f3 100644
--- a/docs/en/12-taos-sql/12-distinguished.md
+++ b/docs/en/12-taos-sql/12-distinguished.md
@@ -38,11 +38,16 @@ Aggregation by time window is supported in TDengine. For example, in the case wh
 window_clause: {
     SESSION(ts_col, tol_val)
   | STATE_WINDOW(col)
-  | INTERVAL(interval [, offset]) [SLIDING sliding] [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})]
+  | INTERVAL(interval_val [, offset]) [SLIDING (sliding_value)] [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})]
   | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
 }
 ```
 
+Both interval_val and sliding_value are time durations which have 3 forms of representation.
+- INTERVAL(1s, 500a) SLIDING(1s), the unit char should be any one of a (millisecond), b (nanosecond), d (day), h (hour), m (minute), n (month), s (second), u (microsecond), w (week), y (year).
+- INTERVAL(1000, 500) SLIDING(1000), the unit will the same as the queried database, if there are more than one databases, higher precision will be used.
+- INTERVAL('1s', '500a') SLIDING('1s'), unit must be specified, no spaces allowed.
+
 The following restrictions apply:
 
 ### Other Rules
diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md
index c1dc6a6363..26313390a6 100644
--- a/docs/zh/12-taos-sql/10-function.md
+++ b/docs/zh/12-taos-sql/10-function.md
@@ -565,7 +565,7 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal)
 - `MONTH`, `MON`, `DAY`, `DY` 以及其他输出为数字的格式的大小写意义相同, 如 `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month`可以被替换为`MONTH` 或者`Month`.
 - 如果同一字段被指定了多次, 那么前面的指定将会被覆盖.  如 `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, 输出年份是`2022`.
 - 为避免转换时使用了非预期的时区，推荐在时间中携带时区信息，例如'2023-10-10 10:10:10+08'，如果未指定时区则默认时区为服务端或客户端指定的时区。
-- 如果没有指定完整的时间，那么默认时间值为指定或默认时区的 `1970-01-01 00:00:00`, 未指定部分使用该默认值中的对应部分.
+- 如果没有指定完整的时间，那么默认时间值为指定或默认时区的 `1970-01-01 00:00:00`, 未指定部分使用该默认值中的对应部分. 暂不支持只指定年日而不指定月日的格式, 如'yyyy-mm-DDD', 支持'yyyy-mm-DD'.
 - 如果格式串中有`AM`, `PM`等, 那么小时必须是12小时制, 范围必须是01-12.
 - `to_timestamp`转换具有一定的容错机制, 在格式串和时间戳串不完全对应时, 有时也可转换, 如: `to_timestamp('200101/2', 'yyyyMM1/dd')`, 格式串中多出来的1会被丢弃. 格式串与时间戳串中多余的空格字符(空格, tab等)也会被 自动忽略. 如`to_timestamp('  23 年 - 1 月 - 01 日  ', 'yy 年-MM月-dd日')` 可以被成功转换. 虽然`MM`等字段需要两个数字对应(只有一位时前面补0), 在`to_timestamp`时, 一个数字也可以成功转换.
 - 输出时间戳的精度与查询表的精度相同, 若查询未指定表, 则输出精度为毫秒. 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')`的输出将会把微妙和纳秒进行截断. 如果指定一张纳秒表, 那么就不会发生截断, 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`.
diff --git a/docs/zh/12-taos-sql/12-distinguished.md b/docs/zh/12-taos-sql/12-distinguished.md
index 62888cc5f7..98bfd3567a 100755
--- a/docs/zh/12-taos-sql/12-distinguished.md
+++ b/docs/zh/12-taos-sql/12-distinguished.md
@@ -44,7 +44,11 @@ window_clause: {
 }
 ```
 
-在上述语法中的具体限制如下
+其中，interval_val 和 sliding_val 都表示时间段, 语法上支持三种方式，举例说明如下:
+ - INTERVAL(1s, 500a) SLIDING(1s), 自带时间单位的形式，其中的时间单位是单字符表示, 分别为: a (毫秒), b (纳秒), d (天), h (小时), m (分钟), n (月), s (秒), u (微妙), w (周), y (年).
+ - INTERVAL(1000, 500) SLIDING(1000), 不带时间单位的形式，将使用查询库的时间精度作为默认时间单位，当存在多个库时默认采用精度更高的库.
+ - INTERVAL('1s', '500a') SLIDING('1s'), 自带时间单位的字符串形式，字符串内部不能有任何空格等其它字符.
+
 
 ### 窗口子句的规则
 
diff --git a/source/dnode/vnode/src/inc/vndCos.h b/include/common/cos.h
similarity index 90%
rename from source/dnode/vnode/src/inc/vndCos.h
rename to include/common/cos.h
index 8581b039f8..21b645f604 100644
--- a/source/dnode/vnode/src/inc/vndCos.h
+++ b/include/common/cos.h
@@ -16,7 +16,7 @@
 #ifndef _TD_VND_COS_H_
 #define _TD_VND_COS_H_
 
-#include "vnd.h"
+#include "os.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -24,6 +24,7 @@ extern "C" {
 
 #define S3_BLOCK_CACHE
 
+extern int8_t  tsS3StreamEnabled;
 extern int8_t  tsS3Enabled;
 extern int32_t tsS3BlockSize;
 extern int32_t tsS3BlockCacheSize;
@@ -38,7 +39,8 @@ void    s3DeleteObjectsByPrefix(const char *prefix);
 void    s3DeleteObjects(const char *object_name[], int nobject);
 bool    s3Exists(const char *object_name);
 bool    s3Get(const char *object_name, const char *path);
-int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock);
+int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock);
+int32_t s3GetObjectsByPrefix(const char *prefix, const char *path);
 void    s3EvictCache(const char *path, long object_size);
 long    s3Size(const char *object_name);
 
diff --git a/include/common/rsync.h b/include/common/rsync.h
new file mode 100644
index 0000000000..6cce645d1e
--- /dev/null
+++ b/include/common/rsync.h
@@ -0,0 +1,24 @@
+//
+// Created by mingming wanng on 2023/11/2.
+//
+
+#ifndef TDENGINE_RSYNC_H
+#define TDENGINE_RSYNC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tarray.h"
+
+void stopRsync();
+void startRsync();
+int  uploadRsync(char* id, char* path);
+int  downloadRsync(char* id, char* path);
+int  deleteRsync(char* id);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_RSYNC_H
diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 72aab9adf0..e072eaa831 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -249,6 +249,7 @@ typedef struct SQueryTableDataCond {
   SColumnInfo* colList;
   int32_t*     pSlotList;  // the column output destation slot, and it may be null
   int32_t      type;       // data block load type:
+  bool         skipRollup;
   STimeWindow  twindows;
   int64_t      startVersion;
   int64_t      endVersion;
diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index f2ffde7b15..8688e07932 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -75,6 +75,13 @@ extern int32_t tsElectInterval;
 extern int32_t tsHeartbeatInterval;
 extern int32_t tsHeartbeatTimeout;
 
+// snode
+extern int32_t tsRsyncPort;
+extern char tsCheckpointBackupDir[];
+
+// vnode checkpoint
+extern char tsSnodeAddress[];       //127.0.0.1:873
+
 // mnode
 extern int64_t tsMndSdbWriteDelta;
 extern int64_t tsMndLogRetention;
diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index d0e2b45937..f78b7a3126 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -49,6 +49,7 @@ typedef struct {
   uint64_t    checkpointId;
   bool        initTableReader;
   bool        initTqReader;
+  bool        skipRollup;
   int32_t     numOfVgroups;
   void*       sContext;  // SSnapContext*
   void*       pStateBackend;
diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index 2e3cd670d7..49435a6317 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -168,7 +168,6 @@ typedef struct {
   struct SStreamFileState *pFileState;
   int32_t                  number;
   SSHashObj               *parNameMap;
-  int64_t                  checkPointId;
   int32_t                  taskId;
   int64_t                  streamId;
   int64_t                  streamBackendRid;
diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h
index dbbe1d92dc..4ffcb616dd 100644
--- a/include/libs/nodes/plannodes.h
+++ b/include/libs/nodes/plannodes.h
@@ -388,6 +388,7 @@ typedef struct SLastRowScanPhysiNode {
   SNodeList*     pGroupTags;
   bool           groupSort;
   bool           ignoreNull;
+  SNodeList*     pTargets;
 } SLastRowScanPhysiNode;
 
 typedef SLastRowScanPhysiNode STableCountScanPhysiNode;
diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 4b760f3f4e..654a0b6abc 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -827,6 +827,7 @@ void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 int32_t      streamMetaReopen(SStreamMeta* pMeta);
 int32_t      streamMetaCommit(SStreamMeta* pMeta);
 int32_t      streamMetaLoadAllTasks(SStreamMeta* pMeta);
+int64_t      streamMetaGetLatestCheckpointId(SStreamMeta* pMeta);
 void         streamMetaNotifyClose(SStreamMeta* pMeta);
 void         streamMetaStartHb(SStreamMeta* pMeta);
 void         streamMetaInitForSnode(SStreamMeta* pMeta);
@@ -841,6 +842,7 @@ void         streamMetaResetStartInfo(STaskStartInfo* pMeta);
 // checkpoint
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
 int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask);
+int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
 void    streamTaskClearCheckInfo(SStreamTask* pTask);
 int32_t streamAlignTransferState(SStreamTask* pTask);
 int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId);
diff --git a/include/os/osDef.h b/include/os/osDef.h
index bad4536fa6..1a831f2e86 100644
--- a/include/os/osDef.h
+++ b/include/os/osDef.h
@@ -237,6 +237,12 @@ void syslog(int unused, const char *format, ...);
 #define TD_DIRSEP "/"
 #endif
 
+#if defined(_WIN32)
+#define TD_DIRSEP_CHAR '\\'
+#else
+#define TD_DIRSEP_CHAR '/'
+#endif
+
 #define TD_LOCALE_LEN   64
 #define TD_CHARSET_LEN  64
 #define TD_TIMEZONE_LEN 96
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 980d5b9177..7665550153 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -754,6 +754,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_FUNC_DUP_TIMESTAMP            TAOS_DEF_ERROR_CODE(0, 0x2805)
 #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR TAOS_DEF_ERROR_CODE(0, 0x2806)
 #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807)
+#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808)
 
 //udf
 #define TSDB_CODE_UDF_STOPPING                  TAOS_DEF_ERROR_CODE(0, 0x2901)
diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt
index c35845f9df..c7831b2b6d 100644
--- a/source/common/CMakeLists.txt
+++ b/source/common/CMakeLists.txt
@@ -46,6 +46,75 @@ target_link_libraries(
     INTERFACE api
 )
 
+if(${BUILD_S3})
+
+    if(${BUILD_WITH_S3})
+        target_include_directories(
+                common
+
+                PUBLIC "$ENV{HOME}/.cos-local.2/include"
+        )
+
+        set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+        set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2)
+        find_library(S3_LIBRARY s3)
+        find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
+        find_library(XML2_LIBRARY xml2)
+        find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
+        find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
+        target_link_libraries(
+                common
+
+                # s3
+                PUBLIC ${S3_LIBRARY}
+                PUBLIC ${CURL_LIBRARY}
+                PUBLIC ${SSL_LIBRARY}
+                PUBLIC ${CRYPTO_LIBRARY}
+                PUBLIC ${XML2_LIBRARY}
+        )
+
+        add_definitions(-DUSE_S3)
+    endif()
+
+    if(${BUILD_WITH_COS})
+
+        set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+        find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
+        find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
+        find_library(MINIXML_LIBRARY mxml)
+        find_library(CURL_LIBRARY curl)
+        target_link_libraries(
+                common
+
+                # s3
+                PUBLIC cos_c_sdk_static
+                PUBLIC ${APR_UTIL_LIBRARY}
+                PUBLIC ${APR_LIBRARY}
+                PUBLIC ${MINIXML_LIBRARY}
+                PUBLIC ${CURL_LIBRARY}
+        )
+
+        # s3
+        FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
+        IF (APR_CONFIG_BIN)
+            EXECUTE_PROCESS(
+                    COMMAND ${APR_CONFIG_BIN} --includedir
+                    OUTPUT_VARIABLE APR_INCLUDE_DIR
+                    OUTPUT_STRIP_TRAILING_WHITESPACE
+            )
+        ENDIF()
+        include_directories (${APR_INCLUDE_DIR})
+        target_include_directories(
+                common
+                PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk"
+                PUBLIC "$ENV{HOME}/.cos-local.1/include"
+        )
+
+        add_definitions(-DUSE_COS)
+    endif(${BUILD_WITH_COS})
+
+endif()
+
 if(${BUILD_TEST})
     ADD_SUBDIRECTORY(test)
 endif(${BUILD_TEST})
diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/common/src/cos.c
similarity index 87%
rename from source/dnode/vnode/src/vnd/vnodeCos.c
rename to source/common/src/cos.c
index 6e36739f5a..0b6b0db885 100644
--- a/source/dnode/vnode/src/vnd/vnodeCos.c
+++ b/source/common/src/cos.c
@@ -1,6 +1,6 @@
 #define ALLOW_FORBID_FUNC
 
-#include "vndCos.h"
+#include "cos.h"
 
 extern char   tsS3Endpoint[];
 extern char   tsS3AccessKeyId[];
@@ -13,6 +13,7 @@ extern int8_t tsS3Https;
 #if defined(USE_S3)
 
 #include "libs3.h"
+#include "tarray.h"
 
 static int         verifyPeerG = 0;
 static const char *awsRegionG = NULL;
@@ -24,7 +25,7 @@ static S3UriStyle uriStyleG = S3UriStylePath;
 static int        retriesG = 5;
 static int        timeoutMsG = 0;
 
-static int32_t s3Begin() {
+int32_t s3Begin() {
   S3Status    status;
   const char *hostname = tsS3Hostname;
   const char *env_hn = getenv("S3_HOSTNAME");
@@ -34,7 +35,7 @@ static int32_t s3Begin() {
   }
 
   if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) {
-    vError("Failed to initialize libs3: %s\n", S3_get_status_name(status));
+    uError("Failed to initialize libs3: %s\n", S3_get_status_name(status));
     return -1;
   }
 
@@ -43,10 +44,12 @@ static int32_t s3Begin() {
   return 0;
 }
 
-static void s3End() { S3_deinitialize(); }
-int32_t     s3Init() { return s3Begin(); }
+void s3End() { S3_deinitialize(); }
 
-void s3CleanUp() { s3End(); }
+int32_t s3Init() { return 0; /*s3Begin();*/ }
+
+void s3CleanUp() { /*s3End();*/
+}
 
 static int should_retry() {
   /*
@@ -65,19 +68,32 @@ static int should_retry() {
 
 static void s3PrintError(const char *func, S3Status status, char error_details[]) {
   if (status < S3StatusErrorAccessDenied) {
-    vError("%s: %s", __func__, S3_get_status_name(status));
+    uError("%s: %s", __func__, S3_get_status_name(status));
   } else {
-    vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details);
+    uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details);
   }
 }
 
+typedef struct {
+  char      err_msg[512];
+  S3Status  status;
+  uint64_t  content_length;
+  TdFilePtr file;
+} TS3GetData;
+
 typedef struct {
   char     err_msg[128];
   S3Status status;
   uint64_t content_length;
   char    *buf;
+  int64_t  buf_pos;
 } TS3SizeCBD;
 
+static S3Status responsePropertiesCallbackNull(const S3ResponseProperties *properties, void *callbackData) {
+  //  (void)callbackData;
+  return S3StatusOK;
+}
+
 static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) {
   //(void)callbackData;
   TS3SizeCBD *cbd = callbackData;
@@ -97,20 +113,22 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro
   int       len = 0;
   const int elen = sizeof(cbd->err_msg);
   if (error) {
-    if (error->message) {
+    if (error->message && elen - len > 0) {
       len += snprintf(&(cbd->err_msg[len]), elen - len, "  Message: %s\n", error->message);
     }
-    if (error->resource) {
+    if (error->resource && elen - len > 0) {
       len += snprintf(&(cbd->err_msg[len]), elen - len, "  Resource: %s\n", error->resource);
     }
-    if (error->furtherDetails) {
+    if (error->furtherDetails && elen - len > 0) {
       len += snprintf(&(cbd->err_msg[len]), elen - len, "  Further Details: %s\n", error->furtherDetails);
     }
-    if (error->extraDetailsCount) {
+    if (error->extraDetailsCount && elen - len > 0) {
       len += snprintf(&(cbd->err_msg[len]), elen - len, "%s", "  Extra Details:\n");
       for (int i = 0; i < error->extraDetailsCount; i++) {
-        len += snprintf(&(cbd->err_msg[len]), elen - len, "    %s: %s\n", error->extraDetails[i].name,
-                        error->extraDetails[i].value);
+        if (elen - len > 0) {
+          len += snprintf(&(cbd->err_msg[len]), elen - len, "    %s: %s\n", error->extraDetails[i].name,
+                          error->extraDetails[i].value);
+        }
       }
     }
   }
@@ -201,8 +219,9 @@ static void growbuffer_destroy(growbuffer *gb) {
 }
 
 typedef struct put_object_callback_data {
-  char     err_msg[128];
+  char     err_msg[512];
   S3Status status;
+  uint64_t content_length;
   // FILE       *infile;
   TdFilePtr   infileFD;
   growbuffer *gb;
@@ -214,8 +233,9 @@ typedef struct put_object_callback_data {
 #define MULTIPART_CHUNK_SIZE (768 << 20)  // multipart is 768M
 
 typedef struct UploadManager {
-  char     err_msg[128];
+  char     err_msg[512];
   S3Status status;
+  uint64_t content_length;
   // used for initial multipart
   char *upload_id;
 
@@ -229,8 +249,9 @@ typedef struct UploadManager {
 } UploadManager;
 
 typedef struct list_parts_callback_data {
-  char           err_msg[128];
+  char           err_msg[512];
   S3Status       status;
+  uint64_t       content_length;
   int            isTruncated;
   char           nextPartNumberMarker[24];
   char           initiatorId[256];
@@ -246,7 +267,7 @@ typedef struct list_parts_callback_data {
 } list_parts_callback_data;
 
 typedef struct MultipartPartData {
-  char                     err_msg[128];
+  char                     err_msg[512];
   S3Status                 status;
   put_object_callback_data put_object_data;
   int                      seq;
@@ -291,7 +312,7 @@ S3Status initial_multipart_callback(const char *upload_id, void *callbackData) {
 }
 
 S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) {
-  responsePropertiesCallback(properties, callbackData);
+  responsePropertiesCallbackNull(properties, callbackData);
 
   MultipartPartData *data = (MultipartPartData *)callbackData;
   int                seq = data->seq;
@@ -390,7 +411,8 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan
   S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
                                    0, awsRegionG};
 
-  S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &listPartsCallback};
+  S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
+&listPartsCallback};
 
   list_parts_callback_data data;
 
@@ -445,13 +467,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
   data.noStatus = noStatus;
 
   if (taosStatFile(file, &contentLength, NULL, NULL) < 0) {
-    vError("ERROR: %s Failed to stat file %s: ", __func__, file);
+    uError("ERROR: %s Failed to stat file %s: ", __func__, file);
     code = TAOS_SYSTEM_ERROR(errno);
     return code;
   }
 
   if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
-    vError("ERROR: %s Failed to open file %s: ", __func__, file);
+    uError("ERROR: %s Failed to open file %s: ", __func__, file);
     code = TAOS_SYSTEM_ERROR(errno);
     return code;
   }
@@ -469,7 +491,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
                                    metaProperties,  useServerSideEncryption};
 
   if (contentLength <= MULTIPART_CHUNK_SIZE) {
-    S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
+    S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
                                            &putObjectDataCallback};
 
     do {
@@ -486,7 +508,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
       s3PrintError(__func__, data.status, data.err_msg);
       code = TAOS_SYSTEM_ERROR(EIO);
     } else if (data.contentLength) {
-      vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
+      uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
              (unsigned long long)data.contentLength);
       code = TAOS_SYSTEM_ERROR(EIO);
     }
@@ -506,14 +528,14 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
     memset(&partData, 0, sizeof(MultipartPartData));
     int partContentLength = 0;
 
-    S3MultipartInitialHandler handler = {{&responsePropertiesCallback, &responseCompleteCallback},
+    S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
                                          &initial_multipart_callback};
 
     S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback},
                                            &putObjectDataCallback};
 
     S3MultipartCommitHandler commit_handler = {
-        {&responsePropertiesCallback, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
+        {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
 
     manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq);
     manager.next_etags_pos = 0;
@@ -609,7 +631,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
 }
 
 typedef struct list_bucket_callback_data {
-  char     err_msg[128];
+  char     err_msg[512];
   S3Status status;
   int      isTruncated;
   char     nextMarker[1024];
@@ -658,19 +680,19 @@ static void s3FreeObjectKey(void *pItem) {
   taosMemoryFree(key);
 }
 
-void s3DeleteObjectsByPrefix(const char *prefix) {
+static SArray *getListByPrefix(const char *prefix) {
   S3BucketContext     bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
                                        0, awsRegionG};
-  S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
+  S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
                                            &listBucketCallback};
 
   const char               *marker = 0, *delimiter = 0;
   int                       maxkeys = 0, allDetails = 0;
   list_bucket_callback_data data;
-  data.objectArray = taosArrayInit(32, POINTER_BYTES);
+  data.objectArray = taosArrayInit(32, sizeof(void *));
   if (!data.objectArray) {
-    vError("%s: %s", __func__, "out of memoty");
-    return;
+    uError("%s: %s", __func__, "out of memoty");
+    return NULL;
   }
   if (marker) {
     snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker);
@@ -693,18 +715,15 @@ void s3DeleteObjectsByPrefix(const char *prefix) {
 
   if (data.status == S3StatusOK) {
     if (data.keyCount > 0) {
-      // printListBucketHeader(allDetails);
-      s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray));
+      return data.objectArray;
     }
   } else {
     s3PrintError(__func__, data.status, data.err_msg);
   }
-
-  taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
+  return NULL;
 }
 
 void s3DeleteObjects(const char *object_name[], int nobject) {
-  int               status = 0;
   S3BucketContext   bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
                                      0, awsRegionG};
   S3ResponseHandler responseHandler = {0, &responseCompleteCallback};
@@ -721,17 +740,28 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
   }
 }
 
+void s3DeleteObjectsByPrefix(const char *prefix) {
+  SArray *objectArray = getListByPrefix(prefix);
+  if (objectArray == NULL) return;
+  s3DeleteObjects(TARRAY_DATA(objectArray), TARRAY_SIZE(objectArray));
+  taosArrayDestroyEx(objectArray, s3FreeObjectKey);
+}
+
 static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData) {
   TS3SizeCBD *cbd = callbackData;
+  /*
   if (cbd->content_length != bufferSize) {
     cbd->status = S3StatusAbortedByCallback;
     return S3StatusAbortedByCallback;
   }
+  */
+  if (!cbd->buf) {
+    cbd->buf = taosMemoryCalloc(1, cbd->content_length);
+  }
 
-  char *buf = taosMemoryCalloc(1, bufferSize);
-  if (buf) {
-    memcpy(buf, buffer, bufferSize);
-    cbd->buf = buf;
+  if (cbd->buf) {
+    memcpy(cbd->buf + cbd->buf_pos, buffer, bufferSize);
+    cbd->buf_pos += bufferSize;
     cbd->status = S3StatusOK;
     return S3StatusOK;
   } else {
@@ -740,7 +770,7 @@ static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *
   }
 }
 
-int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) {
+int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
   int         status = 0;
   int64_t     ifModifiedSince = -1, ifNotModifiedSince = -1;
   const char *ifMatch = 0, *ifNotMatch = 0;
@@ -753,12 +783,18 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
 
   TS3SizeCBD cbd = {0};
   cbd.content_length = size;
+  cbd.buf_pos = 0;
   do {
     S3_get_object(&bucketContext, object_name, &getConditions, offset, size, 0, 0, &getObjectHandler, &cbd);
   } while (S3_status_is_retryable(cbd.status) && should_retry());
 
   if (cbd.status != S3StatusOK) {
-    vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    return TAOS_SYSTEM_ERROR(EIO);
+  }
+
+  if (check && cbd.buf_pos != size) {
+    uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
     return TAOS_SYSTEM_ERROR(EIO);
   }
 
@@ -767,6 +803,66 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
   return 0;
 }
 
+static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) {
+  TS3GetData *cbd = (TS3GetData *)callbackData;
+  size_t      wrote = taosWriteFile(cbd->file, buffer, bufferSize);
+  return ((wrote < (size_t)bufferSize) ? S3StatusAbortedByCallback : S3StatusOK);
+}
+
+int32_t s3GetObjectToFile(const char *object_name, char *fileName) {
+  int64_t     ifModifiedSince = -1, ifNotModifiedSince = -1;
+  const char *ifMatch = 0, *ifNotMatch = 0;
+
+  S3BucketContext    bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
+                                      0, awsRegionG};
+  S3GetConditions    getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch};
+  S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
+                                         &getObjectCallback};
+
+  TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    uError("[s3] open file error, errno:%d, fileName:%s", errno, fileName);
+    return -1;
+  }
+  TS3GetData cbd = {0};
+  cbd.file = pFile;
+  do {
+    S3_get_object(&bucketContext, object_name, &getConditions, 0, 0, 0, 0, &getObjectHandler, &cbd);
+  } while (S3_status_is_retryable(cbd.status) && should_retry());
+
+  if (cbd.status != S3StatusOK) {
+    uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    taosCloseFile(&pFile);
+    return TAOS_SYSTEM_ERROR(EIO);
+  }
+
+  taosCloseFile(&pFile);
+  return 0;
+}
+
+int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
+  SArray *objectArray = getListByPrefix(prefix);
+  if (objectArray == NULL) return -1;
+
+  for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) {
+    char       *object = taosArrayGetP(objectArray, i);
+    const char *tmp = strchr(object, '/');
+    tmp = (tmp == NULL) ? object : tmp + 1;
+    char fileName[PATH_MAX] = {0};
+    if (path[strlen(path) - 1] != TD_DIRSEP_CHAR) {
+      snprintf(fileName, PATH_MAX, "%s%s%s", path, TD_DIRSEP, tmp);
+    } else {
+      snprintf(fileName, PATH_MAX, "%s%s", path, tmp);
+    }
+    if (s3GetObjectToFile(object, fileName) != 0) {
+      taosArrayDestroyEx(objectArray, s3FreeObjectKey);
+      return -1;
+    }
+  }
+  taosArrayDestroyEx(objectArray, s3FreeObjectKey);
+  return 0;
+}
+
 long s3Size(const char *object_name) {
   long size = 0;
   int  status = 0;
@@ -782,7 +878,7 @@ long s3Size(const char *object_name) {
   } while (S3_status_is_retryable(cbd.status) && should_retry());
 
   if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
-    vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
   }
 
   size = cbd.content_length;
@@ -1045,7 +1141,8 @@ bool s3Get(const char *object_name, const char *path) {
   return ret;
 }
 
-int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, uint8_t **ppBlock) {
+int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) {
+  (void)check;
   int32_t                code = 0;
   cos_pool_t            *p = NULL;
   int                    is_cname = 0;
@@ -1237,8 +1334,11 @@ void    s3DeleteObjectsByPrefix(const char *prefix) {}
 void    s3DeleteObjects(const char *object_name[], int nobject) {}
 bool    s3Exists(const char *object_name) { return false; }
 bool    s3Get(const char *object_name, const char *path) { return false; }
-int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) { return 0; }
+int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {
+  return 0;
+}
 void    s3EvictCache(const char *path, long object_size) {}
 long    s3Size(const char *object_name) { return 0; }
+int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { return 0; }
 
 #endif
diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c
new file mode 100644
index 0000000000..ffab85761e
--- /dev/null
+++ b/source/common/src/rsync.c
@@ -0,0 +1,235 @@
+//
+// Created by mingming wanng on 2023/11/2.
+//
+#include "rsync.h"
+#include <stdlib.h>
+#include "tglobal.h"
+
+#define ERRNO_ERR_FORMAT  "errno:%d,msg:%s"
+#define ERRNO_ERR_DATA    errno,strerror(errno)
+
+// deleteRsync function produce empty directories, traverse base directory to remove them
+static void removeEmptyDir(){
+  TdDirPtr pDir = taosOpenDir(tsCheckpointBackupDir);
+  if (pDir == NULL) return;
+
+  TdDirEntryPtr de = NULL;
+  while ((de = taosReadDir(pDir)) != NULL) {
+    if (!taosDirEntryIsDir(de)) {
+      continue;
+    }
+
+    if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue;
+
+    char filename[PATH_MAX] = {0};
+    snprintf(filename, sizeof(filename), "%s%s", tsCheckpointBackupDir, taosGetDirEntryName(de));
+
+    TdDirPtr pDirTmp = taosOpenDir(filename);
+    TdDirEntryPtr deTmp = NULL;
+    bool empty = true;
+    while ((deTmp = taosReadDir(pDirTmp)) != NULL){
+      if (strcmp(taosGetDirEntryName(deTmp), ".") == 0 || strcmp(taosGetDirEntryName(deTmp), "..") == 0) continue;
+      empty = false;
+    }
+    if(empty) taosRemoveDir(filename);
+    taosCloseDir(&pDirTmp);
+  }
+
+  taosCloseDir(&pDir);
+}
+
+#ifdef WINDOWS
+// C:\TDengine\data\backup\checkpoint\ -> /c/TDengine/data/backup/checkpoint/
+static void changeDirFromWindowsToLinux(char* from, char* to){
+  to[0] = '/';
+  to[1] = from[0];
+  for(int i = 2; i < strlen(from); i++) {
+    if (from[i] == '\\') {
+      to[i] = '/';
+    } else {
+      to[i] = from[i];
+    }
+  }
+}
+#endif
+
+static int generateConfigFile(char* confDir){
+  TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    uError("[rsync] open conf file error, dir:%s,"ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA);
+    return -1;
+  }
+
+#ifdef WINDOWS
+  char path[PATH_MAX] = {0};
+  changeDirFromWindowsToLinux(tsCheckpointBackupDir, path);
+#endif
+
+  char confContent[PATH_MAX*4] = {0};
+  snprintf(confContent, PATH_MAX*4,
+#ifndef WINDOWS
+           "uid = root\n"
+           "gid = root\n"
+#endif
+           "use chroot = false\n"
+           "max connections = 200\n"
+           "timeout = 100\n"
+           "lock file = %srsync.lock\n"
+           "log file = %srsync.log\n"
+           "ignore errors = true\n"
+           "read only = false\n"
+           "list = false\n"
+           "[checkpoint]\n"
+           "path = %s", tsCheckpointBackupDir, tsCheckpointBackupDir,
+#ifdef WINDOWS
+           path
+#else
+           tsCheckpointBackupDir
+#endif
+           );
+  uDebug("[rsync] conf:%s", confContent);
+  if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0){
+    uError("[rsync] write conf file error,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
+    taosCloseFile(&pFile);
+    return -1;
+  }
+
+  taosCloseFile(&pFile);
+  return 0;
+}
+
+static int execCommand(char* command){
+  int try = 3;
+  int32_t code = 0;
+  while(try-- > 0) {
+    code = system(command);
+    if (code == 0) {
+      break;
+    }
+    taosMsleep(10);
+  }
+  return code;
+}
+
+void stopRsync(){
+  int code =
+#ifdef WINDOWS
+  system("taskkill /f /im rsync.exe");
+#else
+  system("pkill rsync");
+#endif
+  if(code != 0){
+    uError("[rsync] stop rsync server failed,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA);
+    return;
+  }
+  uDebug("[rsync] stop rsync server successful");
+}
+
+void startRsync(){
+  if(taosMulMkDir(tsCheckpointBackupDir) != 0){
+    uError("[rsync] build checkpoint backup dir failed, dir:%s,"ERRNO_ERR_FORMAT, tsCheckpointBackupDir, ERRNO_ERR_DATA);
+    return;
+  }
+  removeEmptyDir();
+
+  char confDir[PATH_MAX] = {0};
+  snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir);
+
+  int code = generateConfigFile(confDir);
+  if(code != 0){
+    return;
+  }
+
+  char cmd[PATH_MAX] = {0};
+  snprintf(cmd, PATH_MAX, "rsync --daemon --port=%d --config=%s", tsRsyncPort, confDir);
+  // start rsync service to backup checkpoint
+  code = system(cmd);
+  if(code != 0){
+    uError("[rsync] start server failed, code:%d,"ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    return;
+  }
+  uDebug("[rsync] start server successful");
+}
+
+int uploadRsync(char* id, char* path){
+#ifdef WINDOWS
+  char pathTransform[PATH_MAX] = {0};
+  changeDirFromWindowsToLinux(path, pathTransform);
+#endif
+  char command[PATH_MAX] = {0};
+#ifdef WINDOWS
+  if(pathTransform[strlen(pathTransform) - 1] != '/'){
+#else
+  if(path[strlen(path) - 1] != '/'){
+#endif
+    snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s/ rsync://%s/checkpoint/%s/",
+#ifdef WINDOWS
+             pathTransform
+#else
+             path
+#endif
+             , tsSnodeAddress, id);
+  }else{
+    snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s rsync://%s/checkpoint/%s/",
+#ifdef WINDOWS
+             pathTransform
+#else
+             path
+#endif
+             , tsSnodeAddress, id);
+  }
+
+  int code = execCommand(command);
+  if(code != 0){
+    uError("[rsync] send failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    return -1;
+  }
+  uDebug("[rsync] upload data:%s successful", id);
+  return 0;
+}
+
+int downloadRsync(char* id, char* path){
+#ifdef WINDOWS
+  char pathTransform[PATH_MAX] = {0};
+  changeDirFromWindowsToLinux(path, pathTransform);
+#endif
+  char command[PATH_MAX] = {0};
+  snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s",
+           tsSnodeAddress, id,
+#ifdef WINDOWS
+           pathTransform
+#else
+           path
+#endif
+           );
+
+  int code = execCommand(command);
+  if(code != 0){
+    uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    return -1;
+  }
+  uDebug("[rsync] down data:%s successful", id);
+  return 0;
+}
+
+int deleteRsync(char* id){
+  char* tmp = "./tmp_empty/";
+  int code = taosMkDir(tmp);
+  if(code != 0){
+    uError("[rsync] make tmp dir failed. code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    return -1;
+  }
+  char command[PATH_MAX] = {0};
+  snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 %s rsync://%s/checkpoint/%s/",
+           tmp, tsSnodeAddress, id);
+
+  code = execCommand(command);
+  taosRemoveDir(tmp);
+  if(code != 0){
+    uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA);
+    return -1;
+  }
+  uDebug("[rsync] delete data:%s successful", id);
+
+  return 0;
+}
\ No newline at end of file
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 142f7f8078..76d8a0476e 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -94,8 +94,8 @@ int32_t  tsMonitorMaxLogs = 100;
 bool     tsMonitorComp = false;
 
 // audit
-bool     tsEnableAudit = true;
-bool     tsEnableAuditCreateTable = true;
+bool tsEnableAudit = true;
+bool tsEnableAuditCreateTable = true;
 
 // telem
 #ifdef TD_ENTERPRISE
@@ -127,6 +127,15 @@ char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = "";
 // bool    tsSmlDataFormat = false;
 // int32_t tsSmlBatchSize = 10000;
 
+// checkpoint backup
+char    tsSnodeAddress[TSDB_FQDN_LEN] = {0};
+int32_t tsRsyncPort = 873;
+#ifdef WINDOWS
+char tsCheckpointBackupDir[PATH_MAX] = "C:\\TDengine\\data\\backup\\checkpoint\\";
+#else
+char    tsCheckpointBackupDir[PATH_MAX] = "/var/lib/taos/backup/checkpoint/";
+#endif
+
 // tmq
 int32_t tmqMaxTopicNum = 20;
 // query
@@ -222,7 +231,7 @@ char     tsCompressor[32] = "ZSTD_COMPRESSOR";  // ZSTD_COMPRESSOR or GZIP_COMPR
 #ifdef WINDOWS
 bool tsStartUdfd = false;
 #else
-bool tsStartUdfd = true;
+bool    tsStartUdfd = true;
 #endif
 
 // wal
@@ -260,6 +269,7 @@ char   tsS3AccessKeySecret[TSDB_FQDN_LEN] = "<accesskeysecrect>";
 char   tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
 char   tsS3AppId[TSDB_FQDN_LEN] = "<appid>";
 int8_t tsS3Enabled = false;
+int8_t tsS3StreamEnabled = false;
 
 int8_t tsS3Https = true;
 char   tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
@@ -323,16 +333,19 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
       tstrncpy(tsS3AppId, appid + 1, TSDB_FQDN_LEN);
     }
   }
-  if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) {
+  if (tsS3BucketName[0] != '<') {
 #if defined(USE_COS) || defined(USE_S3)
-    tsS3Enabled = true;
+    if (tsDiskCfgNum > 1) tsS3Enabled = true;
+    tsS3StreamEnabled = true;
 #endif
   }
 
   return 0;
 }
 
-struct SConfig *taosGetCfg() { return tsCfg; }
+struct SConfig *taosGetCfg() {
+  return tsCfg;
+}
 
 static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile,
                            char *apolloUrl) {
@@ -442,8 +455,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
   if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
-  if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) !=
-      0)
+  if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT,
+                  CFG_DYN_CLIENT) != 0)
     return -1;
   if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
     return -1;
@@ -459,7 +472,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   //  if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0)
   //  return -1;
   if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1;
-  if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
+  if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) !=
+      0)
     return -1;
   if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
     return -1;
@@ -547,7 +561,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
   if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
+  if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
+    return -1;
   if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
     return -1;
 
@@ -595,7 +610,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
       0)
     return -1;
 
-  tsNumOfVnodeRsmaThreads = tsNumOfCores;
+  tsNumOfVnodeRsmaThreads = tsNumOfCores / 2;
   tsNumOfVnodeRsmaThreads = TMAX(tsNumOfVnodeRsmaThreads, 4);
   if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
     return -1;
@@ -662,6 +677,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
   if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
 
+  if (cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER) != 0) return -1;
+  if (cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
+  if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0)
+    return -1;
+
   if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
     return -1;
 
@@ -686,7 +706,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
     return -1;
-  if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
+  if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
+      0)
     return -1;
 
   if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX,
@@ -729,7 +750,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -100, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
+  if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
     return -1;
   if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
       0)
@@ -1100,7 +1121,10 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval;
   tsTtlFlushThreshold = cfgGetItem(pCfg, "ttlFlushThreshold")->i32;
   tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32;
+  tsRsyncPort = cfgGetItem(pCfg, "rsyncPort")->i32;
   tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN);
+  tstrncpy(tsSnodeAddress, cfgGetItem(pCfg, "snodeAddress")->str, TSDB_FQDN_LEN);
+  tstrncpy(tsCheckpointBackupDir, cfgGetItem(pCfg, "checkpointBackupDir")->str, PATH_MAX);
   tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32;
 
   tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32;
@@ -1277,7 +1301,7 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) {
       if (strcasecmp("keepColumnName", name) == 0) {
         tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
       } else if (strcasecmp("keepAliveIdle", name) == 0) {
-        tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->bval;
+        tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
       } else {
         matchItem = false;
       }
@@ -1670,6 +1694,7 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
         {"ttlBatchDropNum", &tsTtlBatchDropNum},
         {"ttlFlushThreshold", &tsTtlFlushThreshold},
         {"ttlPushInterval", &tsTtlPushIntervalSec},
+        //{"s3BlockSize", &tsS3BlockSize},
         {"s3BlockCacheSize", &tsS3BlockCacheSize},
         {"s3PageCacheSize", &tsS3PageCacheSize},
         {"s3UploadDelaySec", &tsS3UploadDelaySec},
@@ -1693,8 +1718,8 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
 
       switch (pItem->dtype) {
         case CFG_DTYPE_BOOL: {
-          int32_t  flag = atoi(value);
-          bool *pVar = options[d].optionVar;
+          int32_t flag = atoi(value);
+          bool   *pVar = options[d].optionVar;
           uInfo("%s set from %d to %d", optName, *pVar, flag);
           *pVar = flag;
         } break;
diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c
index 4b0848e5e9..a701c88a24 100644
--- a/source/common/src/ttime.c
+++ b/source/common/src/ttime.c
@@ -1580,6 +1580,7 @@ static bool needMoreDigits(SArray* formats, int32_t curIdx) {
 /// @retval 0 for success
 /// @retval -1 for format and s mismatch error
 /// @retval -2 if datetime err, like 2023-13-32 25:61:69
+/// @retval -3 if not supported
 static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t precision, const char** sErrPos,
                        int32_t* fErrIdx) {
   int32_t size = taosArrayGetSize(formats);
@@ -1589,6 +1590,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec
   int32_t hour = 0, min = 0, sec = 0, us = 0, ms = 0, ns = 0;
   int32_t tzSign = 1, tz = tsTimezone;
   int32_t err = 0;
+  bool    withYD = false, withMD = false;
 
   for (int32_t i = 0; i < size && *s != '\0'; ++i) {
     while (isspace(*s) && *s != '\0') {
@@ -1782,6 +1784,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec
         } else {
           s = newPos;
         }
+        withYD = true;
       } break;
       case TSFKW_DD: {
         const char* newPos = tsFormatStr2Int32(&md, s, 2, needMoreDigits(formats, i));
@@ -1790,6 +1793,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec
         } else {
           s = newPos;
         }
+        withMD = true;
       } break;
       case TSFKW_D: {
         const char* newPos = tsFormatStr2Int32(&wd, s, 1, needMoreDigits(formats, i));
@@ -1843,6 +1847,10 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec
       return err;
     }
   }
+  if (!withMD) {
+    // yyyy-mm-DDD, currently, the c api can't convert to correct timestamp, return not supported
+    if (withYD) return -3;
+  }
   struct STm tm = {0};
   tm.tm.tm_year = year - 1900;
   tm.tm.tm_mon = mon;
@@ -1892,8 +1900,13 @@ int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int
     TSFormatNode* fNode = (taosArrayGet(*formats, fErrIdx));
     snprintf(errMsg, errMsgLen, "mismatch format for: %s and %s", sErrPos,
              fErrIdx < taosArrayGetSize(*formats) ? ((TSFormatNode*)taosArrayGet(*formats, fErrIdx))->key->name : "");
+    code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR;
   } else if (code == -2) {
     snprintf(errMsg, errMsgLen, "timestamp format error: %s -> %s", tsStr, format);
+    code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR;
+  } else if (code == -3) {
+    snprintf(errMsg, errMsgLen, "timestamp format not supported");
+    code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED;
   }
   return code;
 }
diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp
index c65d8761b7..107276d7f9 100644
--- a/source/common/test/commonTests.cpp
+++ b/source/common/test/commonTests.cpp
@@ -489,6 +489,7 @@ TEST(timeTest, char2ts) {
   ASSERT_EQ(ts, 0);
   ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a a a 1/1+0"));
   ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a a a a a a a a a a a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a     "));
+  ASSERT_EQ(-3, TEST_char2ts("yyyy-mm-DDD", &ts, TSDB_TIME_PRECISION_MILLI, "1970-01-001"));
 }
 
 #pragma GCC diagnostic pop
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
index d2093ff77c..21b791eb4d 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
@@ -144,7 +144,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
   char path[TSDB_FILENAME_LEN] = {0};
   bool atExit = true;
 
-  if (vnodeIsLeader(pVnode->pImpl)) {
+  if (pVnode->pImpl && vnodeIsLeader(pVnode->pImpl)) {
     vnodeProposeCommitOnNeed(pVnode->pImpl, atExit);
   }
 
@@ -153,6 +153,10 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
   taosThreadRwlockUnlock(&pMgmt->lock);
   vmReleaseVnode(pMgmt, pVnode);
 
+  if (pVnode->failed) {
+    ASSERT(pVnode->pImpl == NULL);
+    goto _closed;
+  }
   dInfo("vgId:%d, pre close", pVnode->vgId);
   vnodePreClose(pVnode->pImpl);
 
@@ -202,6 +206,8 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
 
   vnodeClose(pVnode->pImpl);
   pVnode->pImpl = NULL;
+
+_closed:
   dInfo("vgId:%d, vnode is closed", pVnode->vgId);
 
   if (commitAndRemoveWal) {
@@ -386,7 +392,6 @@ static void *vmCloseVnodeInThread(void *param) {
 
   for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
     SVnodeObj *pVnode = pThread->ppVnodes[v];
-    if (pVnode->failed) continue;
 
     char stepDesc[TSDB_STEP_DESC_LEN] = {0};
     snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to close, %d of %d have been closed", pVnode->vgId,
diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt
index f1be20289a..6b875db860 100644
--- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt
+++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt
@@ -3,6 +3,17 @@ add_library(dnode STATIC ${IMPLEMENT_SRC})
 target_link_libraries(
     dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode
 )
+
+IF (TD_STORAGE)
+
+  IF(${BUILD_WITH_S3})
+    add_definitions(-DUSE_S3)
+  ELSEIF(${BUILD_WITH_COS})
+    add_definitions(-DUSE_COS)
+  ENDIF()
+
+ENDIF ()
+
 target_include_directories(
     dnode
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
index d560ba1644..6f13abcebc 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c
@@ -18,17 +18,17 @@
 #include "audit.h"
 #include "libs/function/tudf.h"
 
-#define DM_INIT_AUDIT()                 \
-  do {                                  \
-    auditCfg.port = tsMonitorPort;        \
-    auditCfg.server = tsMonitorFqdn;      \
-    auditCfg.comp = tsMonitorComp;      \
-    if (auditInit(&auditCfg) != 0) {    \
-      return -1;                        \
-    }                                   \
+#define DM_INIT_AUDIT()              \
+  do {                               \
+    auditCfg.port = tsMonitorPort;   \
+    auditCfg.server = tsMonitorFqdn; \
+    auditCfg.comp = tsMonitorComp;   \
+    if (auditInit(&auditCfg) != 0) { \
+      return -1;                     \
+    }                                \
   } while (0)
 
-static SDnode      globalDnode = {0};
+static SDnode globalDnode = {0};
 
 SDnode *dmInstance() { return &globalDnode; }
 
@@ -146,6 +146,13 @@ static bool dmCheckDataDirVersion() {
   return true;
 }
 
+#if defined(USE_S3)
+
+extern int32_t s3Begin();
+extern void    s3End();
+
+#endif
+
 int32_t dmInit() {
   dInfo("start to init dnode env");
   if (dmDiskInit() != 0) return -1;
@@ -156,6 +163,9 @@ int32_t dmInit() {
   if (dmInitMonitor() != 0) return -1;
   if (dmInitAudit() != 0) return -1;
   if (dmInitDnode(dmInstance()) != 0) return -1;
+#if defined(USE_S3)
+  if (s3Begin() != 0) return -1;
+#endif
 
   dInfo("dnode env is initialized");
   return 0;
@@ -181,6 +191,9 @@ void dmCleanup() {
   udfStopUdfd();
   taosStopCacheRefreshWorker();
   dmDiskClose();
+#if defined(USE_S3)
+  s3End();
+#endif
   dInfo("dnode env is cleaned up");
 
   taosCleanupCfg();
@@ -265,19 +278,19 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
 
   pWrapper = &pDnode->wrappers[ntype];
 
-  if(pWrapper->func.nodeRoleFp != NULL){
+  if (pWrapper->func.nodeRoleFp != NULL) {
     ESyncRole role = (*pWrapper->func.nodeRoleFp)(pWrapper->pMgmt);
     dInfo("node:%s, checking node role:%d", pWrapper->name, role);
-    if(role == TAOS_SYNC_ROLE_VOTER){
+    if (role == TAOS_SYNC_ROLE_VOTER) {
       dError("node:%s, failed to alter node type since node already is role:%d", pWrapper->name, role);
       terrno = TSDB_CODE_MNODE_ALREADY_IS_VOTER;
       return -1;
     }
   }
 
-  if(pWrapper->func.isCatchUpFp != NULL){
+  if (pWrapper->func.isCatchUpFp != NULL) {
     dInfo("node:%s, checking node catch up", pWrapper->name);
-    if((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1){
+    if ((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1) {
       terrno = TSDB_CODE_MNODE_NOT_CATCH_UP;
       return -1;
     }
@@ -394,7 +407,4 @@ void dmReportStartup(const char *pName, const char *pDesc) {
   dDebug("step:%s, %s", pStartup->name, pStartup->desc);
 }
 
-int64_t dmGetClusterId() {
-  return globalDnode.data.clusterId;
-}
-
+int64_t dmGetClusterId() { return globalDnode.data.clusterId; }
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index 85e4ef0fc2..f4108b52c6 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -1310,6 +1310,22 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
     }
     tFreeSMCfgDnodeReq(&cfgReq);
     return 0;
+  } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) {
+    int32_t optLen = strlen("s3blocksize");
+    int32_t flag = -1;
+    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
+    if (code < 0) return code;
+
+    if (flag > 1024 * 1024 || (flag > -1 && flag < 4) || flag < -1) {
+      mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [4, 1024 * 1024]",
+             cfgReq.dnodeId, flag);
+      terrno = TSDB_CODE_INVALID_CFG;
+      tFreeSMCfgDnodeReq(&cfgReq);
+      return -1;
+    }
+
+    strcpy(dcfgReq.config, "s3blocksize");
+    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
 #endif
   } else {
     mndMCfg2DCfg(&cfgReq, &dcfgReq);
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 0362b328ae..7c3c54537d 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -37,23 +37,23 @@
 
 typedef struct SNodeEntry {
   int32_t nodeId;
-  bool    stageUpdated; // the stage has been updated due to the leader/follower change or node reboot.
-  SEpSet  epset;        // compare the epset to identify the vgroup tranferring between different dnodes.
-  int64_t hbTimestamp;  // second
+  bool    stageUpdated;  // the stage has been updated due to the leader/follower change or node reboot.
+  SEpSet  epset;         // compare the epset to identify the vgroup tranferring between different dnodes.
+  int64_t hbTimestamp;   // second
 } SNodeEntry;
 
 typedef struct SStreamExecInfo {
   SArray       *pNodeList;
   int64_t       ts;                // snapshot ts
   int64_t       activeCheckpoint;  // active check point id
-  SHashObj     *pTaskMap;
-  SArray       *pTaskList;
+  SHashObj *    pTaskMap;
+  SArray *      pTaskList;
   TdThreadMutex lock;
 } SStreamExecInfo;
 
 typedef struct SVgroupChangeInfo {
   SHashObj *pDBMap;
-  SArray   *pUpdateNodeList;  // SArray<SNodeUpdateInfo>
+  SArray *  pUpdateNodeList;  // SArray<SNodeUpdateInfo>
 } SVgroupChangeInfo;
 
 static int32_t         mndNodeCheckSentinel = 0;
@@ -78,7 +78,7 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in
 static int32_t mndProcessNodeCheck(SRpcMsg *pReq);
 static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
 static SArray *extractNodeListFromStream(SMnode *pMnode);
-static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady);
+static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady);
 
 static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList);
 
@@ -91,7 +91,7 @@ static void    removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExe
 static void    keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
 static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot);
 static int32_t doKillActiveCheckpointTrans(SMnode *pMnode);
-static int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList);
+static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList);
 
 int32_t mndInitStream(SMnode *pMnode) {
   SSdbTable table = {
@@ -193,9 +193,9 @@ STREAM_ENCODE_OVER:
 
 SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
-  SSdbRow    *pRow = NULL;
+  SSdbRow *   pRow = NULL;
   SStreamObj *pStream = NULL;
-  void       *buf = NULL;
+  void *      buf = NULL;
 
   int8_t sver = 0;
   if (sdbGetRawSoftVer(pRaw, &sver) != 0) {
@@ -272,7 +272,7 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream
 }
 
 SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName) {
-  SSdb       *pSdb = pMnode->pSdb;
+  SSdb *      pSdb = pMnode->pSdb;
   SStreamObj *pStream = sdbAcquire(pSdb, SDB_STREAM, streamName);
   if (pStream == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
     terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
@@ -325,7 +325,7 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64
     return TSDB_CODE_SUCCESS;
   }
 
-  SNode  *pAst = NULL;
+  SNode * pAst = NULL;
   int32_t code = nodesStringToNode(ast, &pAst);
 
   SQueryPlan *pPlan = NULL;
@@ -350,7 +350,7 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64
 }
 
 static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SCMCreateStreamReq *pCreate) {
-  SNode      *pAst = NULL;
+  SNode *     pAst = NULL;
   SQueryPlan *pPlan = NULL;
 
   mInfo("stream:%s to create", pCreate->name);
@@ -589,7 +589,7 @@ int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr
 
 static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) {
   SStbObj *pStb = NULL;
-  SDbObj  *pDb = NULL;
+  SDbObj * pDb = NULL;
 
   SMCreateStbReq createReq = {0};
   tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN);
@@ -715,10 +715,10 @@ int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream)
 }
 
 static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
-  SMnode            *pMnode = pReq->info.node;
+  SMnode *           pMnode = pReq->info.node;
   int32_t            code = -1;
-  SStreamObj        *pStream = NULL;
-  SDbObj            *pDb = NULL;
+  SStreamObj *       pStream = NULL;
+  SDbObj *           pDb = NULL;
   SCMCreateStreamReq createStreamReq = {0};
   SStreamObj         streamObj = {0};
 
@@ -761,7 +761,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
     int32_t numOfStream = 0;
 
     SStreamObj *pStream = NULL;
-    void       *pIter = NULL;
+    void *      pIter = NULL;
 
     while (1) {
       pIter = sdbFetch(pMnode->pSdb, SDB_STREAM, pIter, (void **)&pStream);
@@ -858,12 +858,12 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
 
   SName name = {0};
   tNameFromString(&name, createStreamReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-  //reuse this function for stream
+  // reuse this function for stream
 
-  //TODO
+  // TODO
   if (createStreamReq.sql != NULL) {
-    auditRecord(pReq, pMnode->clusterId, "createStream", name.dbname, name.tname, 
-                createStreamReq.sql, strlen(createStreamReq.sql));
+    auditRecord(pReq, pMnode->clusterId, "createStream", name.dbname, name.tname, createStreamReq.sql,
+                strlen(createStreamReq.sql));
   }
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
@@ -877,15 +877,30 @@ _OVER:
   return code;
 }
 
+int64_t mndStreamGenChkpId(SMnode *pMnode) {
+  SStreamObj *pStream = NULL;
+  void *      pIter = NULL;
+  SSdb *      pSdb = pMnode->pSdb;
+  int64_t     maxChkpId = 0;
+  while (1) {
+    pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
+    if (pIter == NULL) break;
+
+    maxChkpId = TMAX(maxChkpId, pStream->checkpointId);
+    sdbRelease(pSdb, pStream);
+  }
+  return maxChkpId + 1;
+}
+
 static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
   SMnode *pMnode = pReq->info.node;
-  SSdb   *pSdb = pMnode->pSdb;
+  SSdb *  pSdb = pMnode->pSdb;
   if (sdbGetSize(pSdb, SDB_STREAM) <= 0) {
     return 0;
   }
 
   SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
-  pMsg->checkpointId = taosGetTimestampMs();
+  pMsg->checkpointId = mndStreamGenChkpId(pMnode);
 
   int32_t size = sizeof(SMStreamDoCheckpointMsg);
   SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size};
@@ -919,7 +934,7 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in
     return -1;
   }
 
-  void    *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+  void *   abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
   SEncoder encoder;
   tEncoderInit(&encoder, abuf, tlen);
   tEncodeStreamCheckpointSourceReq(&encoder, &req);
@@ -1042,7 +1057,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
 
   int32_t totLevel = taosArrayGetSize(pStream->tasks);
   for (int32_t i = 0; i < totLevel; i++) {
-    SArray      *pLevel = taosArrayGetP(pStream->tasks, i);
+    SArray *     pLevel = taosArrayGetP(pStream->tasks, i);
     SStreamTask *pTask = taosArrayGetP(pLevel, 0);
 
     if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@@ -1059,7 +1074,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
           return -1;
         }
 
-        void   *buf;
+        void *  buf;
         int32_t tlen;
         if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, chkptId, pTask->id.streamId,
                                                pTask->id.taskId) < 0) {
@@ -1070,7 +1085,8 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
 
         STransAction action = {0};
         SEpSet       epset = mndGetVgroupEpset(pMnode, pVgObj);
-        initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY);
+        initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset,
+                        TSDB_CODE_SYN_PROPOSE_NOT_READY);
         mndReleaseVgroup(pMnode, pVgObj);
 
         if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@@ -1110,9 +1126,9 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
 }
 
 static const char *mndGetStreamDB(SMnode *pMnode) {
-  SSdb       *pSdb = pMnode->pSdb;
+  SSdb *      pSdb = pMnode->pSdb;
   SStreamObj *pStream = NULL;
-  void       *pIter = NULL;
+  void *      pIter = NULL;
 
   pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
   if (pIter == NULL) {
@@ -1135,9 +1151,9 @@ static int32_t initStreamNodeList(SMnode* pMnode) {
 }
 
 static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
-  SMnode     *pMnode = pReq->info.node;
-  SSdb       *pSdb = pMnode->pSdb;
-  void       *pIter = NULL;
+  SMnode *    pMnode = pReq->info.node;
+  SSdb *      pSdb = pMnode->pSdb;
+  void *      pIter = NULL;
   SStreamObj *pStream = NULL;
   int32_t     code = 0;
 
@@ -1162,11 +1178,10 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
       }
     }
 
-    bool allReady = true;
+    bool    allReady = true;
     SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady);
     if (!allReady) {
-      mWarn("not all vnodes are ready, ignore the checkpoint")
-      taosArrayDestroy(pNodeSnapshot);
+      mWarn("not all vnodes are ready, ignore the checkpoint") taosArrayDestroy(pNodeSnapshot);
       return 0;
     }
 
@@ -1187,15 +1202,15 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
 
     taosThreadMutexLock(&execInfo.lock);
     for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
-      STaskId *p = taosArrayGet(execInfo.pTaskList, i);
-      STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
+      STaskId *         p = taosArrayGet(execInfo.pTaskList, i);
+      STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
       if (pEntry == NULL) {
         continue;
       }
 
       if (pEntry->status != TASK_STATUS__READY) {
         mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued",
-              pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamTaskGetStatusStr(pEntry->status));
+               pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamTaskGetStatusStr(pEntry->status));
         ready = false;
         break;
       }
@@ -1255,7 +1270,7 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
 }
 
 static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
-  SMnode     *pMnode = pReq->info.node;
+  SMnode *    pMnode = pReq->info.node;
   SStreamObj *pStream = NULL;
 
   SMDropStreamReq dropReq = {0};
@@ -1332,7 +1347,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
 
   SName name = {0};
   tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-  //reuse this function for stream
+  // reuse this function for stream
 
   auditRecord(pReq, pMnode->clusterId, "dropStream", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen);
 
@@ -1384,7 +1399,7 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
 }
 
 int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) {
-  SSdb   *pSdb = pMnode->pSdb;
+  SSdb *  pSdb = pMnode->pSdb;
   SDbObj *pDb = mndAcquireDb(pMnode, dbName);
   if (pDb == NULL) {
     terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
@@ -1392,7 +1407,7 @@ int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams)
   }
 
   int32_t numOfStreams = 0;
-  void   *pIter = NULL;
+  void *  pIter = NULL;
   while (1) {
     SStreamObj *pStream = NULL;
     pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
@@ -1411,8 +1426,8 @@ int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams)
 }
 
 static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
-  SMnode     *pMnode = pReq->info.node;
-  SSdb       *pSdb = pMnode->pSdb;
+  SMnode *    pMnode = pReq->info.node;
+  SSdb *      pSdb = pMnode->pSdb;
   int32_t     numOfRows = 0;
   SStreamObj *pStream = NULL;
 
@@ -1488,8 +1503,8 @@ static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) {
 }
 
 static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
-  SMnode     *pMnode = pReq->info.node;
-  SSdb       *pSdb = pMnode->pSdb;
+  SMnode *    pMnode = pReq->info.node;
+  SSdb *      pSdb = pMnode->pSdb;
   int32_t     numOfRows = 0;
   SStreamObj *pStream = NULL;
 
@@ -1578,13 +1593,13 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         // status
         char status[20 + VARSTR_HEADER_SIZE] = {0};
 
-        STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
-        STaskStatusEntry* pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
+        STaskId           id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
+        STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
         if (pe == NULL) {
           continue;
         }
 
-        const char* pStatus = streamTaskGetStatusStr(pe->status);
+        const char *pStatus = streamTaskGetStatusStr(pe->status);
         STR_TO_VARSTR(status, pStatus);
 
         // status
@@ -1596,24 +1611,24 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         colDataSetVal(pColInfo, numOfRows, (const char *)&pe->stage, false);
 
         // input queue
-        char vbuf[30] = {0};
-        char buf[25] = {0};
-        const char* queueInfoStr = "%4.2fMiB (%5.2f%)";
+        char        vbuf[30] = {0};
+        char        buf[25] = {0};
+        const char *queueInfoStr = "%4.2fMiB (%5.2f%)";
         sprintf(buf, queueInfoStr, pe->inputQUsed, pe->inputRate);
         STR_TO_VARSTR(vbuf, buf);
 
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
 
         // output queue
-//        sprintf(buf, queueInfoStr, pe->outputQUsed, pe->outputRate);
-//        STR_TO_VARSTR(vbuf, buf);
+        //        sprintf(buf, queueInfoStr, pe->outputQUsed, pe->outputRate);
+        //        STR_TO_VARSTR(vbuf, buf);
 
-//        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-//        colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false);
+        //        pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+        //        colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false);
 
         if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
-          const char* sinkStr = "%.2fMiB";
+          const char *sinkStr = "%.2fMiB";
           sprintf(buf, sinkStr, pe->sinkDataSize);
         } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
           // offset info
@@ -1624,7 +1639,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
         STR_TO_VARSTR(vbuf, buf);
 
         pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-        colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false);
+        colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
 
         numOfRows++;
       }
@@ -1668,7 +1683,7 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) {
 }
 
 int32_t mndPauseAllStreamTasks(STrans *pTrans, SStreamObj *pStream) {
-  SArray* tasks = pStream->tasks;
+  SArray *tasks = pStream->tasks;
 
   int32_t size = taosArrayGetSize(tasks);
   for (int32_t i = 0; i < size; i++) {
@@ -1705,7 +1720,7 @@ static int32_t mndPersistStreamLog(STrans *pTrans, const SStreamObj *pStream, in
 }
 
 static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
-  SMnode     *pMnode = pReq->info.node;
+  SMnode *    pMnode = pReq->info.node;
   SStreamObj *pStream = NULL;
 
   SMPauseStreamReq pauseReq = {0};
@@ -1821,7 +1836,7 @@ int32_t mndResumeAllStreamTasks(STrans *pTrans, SStreamObj *pStream, int8_t igUn
 }
 
 static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
-  SMnode     *pMnode = pReq->info.node;
+  SMnode *    pMnode = pReq->info.node;
   SStreamObj *pStream = NULL;
 
   SMResumeStreamReq pauseReq = {0};
@@ -1906,7 +1921,7 @@ static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChang
 }
 
 static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId,
-                                          SStreamTaskId* pId, int32_t transId) {
+                                          SStreamTaskId *pId, int32_t transId) {
   SStreamTaskNodeUpdateMsg req = {0};
   initNodeUpdateMsg(&req, pInfo, pId, transId);
 
@@ -1929,7 +1944,7 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
     return -1;
   }
 
-  void    *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+  void *   abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
   SEncoder encoder;
   tEncoderInit(&encoder, abuf, tlen);
   tEncodeStreamTaskUpdateMsg(&encoder, &req);
@@ -1996,7 +2011,7 @@ static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *p
     for (int32_t k = 0; k < numOfTasks; ++k) {
       SStreamTask *pTask = taosArrayGetP(pLevel, k);
 
-      void   *pBuf = NULL;
+      void *  pBuf = NULL;
       int32_t len = 0;
       streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
       doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
@@ -2017,7 +2032,7 @@ static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *p
 
 static bool isNodeEpsetChanged(const SEpSet *pPrevEpset, const SEpSet *pCurrent) {
   const SEp *pEp = GET_ACTIVE_EP(pPrevEpset);
-  const SEp* p = GET_ACTIVE_EP(pCurrent);
+  const SEp *p = GET_ACTIVE_EP(pCurrent);
 
   if (pEp->port == p->port && strncmp(pEp->fqdn, p->fqdn, TSDB_FQDN_LEN) == 0) {
     return false;
@@ -2071,9 +2086,9 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP
   return info;
 }
 
-static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady) {
-  SSdb   *pSdb = pMnode->pSdb;
-  void   *pIter = NULL;
+static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
+  SSdb *  pSdb = pMnode->pSdb;
+  void *  pIter = NULL;
   SVgObj *pVgroup = NULL;
 
   *allReady = true;
@@ -2121,8 +2136,8 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
 
   // check all streams that involved this vnode should update the epset info
   SStreamObj *pStream = NULL;
-  void       *pIter = NULL;
-  STrans     *pTrans = NULL;
+  void *      pIter = NULL;
+  STrans *    pTrans = NULL;
 
   while (1) {
     pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
@@ -2183,9 +2198,9 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
 }
 
 static SArray *extractNodeListFromStream(SMnode *pMnode) {
-  SSdb       *pSdb = pMnode->pSdb;
+  SSdb *      pSdb = pMnode->pSdb;
   SStreamObj *pStream = NULL;
-  void       *pIter = NULL;
+  void *      pIter = NULL;
 
   SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
   while (1) {
@@ -2232,9 +2247,9 @@ static SArray *extractNodeListFromStream(SMnode *pMnode) {
 }
 
 static void doExtractTasksFromStream(SMnode *pMnode) {
-  SSdb       *pSdb = pMnode->pSdb;
+  SSdb *      pSdb = pMnode->pSdb;
   SStreamObj *pStream = NULL;
-  void       *pIter = NULL;
+  void *      pIter = NULL;
 
   while (1) {
     pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
@@ -2269,11 +2284,11 @@ static int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
   return TSDB_CODE_SUCCESS;
 }
 
-static bool taskNodeExists(SArray* pList, int32_t nodeId) {
+static bool taskNodeExists(SArray *pList, int32_t nodeId) {
   size_t num = taosArrayGetSize(pList);
 
-  for(int32_t i = 0; i < num; ++i) {
-    SNodeEntry* pEntry = taosArrayGet(pList, i);
+  for (int32_t i = 0; i < num; ++i) {
+    SNodeEntry *pEntry = taosArrayGet(pList, i);
     if (pEntry->nodeId == nodeId) {
       return true;
     }
@@ -2283,12 +2298,12 @@ static bool taskNodeExists(SArray* pList, int32_t nodeId) {
 }
 
 int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
-  SArray* pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
+  SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
 
   int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
-  for(int32_t i = 0; i < numOfTask; ++i) {
-    STaskId* pId = taosArrayGet(execInfo.pTaskList, i);
-    STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
+  for (int32_t i = 0; i < numOfTask; ++i) {
+    STaskId *         pId = taosArrayGet(execInfo.pTaskList, i);
+    STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
 
     bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
     if (!existed) {
@@ -2296,21 +2311,21 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
     }
   }
 
-  for(int32_t i = 0; i < taosArrayGetSize(pRemovedTasks); ++i) {
-    STaskId* pId = taosArrayGet(pRemovedTasks, i);
+  for (int32_t i = 0; i < taosArrayGetSize(pRemovedTasks); ++i) {
+    STaskId *pId = taosArrayGet(pRemovedTasks, i);
     doRemoveTasks(&execInfo, pId);
   }
 
   mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
-         (int32_t) taosArrayGetSize(execInfo.pTaskList));
+         (int32_t)taosArrayGetSize(execInfo.pTaskList));
 
   int32_t size = taosArrayGetSize(pNodeSnapshot);
   SArray* pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry));
   for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeList); ++i) {
     SNodeEntry* p = taosArrayGet(execInfo.pNodeList, i);
 
-    for(int32_t j = 0; j < size; ++j) {
-      SNodeEntry* pEntry = taosArrayGet(pNodeSnapshot, j);
+    for (int32_t j = 0; j < size; ++j) {
+      SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j);
       if (pEntry->nodeId == p->nodeId) {
         taosArrayPush(pValidNodeEntryList, p);
         break;
@@ -2321,7 +2336,7 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
   taosArrayDestroy(execInfo.pNodeList);
   execInfo.pNodeList = pValidNodeEntryList;
 
-  mDebug("remain %d valid node entries", (int32_t) taosArrayGetSize(pValidNodeEntryList));
+  mDebug("remain %d valid node entries", (int32_t)taosArrayGetSize(pValidNodeEntryList));
   taosArrayDestroy(pRemovedTasks);
   return 0;
 }
@@ -2366,7 +2381,6 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
 
   SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeList, pNodeSnapshot);
   if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) {
-
     // kill current active checkpoint transaction, since the transaction is vnode wide.
     doKillActiveCheckpointTrans(pMnode);
     code = mndProcessVgroupChange(pMnode, &changeInfo);
@@ -2401,7 +2415,7 @@ typedef struct SMStreamNodeCheckMsg {
 
 static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
   SMnode *pMnode = pReq->info.node;
-  SSdb   *pSdb = pMnode->pSdb;
+  SSdb *  pSdb = pMnode->pSdb;
   if (sdbGetSize(pSdb, SDB_STREAM) <= 0) {
     return 0;
   }
@@ -2425,7 +2439,7 @@ void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
       SStreamTask *pTask = taosArrayGetP(pLevel, j);
 
       STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
-      void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
+      void *  p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
       if (p == NULL) {
         STaskStatusEntry entry = {0};
         streamTaskStatusInit(&entry, pTask);
@@ -2439,7 +2453,7 @@ void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
   }
 }
 
-void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) {
+void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
   int32_t level = taosArrayGetSize(pStream->tasks);
   for (int32_t i = 0; i < level; i++) {
     SArray *pLevel = taosArrayGetP(pStream->tasks, i);
@@ -2449,12 +2463,12 @@ void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) {
       SStreamTask *pTask = taosArrayGetP(pLevel, j);
 
       STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
-      void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
+      void *  p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id));
       if (p != NULL) {
         taosHashRemove(pExecNode->pTaskMap, &id, sizeof(id));
 
-        for(int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
-          STaskId* pId = taosArrayGet(pExecNode->pTaskList, k);
+        for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
+          STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
           if (pId->taskId == id.taskId && pId->streamId == id.streamId) {
             taosArrayRemove(pExecNode->pTaskList, k);
             mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t)id.taskId,
@@ -2462,7 +2476,6 @@ void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) {
             break;
           }
         }
-
       }
     }
   }
@@ -2492,7 +2505,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, const char *name) {
   return pTrans;
 }
 
-int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) {
+int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) {
   STrans *pTrans = doCreateTrans(pMnode, pStream, "stream-task-reset");
   if (pTrans == NULL) {
     return terrno;
@@ -2509,7 +2522,7 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) {
       SStreamTask *pTask = taosArrayGetP(pLevel, k);
 
       // todo extract method, with pause stream task
-      SVResetStreamTaskReq* pReq = taosMemoryCalloc(1, sizeof(SVResetStreamTaskReq));
+      SVResetStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResetStreamTaskReq));
       if (pReq == NULL) {
         terrno = TSDB_CODE_OUT_OF_MEMORY;
         mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVResetStreamTaskReq),
@@ -2555,9 +2568,9 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) {
 
 int32_t doKillActiveCheckpointTrans(SMnode *pMnode) {
   int32_t transId = 0;
-  SSdb   *pSdb = pMnode->pSdb;
+  SSdb *  pSdb = pMnode->pSdb;
   STrans *pTrans = NULL;
-  void   *pIter = NULL;
+  void *  pIter = NULL;
 
   while (1) {
     pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pTrans);
@@ -2588,13 +2601,13 @@ int32_t doKillActiveCheckpointTrans(SMnode *pMnode) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t mndResetFromCheckpoint(SMnode* pMnode) {
+int32_t mndResetFromCheckpoint(SMnode *pMnode) {
   doKillActiveCheckpointTrans(pMnode);
 
   // set all tasks status to be normal, refactor later to be stream level, instead of vnode level.
-  SSdb *pSdb = pMnode->pSdb;
+  SSdb *      pSdb = pMnode->pSdb;
   SStreamObj *pStream = NULL;
-  void       *pIter = NULL;
+  void *      pIter = NULL;
   while (1) {
     pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
     if (pIter == NULL) {
@@ -2613,7 +2626,7 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
   return 0;
 }
 
-int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList) {
+int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) {
   int32_t num = taosArrayGetSize(pNodeList);
   mInfo("set node expired for %d nodes", num);
 
@@ -2641,7 +2654,6 @@ static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) {
   for(int32_t j = 0; j < numOfNodes; ++j) {
     SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeList, j);
     if (pNodeEntry->nodeId == pTaskEntry->nodeId) {
-
       mInfo("vgId:%d stage updated from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId,
             pTaskEntry->stage, stage, pTaskEntry->id.taskId);
 
@@ -2653,7 +2665,7 @@ static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) {
 }
 
 int32_t mndProcessStreamHb(SRpcMsg *pReq) {
-  SMnode      *pMnode = pReq->info.node;
+  SMnode *     pMnode = pReq->info.node;
   SStreamHbMsg req = {0};
 
   bool    checkpointFailed = false;
@@ -2714,15 +2726,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
 
     pTaskEntry->status = p->status;
     if (p->status != TASK_STATUS__READY) {
-      mDebug("received s-task:0x%"PRIx64" not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status));
+      mDebug("received s-task:0x%" PRIx64 " not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status));
     }
   }
 
   // current checkpoint is failed, rollback from the checkpoint trans
   // kill the checkpoint trans and then set all tasks status to be normal
   if (checkpointFailed && activeCheckpointId != 0) {
-    bool allReady = true;
-    SArray* p = mndTakeVgroupSnapshot(pMnode, &allReady);
+    bool    allReady = true;
+    SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady);
     taosArrayDestroy(p);
 
     if (allReady) {
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index f2ef00c534..6f5b370826 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "rsync.h"
 #include "executor.h"
 #include "sndInt.h"
 #include "tstream.h"
@@ -122,6 +123,9 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
     goto FAIL;
   }
 
+  stopRsync();
+  startRsync();
+
   // todo fix it: send msg to mnode to rollback to an existed checkpoint
   streamMetaInitForSnode(pSnode->pMeta);
   return pSnode;
diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index 2a29a10822..dc43da7fe7 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -8,7 +8,6 @@ set(
   "src/vnd/vnodeCommit.c"
   "src/vnd/vnodeQuery.c"
   "src/vnd/vnodeModule.c"
-  "src/vnd/vnodeCos.c"
   "src/vnd/vnodeSvr.c"
   "src/vnd/vnodeSync.c"
   "src/vnd/vnodeSnapshot.c"
@@ -161,75 +160,6 @@ target_link_libraries(
     PUBLIC index
 )
 
-if(${BUILD_S3})
-
-if(${BUILD_WITH_S3})
-  target_include_directories(
-    vnode
-
-    PUBLIC "$ENV{HOME}/.cos-local.2/include"
-    )
-
-  set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
-  set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2)
-  find_library(S3_LIBRARY s3)
-  find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
-  find_library(XML2_LIBRARY xml2)
-  find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
-  find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
-  target_link_libraries(
-    vnode
-
-    # s3
-    PUBLIC ${S3_LIBRARY}
-    PUBLIC ${CURL_LIBRARY}
-    PUBLIC ${SSL_LIBRARY}
-    PUBLIC ${CRYPTO_LIBRARY}
-    PUBLIC ${XML2_LIBRARY}
-  )
-
-  add_definitions(-DUSE_S3)
-endif()
-
-if(${BUILD_WITH_COS})
-
-set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
-find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/)
-find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/)
-find_library(MINIXML_LIBRARY mxml)
-find_library(CURL_LIBRARY curl)
-target_link_libraries(
-    vnode
-
-    # s3
-    PUBLIC cos_c_sdk_static
-    PUBLIC ${APR_UTIL_LIBRARY}
-    PUBLIC ${APR_LIBRARY}
-    PUBLIC ${MINIXML_LIBRARY}
-    PUBLIC ${CURL_LIBRARY}
-)
-
-# s3
-FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/)
-IF (APR_CONFIG_BIN)
-  EXECUTE_PROCESS(
-    COMMAND ${APR_CONFIG_BIN} --includedir
-    OUTPUT_VARIABLE APR_INCLUDE_DIR
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-ENDIF()
-include_directories (${APR_INCLUDE_DIR})
-target_include_directories(
-  vnode
-  PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk"
-  PUBLIC "$ENV{HOME}/.cos-local.1/include"
-  )
-
-  add_definitions(-DUSE_COS)
-endif(${BUILD_WITH_COS})
-
-endif()
-
 IF (TD_GRANT)
   TARGET_LINK_LIBRARIES(vnode PUBLIC grant)
 ENDIF ()
diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h
index aaf0973b41..29eaa0509a 100644
--- a/source/dnode/vnode/src/inc/sma.h
+++ b/source/dnode/vnode/src/inc/sma.h
@@ -108,6 +108,7 @@ struct SRSmaStat {
   int64_t          refId;        // shared by fetch tasks
   volatile int64_t nBufItems;    // number of items in queue buffer
   SRWLatch         lock;         // r/w lock for rsma fs(e.g. qtaskinfo)
+  volatile int32_t execStat;     // 0 succeed, other failed
   volatile int32_t nFetchAll;    // active number of fetch all
   volatile int8_t  triggerStat;  // shared by fetch tasks
   volatile int8_t  commitStat;   // 0 not in committing, 1 in committing
@@ -115,6 +116,7 @@ struct SRSmaStat {
   SRSmaFS          fs;           // for recovery/snapshot r/w
   SHashObj        *infoHash;     // key: suid, value: SRSmaInfo
   tsem_t           notEmpty;     // has items in queue buffer
+  SArray          *blocks;       // SArray<SSDataBlock>
 };
 
 struct SSmaStat {
@@ -136,13 +138,18 @@ struct SSmaStat {
 #define RSMA_FS_LOCK(r)      (&(r)->lock)
 
 struct SRSmaInfoItem {
-  int8_t   level : 4;
-  int8_t   fetchLevel : 4;
-  int8_t   triggerStat;
-  uint16_t nScanned;
-  int32_t  maxDelay;  // ms
-  tmr_h    tmrId;
-  void    *pStreamState;
+  int8_t  level;
+  int8_t  fetchLevel;
+  int8_t  triggerStat;
+  int32_t nScanned;
+  int32_t streamFlushed : 1;
+  int32_t maxDelay : 31;  // ms
+  int64_t submitReqVer;
+  int64_t fetchResultVer;
+  tmr_h   tmrId;
+  void   *pStreamState;
+  void   *pStreamTask;  // SStreamTask
+  SArray *pResList;
 };
 
 struct SRSmaInfo {
@@ -160,12 +167,10 @@ struct SRSmaInfo {
   STaosQall    *qall;                         // buffer qall of SubmitReq
 };
 
-#define RSMA_INFO_HEAD_LEN     offsetof(SRSmaInfo, items)
-#define RSMA_INFO_IS_DEL(r)    ((r)->delFlag == 1)
-#define RSMA_INFO_SET_DEL(r)   ((r)->delFlag = 1)
-#define RSMA_INFO_QTASK(r, i)  ((r)->taskInfo[i])
-#define RSMA_INFO_IQTASK(r, i) ((r)->iTaskInfo[i])
-#define RSMA_INFO_ITEM(r, i)   (&(r)->items[i])
+#define RSMA_INFO_IS_DEL(r)   ((r)->delFlag == 1)
+#define RSMA_INFO_SET_DEL(r)  ((r)->delFlag = 1)
+#define RSMA_INFO_QTASK(r, i) ((r)->taskInfo[i])
+#define RSMA_INFO_ITEM(r, i)  (&(r)->items[i])
 
 enum {
   TASK_TRIGGER_STAT_INIT = 0,
@@ -214,11 +219,11 @@ static FORCE_INLINE void tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) {
 int32_t smaPreClose(SSma *pSma);
 
 // rsma
-void   *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree);
+void   *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo);
 int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback);
 int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
 int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type);
-// int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash);
+int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash);
 int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback);
 void    tdRSmaQTaskInfoGetFullPath(SVnode *pVnode, tb_uid_t suid, int8_t level, STfs *pTfs, char *outputName);
 
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 2a9510f6ad..264c64e0fc 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -209,6 +209,7 @@ int32_t tsdbBegin(STsdb* pTsdb);
 // int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo);
 int32_t tsdbCacheCommit(STsdb* pTsdb);
 int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo);
+int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync);
 // int32_t tsdbFinishCommit(STsdb* pTsdb);
 // int32_t tsdbRollbackCommit(STsdb* pTsdb);
 int     tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg);
@@ -279,7 +280,7 @@ int32_t smaPrepareAsyncCommit(SSma* pSma);
 int32_t smaCommit(SSma* pSma, SCommitInfo* pInfo);
 int32_t smaFinishCommit(SSma* pSma);
 int32_t smaPostCommit(SSma* pSma);
-int32_t smaDoRetention(SSma* pSma, int64_t now);
+int32_t smaRetention(SSma* pSma, int64_t now);
 
 int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
 int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c
index c26157f4b7..92b8c09fbc 100644
--- a/source/dnode/vnode/src/sma/smaCommit.c
+++ b/source/dnode/vnode/src/sma/smaCommit.c
@@ -156,10 +156,10 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) {
   nLoops = 0;
   while (1) {
     if (atomic_load_32(&pRSmaStat->nFetchAll) <= 0) {
-      smaDebug("vgId:%d, rsma commit:%d, fetch tasks are all finished", SMA_VID(pSma), isCommit);
+      smaDebug("vgId:%d, rsma commit, type:%d, fetch tasks are all finished", SMA_VID(pSma), isCommit);
       break;
     } else {
-      smaDebug("vgId:%d, rsma commit%d, fetch tasks are not all finished yet", SMA_VID(pSma), isCommit);
+      smaDebug("vgId:%d, rsma commit, type:%d, fetch tasks are not all finished yet", SMA_VID(pSma), isCommit);
     }
     TD_SMA_LOOPS_CHECK(nLoops, 1000);
   }
@@ -169,22 +169,24 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) {
    *  1) This is high cost task and should not put in asyncPreCommit originally.
    *  2) But, if put in asyncCommit, would trigger taskInfo cloning frequently.
    */
-  smaInfo("vgId:%d, rsma commit:%d, wait for all items to be consumed, TID:%p", SMA_VID(pSma), isCommit,
+  smaInfo("vgId:%d, rsma commit, type:%d, wait for all items to be consumed, TID:%p", SMA_VID(pSma), isCommit,
           (void *)taosGetSelfPthreadId());
   nLoops = 0;
   while (atomic_load_64(&pRSmaStat->nBufItems) > 0) {
     TD_SMA_LOOPS_CHECK(nLoops, 1000);
   }
+  smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
 
   if (!isCommit) goto _exit;
 
-  // code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat));
+  code = atomic_load_32(&pRSmaStat->execStat);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat));
   TSDB_CHECK_CODE(code, lino, _exit);
 
   smaInfo("vgId:%d, rsma commit, operator state committed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
 
-  smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId());
-
   // all rsma results are written completely
   STsdb *pTsdb = NULL;
   if ((pTsdb = VND_RSMA1(pSma->pVnode))) {
diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c
index 04a254fc7a..dd12f2bca2 100644
--- a/source/dnode/vnode/src/sma/smaEnv.c
+++ b/source/dnode/vnode/src/sma/smaEnv.c
@@ -179,7 +179,7 @@ static void tRSmaInfoHashFreeNode(void *data) {
     if ((pItem = RSMA_INFO_ITEM((SRSmaInfo *)pRSmaInfo, 1)) && pItem->level) {
       taosHashRemove(smaMgmt.refHash, &pItem, POINTER_BYTES);
     }
-    tdFreeRSmaInfo(pRSmaInfo->pSma, pRSmaInfo, true);
+    tdFreeRSmaInfo(pRSmaInfo->pSma, pRSmaInfo);
   }
 }
 
@@ -209,6 +209,12 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
       pRSmaStat->pSma = (SSma *)pSma;
       atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT);
       tsem_init(&pRSmaStat->notEmpty, 0, 0);
+      if (!(pRSmaStat->blocks = taosArrayInit(1, sizeof(SSDataBlock)))) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        TSDB_CHECK_CODE(code, lino, _exit);
+      }
+      SSDataBlock datablock = {.info.type = STREAM_CHECKPOINT};
+      taosArrayPush(pRSmaStat->blocks, &datablock);
 
       // init smaMgmt
       smaInit();
@@ -290,6 +296,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
 
     // step 5: free pStat
     tsem_destroy(&(pStat->notEmpty));
+    taosArrayDestroy(pStat->blocks);
     taosMemoryFreeClear(pStat);
   }
 }
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index 14c5baa402..252a3ade36 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -15,12 +15,14 @@
 
 #include "sma.h"
 #include "tq.h"
+#include "tstream.h"
 
 #define RSMA_QTASKEXEC_SMOOTH_SIZE (100)     // cnt
 #define RSMA_SUBMIT_BATCH_SIZE     (1024)    // cnt
 #define RSMA_FETCH_DELAY_MAX       (120000)  // ms
 #define RSMA_FETCH_ACTIVE_MAX      (1000)    // ms
 #define RSMA_FETCH_INTERVAL        (5000)    // ms
+#define RSMA_TASK_FLAG             "rsma"
 
 #define RSMA_NEED_FETCH(r) (RSMA_INFO_ITEM((r), 0)->fetchLevel || RSMA_INFO_ITEM((r), 1)->fetchLevel)
 
@@ -42,8 +44,8 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid);
 static void       tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo);
 static void       tdFreeRSmaSubmitItems(SArray *pItems);
 static int32_t    tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo);
-static int32_t    tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema,
-                                            int64_t suid);
+static int32_t    tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, SRSmaInfo *pInfo,
+                                            int32_t execType, int8_t *streamFlushed);
 static void       tdRSmaFetchTrigger(void *param, void *tmrId);
 static void       tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level);
 static int32_t    tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables);
@@ -72,41 +74,39 @@ static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t l
  *
  * @param pSma
  * @param pInfo
- * @param isDeepFree Only stop tmrId and free pTSchema for deep free
  * @return void*
  */
-void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) {
+void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) {
   if (pInfo) {
     for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
 
-      if (isDeepFree && pItem->tmrId) {
+      if (pItem->tmrId) {
         smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pItem->tmrId,
                  pInfo->suid, i + 1);
         taosTmrStopA(&pItem->tmrId);
       }
 
-      if (isDeepFree && pItem->pStreamState) {
+      if (pItem->pStreamState) {
         streamStateClose(pItem->pStreamState, false);
       }
 
-      if (isDeepFree && pInfo->taskInfo[i]) {
-        tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1);
+      if (pItem->pStreamTask) {
+        tFreeStreamTask(pItem->pStreamTask);
       }
-    }
-    if (isDeepFree) {
-      taosMemoryFreeClear(pInfo->pTSchema);
+      taosArrayDestroy(pItem->pResList);
+      tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1);
     }
 
-    if (isDeepFree) {
-      if (pInfo->queue) {
-        taosCloseQueue(pInfo->queue);
-        pInfo->queue = NULL;
-      }
-      if (pInfo->qall) {
-        taosFreeQall(pInfo->qall);
-        pInfo->qall = NULL;
-      }
+    taosMemoryFreeClear(pInfo->pTSchema);
+
+    if (pInfo->queue) {
+      taosCloseQueue(pInfo->queue);
+      pInfo->queue = NULL;
+    }
+    if (pInfo->qall) {
+      taosFreeQall(pInfo->qall);
+      pInfo->qall = NULL;
     }
 
     taosMemoryFree(pInfo);
@@ -135,7 +135,9 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids,
     return TSDB_CODE_FAILED;
   }
 
-  if (!taosArrayGetSize(tbUids)) {
+  int32_t nTables = taosArrayGetSize(tbUids);
+
+  if (0 == nTables) {
     smaDebug("vgId:%d, no need to update tbUidList for suid:%" PRIi64 " since Empty tbUids", SMA_VID(pSma), *suid);
     return TSDB_CODE_SUCCESS;
   }
@@ -156,8 +158,9 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids,
                  terrstr());
         return TSDB_CODE_FAILED;
       }
-      smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 " uid:%" PRIi64 " level %d",
-               SMA_VID(pSma), pRSmaInfo->taskInfo[i], *suid, *(int64_t *)taosArrayGet(tbUids, 0), i);
+      smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p. suid:%" PRIi64 " uid:%" PRIi64
+               "nTables:%d level %d",
+               SMA_VID(pSma), pRSmaInfo->taskInfo[i], *suid, *(int64_t *)TARRAY_GET_ELEM(tbUids, 0), nTables, i);
     }
   }
 
@@ -174,8 +177,8 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) {
     return TSDB_CODE_FAILED;
   }
 
-  void *pIter = taosHashIterate(pStore->uidHash, NULL);
-  while (pIter) {
+  void *pIter = NULL;
+  while ((pIter = taosHashIterate(pStore->uidHash, pIter))) {
     tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
     SArray   *pTbUids = *(SArray **)pIter;
 
@@ -183,8 +186,6 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) {
       taosHashCancelIterate(pStore->uidHash, pIter);
       return TSDB_CODE_FAILED;
     }
-
-    pIter = taosHashIterate(pStore->uidHash, pIter);
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -232,14 +233,37 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
   return TSDB_CODE_SUCCESS;
 }
 
+static void tdRSmaTaskInit(SStreamMeta *pMeta, SRSmaInfoItem *pItem, SStreamTaskId *pId) {
+  STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId};
+  taosRLockLatch(&pMeta->lock);
+  SStreamTask **ppTask = (SStreamTask **)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
+  if (ppTask && *ppTask) {
+    pItem->submitReqVer = (*ppTask)->chkInfo.checkpointVer;
+    pItem->fetchResultVer = (*ppTask)->info.triggerParam;
+  }
+  taosRUnLockLatch(&pMeta->lock);
+}
+
+static void tdRSmaTaskRemove(SStreamMeta *pMeta, int64_t streamId, int32_t taskId) {
+  streamMetaUnregisterTask(pMeta, streamId, taskId);
+  taosWLockLatch(&pMeta->lock);
+  int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
+  if (streamMetaCommit(pMeta) < 0) {
+    // persist to disk
+  }
+  taosWUnLockLatch(&pMeta->lock);
+  smaDebug("vgId:%d, rsma task:%" PRIi64 ",%d dropped, remain tasks:%d", pMeta->vgId, streamId, taskId, numOfTasks);
+}
+
 static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
                                        int8_t idx) {
   if ((param->qmsgLen > 0) && param->qmsg[idx]) {
-    SRetention *pRetention = SMA_RETENTION(pSma);
-    STsdbCfg   *pTsdbCfg = SMA_TSDB_CFG(pSma);
-    SVnode     *pVnode = pSma->pVnode;
-    char        taskInfDir[TSDB_FILENAME_LEN] = {0};
-    void       *pStreamState = NULL;
+    SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
+    SRetention    *pRetention = SMA_RETENTION(pSma);
+    STsdbCfg      *pTsdbCfg = SMA_TSDB_CFG(pSma);
+    SVnode        *pVnode = pSma->pVnode;
+    char           taskInfDir[TSDB_FILENAME_LEN] = {0};
+    void          *pStreamState = NULL;
 
     // set the backend of stream state
     tdRSmaQTaskInfoGetFullPath(pVnode, pRSmaInfo->suid, idx + 1, pVnode->pTfs, taskInfDir);
@@ -254,25 +278,46 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
       taosMemoryFree(s);
     }
 
-    SStreamTask task = {.id.taskId = 0, .id.streamId = 0};  // TODO: assign value
-    task.pMeta = pVnode->pTq->pStreamMeta;
-    pStreamState = streamStateOpen(taskInfDir, &task, true, -1, -1);
+    SStreamTask *pStreamTask = taosMemoryCalloc(1, sizeof(*pStreamTask));
+    if (!pStreamTask) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return TSDB_CODE_FAILED;
+    }
+    pItem->pStreamTask = pStreamTask;
+    pStreamTask->id.taskId = 0;
+    pStreamTask->id.streamId = pRSmaInfo->suid + idx;
+    pStreamTask->chkInfo.startTs = taosGetTimestampMs();
+    pStreamTask->pMeta = pVnode->pTq->pStreamMeta;
+    pStreamTask->exec.qmsg = taosMemoryMalloc(strlen(RSMA_TASK_FLAG) + 1);
+    sprintf(pStreamTask->exec.qmsg, "%s", RSMA_TASK_FLAG);
+    pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta);
+    tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id);
+    pStreamState = streamStateOpen(taskInfDir, pStreamTask, true, -1, -1);
     if (!pStreamState) {
       terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN;
       return TSDB_CODE_FAILED;
     }
+    pItem->pStreamState = pStreamState;
 
-    SReadHandle handle = {.vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState};
+    tdRSmaTaskRemove(pStreamTask->pMeta, pStreamTask->id.streamId, pStreamTask->id.taskId);
+
+    SReadHandle handle = {.vnode = pVnode, .initTqReader = 1, .skipRollup = 1, .pStateBackend = pStreamState};
     initStorageAPI(&handle.api);
-
     pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode), 0);
     if (!pRSmaInfo->taskInfo[idx]) {
       terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
       return TSDB_CODE_FAILED;
     }
-    SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
-    pItem->triggerStat = TASK_TRIGGER_STAT_ACTIVE;  // fetch the data when reboot
-    pItem->pStreamState = pStreamState;
+
+    if (!(pItem->pResList = taosArrayInit(1, POINTER_BYTES))) {
+      return TSDB_CODE_FAILED;
+    }
+
+    if (pItem->fetchResultVer < pItem->submitReqVer) {
+      // fetch the data when reboot
+      pItem->triggerStat = TASK_TRIGGER_STAT_ACTIVE;
+    }
+
     if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) {
       int64_t msInterval =
           convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND);
@@ -291,10 +336,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
 
     taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
 
-    smaInfo("vgId:%d, item:%p table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64
+    smaInfo("vgId:%d, open rsma task:%p table:%" PRIi64 " level:%" PRIi8 ", checkpointId:%" PRIi64
+            ", submitReqVer:%" PRIi64 ", fetchResultVer:%" PRIi64 ", maxdelay:%" PRIi64 " watermark:%" PRIi64
             ", finally maxdelay:%" PRIi32,
-            TD_VID(pVnode), pItem, pRSmaInfo->suid, (int8_t)(idx + 1), param->maxdelay[idx], param->watermark[idx],
-            pItem->maxDelay);
+            TD_VID(pVnode), pItem->pStreamTask, pRSmaInfo->suid, (int8_t)(idx + 1), pStreamTask->chkInfo.checkpointId,
+            pItem->submitReqVer, pItem->fetchResultVer, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay);
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -362,7 +408,7 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
 
   return TSDB_CODE_SUCCESS;
 _err:
-  tdFreeRSmaInfo(pSma, pRSmaInfo, true);
+  tdFreeRSmaInfo(pSma, pRSmaInfo);
   return TSDB_CODE_FAILED;
 }
 
@@ -498,11 +544,10 @@ static void tdUidStoreDestory(STbUidStore *pStore) {
     if (pStore->uidHash) {
       if (pStore->tbUids) {
         // When pStore->tbUids not NULL, the pStore->uidHash has k/v; otherwise pStore->uidHash only has keys.
-        void *pIter = taosHashIterate(pStore->uidHash, NULL);
-        while (pIter) {
+        void *pIter = NULL;
+        while ((pIter = taosHashIterate(pStore->uidHash, pIter))) {
           SArray *arr = *(SArray **)pIter;
           taosArrayDestroy(arr);
-          pIter = taosHashIterate(pStore->uidHash, pIter);
         }
       }
       taosHashCleanup(pStore->uidHash);
@@ -562,7 +607,7 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) {
  * @param now
  * @return int32_t
  */
-int32_t smaDoRetention(SSma *pSma, int64_t now) {
+int32_t smaRetention(SSma *pSma, int64_t now) {
   int32_t code = TSDB_CODE_SUCCESS;
   if (!VND_IS_RSMA(pSma->pVnode)) {
     return code;
@@ -570,8 +615,8 @@ int32_t smaDoRetention(SSma *pSma, int64_t now) {
 
   for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
     if (pSma->pRSmaTsdb[i]) {
-      // code = tsdbDoRetention(pSma->pRSmaTsdb[i], now);
-      // if (code) goto _end;
+      code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1);
+      if (code) goto _end;
     }
   }
 
@@ -579,17 +624,14 @@ _end:
   return code;
 }
 
-static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema,
-                                         int64_t suid) {
+static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, SRSmaInfo *pInfo,
+                                         int32_t execType, int8_t *streamFlushed) {
   int32_t      code = 0;
   int32_t      lino = 0;
   SSDataBlock *output = NULL;
-
-  SArray *pResList = taosArrayInit(1, POINTER_BYTES);
-  if (pResList == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
+  SArray      *pResList = pItem->pResList;
+  STSchema    *pTSchema = pInfo->pTSchema;
+  int64_t      suid = pInfo->suid;
 
   while (1) {
     uint64_t ts;
@@ -604,32 +646,46 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
     if (taosArrayGetSize(pResList) == 0) {
       break;
     }
-#if 0
-    char flag[10] = {0};
-    snprintf(flag, 10, "level %" PRIi8, pItem->level);
-    blockDebugShowDataBlocks(pResList, flag);
-#endif
+
     for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) {
       output = taosArrayGetP(pResList, i);
+      if (output->info.type == STREAM_CHECKPOINT) {
+        if (streamFlushed) *streamFlushed = 1;
+        continue;
+      }
       smaDebug("vgId:%d, result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRIi64, SMA_VID(pSma),
                output->info.id.uid, output->info.id.groupId, output->info.rows);
 
       STsdb       *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]);
       SSubmitReq2 *pReq = NULL;
 
-      // TODO: the schema update should be handled later(TD-17965)
       if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) {
         code = terrno ? terrno : TSDB_CODE_RSMA_RESULT;
         TSDB_CHECK_CODE(code, lino, _exit);
       }
 
+      // reset the output version to handle reboot
+      if (STREAM_GET_ALL == execType && output->info.version == 0) {
+        // the submitReqVer keeps unchanged since tdExecuteRSmaImpl and tdRSmaFetchAllResult are executed synchronously
+        output->info.version = pItem->submitReqVer;
+      }
+
       if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) {
-        code = terrno ? terrno : TSDB_CODE_RSMA_RESULT;
+        if (terrno == TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE) {
+          // TODO: reconfigure SSubmitReq2
+        } else {
+          if (terrno == 0) terrno = TSDB_CODE_RSMA_RESULT;
+          code = terrno;
+        }
         tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
         taosMemoryFree(pReq);
         TSDB_CHECK_CODE(code, lino, _exit);
       }
 
+      if (STREAM_GET_ALL == execType) {
+        atomic_store_64(&pItem->fetchResultVer, output->info.version);
+      }
+
       smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64,
                SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version);
 
@@ -648,7 +704,6 @@ _exit:
   } else {
     smaDebug("vgId:%d, %s succeed, suid:%" PRIi64 ", level:%" PRIi8, SMA_VID(pSma), __func__, suid, pItem->level);
   }
-  taosArrayDestroy(pResList);
   qCleanExecTaskBlockBuf(taskInfo);
   return code;
 }
@@ -743,8 +798,9 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) {
  */
 static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo,
                                  ERsmaExecType type, int8_t level) {
-  int32_t idx = level - 1;
-  void   *qTaskInfo = RSMA_INFO_QTASK(pInfo, idx);
+  int32_t        idx = level - 1;
+  void          *qTaskInfo = RSMA_INFO_QTASK(pInfo, idx);
+  SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx);
 
   if (!qTaskInfo) {
     smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level,
@@ -772,10 +828,14 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize,
     return TSDB_CODE_FAILED;
   }
 
-  SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx);
-  tdRSmaExecAndSubmitResult(pSma, qTaskInfo, pItem, pInfo->pTSchema, pInfo->suid);
+  if (STREAM_INPUT__MERGED_SUBMIT == inputType) {
+    SPackedData *packData = POINTER_SHIFT(pMsg, sizeof(SPackedData) * (msgSize - 1));
+    atomic_store_64(&pItem->submitReqVer, packData->ver);
+  }
 
-  return TSDB_CODE_SUCCESS;
+  terrno = tdRSmaExecAndSubmitResult(pSma, qTaskInfo, pItem, pInfo, STREAM_NORMAL, NULL);
+
+  return terrno ? TSDB_CODE_FAILED : TDB_CODE_SUCCESS;
 }
 
 /**
@@ -881,7 +941,12 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg,
   SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
   if (!pEnv) {
     // only applicable when rsma env exists
-    return TSDB_CODE_SUCCESS;
+    return TDB_CODE_SUCCESS;
+  }
+
+  if ((terrno = atomic_load_32(&SMA_RSMA_STAT(pSma)->execStat))) {
+    smaError("vgId:%d, failed to process rsma submit since invalid exec code: %s", SMA_VID(pSma), terrstr());
+    goto _err;
   }
 
   STbUidStore uidStore = {0};
@@ -913,7 +978,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg,
   return TSDB_CODE_SUCCESS;
 _err:
   tdUidStoreDestory(&uidStore);
-  return TSDB_CODE_FAILED;
+  return terrno;
 }
 
 /**
@@ -988,7 +1053,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
         code = terrno;
         TSDB_CHECK_CODE(code, lino, _exit);
       }
-
+#if 0
       // reload all ctbUids for suid
       uidStore.suid = suid;
       if (vnodeGetCtbIdList(pVnode, suid, uidStore.tbUids) < 0) {
@@ -1002,7 +1067,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
       }
 
       taosArrayClear(uidStore.tbUids);
-
+#endif
       smaDebug("vgId:%d, rsma restore env success for %" PRIi64, TD_VID(pVnode), suid);
     }
   }
@@ -1050,39 +1115,141 @@ _err:
 
   return code;
 }
-#if 0
+
 int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
   int32_t code = 0;
   int32_t lino = 0;
+  int32_t nTaskInfo = 0;
   SSma   *pSma = pRSmaStat->pSma;
   SVnode *pVnode = pSma->pVnode;
-  SRSmaFS fs = {0};
 
   if (taosHashGetSize(pInfoHash) <= 0) {
     return TSDB_CODE_SUCCESS;
   }
 
-  void *infoHash = NULL;
-  while ((infoHash = taosHashIterate(pInfoHash, infoHash))) {
-    SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
-
-    if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
-      continue;
-    }
-
-    for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
-      SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, i);
-      if (pItem && pItem->pStreamState) {
-        if (streamStateCommit(pItem->pStreamState) < 0) {
-          code = TSDB_CODE_RSMA_STREAM_STATE_COMMIT;
-          TSDB_CHECK_CODE(code, lino, _exit);
+  // stream state: trigger checkpoint
+  do {
+    void *infoHash = NULL;
+    while ((infoHash = taosHashIterate(pInfoHash, infoHash))) {
+      SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
+      if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
+        continue;
+      }
+      for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
+        if (pRSmaInfo->taskInfo[i]) {
+          code = qSetSMAInput(pRSmaInfo->taskInfo[i], pRSmaStat->blocks, 1, STREAM_INPUT__CHECKPOINT);
+          if (code) {
+            taosHashCancelIterate(pInfoHash, infoHash);
+            TSDB_CHECK_CODE(code, lino, _exit);
+          }
+          pRSmaInfo->items[i].streamFlushed = 0;
+          ++nTaskInfo;
         }
-        smaDebug("vgId:%d, rsma persist, stream state commit success, table %" PRIi64 ", level %d", TD_VID(pVnode),
-                 pRSmaInfo->suid, i + 1);
       }
     }
-  }
+  } while (0);
 
+  // stream state: wait checkpoint ready in async mode
+  do {
+    int32_t nStreamFlushed = 0;
+    int32_t nSleep = 0;
+    void   *infoHash = NULL;
+    while (true) {
+      while ((infoHash = taosHashIterate(pInfoHash, infoHash))) {
+        SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
+        if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
+          continue;
+        }
+        for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
+          if (pRSmaInfo->taskInfo[i] && (0 == pRSmaInfo->items[i].streamFlushed)) {
+            int8_t streamFlushed = 0;
+            code = tdRSmaExecAndSubmitResult(pSma, pRSmaInfo->taskInfo[i], &pRSmaInfo->items[i], pRSmaInfo,
+                                             STREAM_CHECKPOINT, &streamFlushed);
+            if (code) {
+              taosHashCancelIterate(pInfoHash, infoHash);
+              TSDB_CHECK_CODE(code, lino, _exit);
+            }
+
+            if (streamFlushed) {
+              pRSmaInfo->items[i].streamFlushed = 1;
+              if (++nStreamFlushed >= nTaskInfo) {
+                smaInfo("vgId:%d, rsma commit, checkpoint ready, %d us consumed, received/total: %d/%d", TD_VID(pVnode),
+                        nSleep * 10, nStreamFlushed, nTaskInfo);
+                taosHashCancelIterate(pInfoHash, infoHash);
+                goto _checkpoint;
+              }
+            }
+          }
+        }
+      }
+      taosUsleep(10);
+      ++nSleep;
+      smaDebug("vgId:%d, rsma commit, wait for checkpoint ready, %d us elapsed, received/total: %d/%d", TD_VID(pVnode),
+               nSleep * 10, nStreamFlushed, nTaskInfo);
+    }
+  } while (0);
+
+_checkpoint:
+  // stream state: build checkpoint in backend
+  do {
+    SStreamMeta *pMeta = NULL;
+    int64_t      checkpointId = taosGetTimestampNs();
+    bool         checkpointBuilt = false;
+    void        *infoHash = NULL;
+    while ((infoHash = taosHashIterate(pInfoHash, infoHash))) {
+      SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
+      if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
+        continue;
+      }
+
+      for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
+        SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, i);
+        if (pItem && pItem->pStreamTask) {
+          SStreamTask *pTask = pItem->pStreamTask;
+          atomic_store_32(&pTask->pMeta->chkptNotReadyTasks, 1);
+          pTask->checkpointingId = checkpointId;
+          pTask->chkInfo.checkpointId = pTask->checkpointingId;
+          pTask->chkInfo.checkpointVer = pItem->submitReqVer;
+          pTask->info.triggerParam = pItem->fetchResultVer;
+
+          if (!checkpointBuilt) {
+            // the stream states share one checkpoint
+            code = streamTaskBuildCheckpoint(pTask);
+            if (code) {
+              taosHashCancelIterate(pInfoHash, infoHash);
+              TSDB_CHECK_CODE(code, lino, _exit);
+            }
+            pMeta = pTask->pMeta;
+            checkpointBuilt = true;
+          }
+
+          taosWLockLatch(&pMeta->lock);
+          if (streamMetaSaveTask(pMeta, pTask)) {
+            taosWUnLockLatch(&pMeta->lock);
+            code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY;
+            taosHashCancelIterate(pInfoHash, infoHash);
+            TSDB_CHECK_CODE(code, lino, _exit);
+          }
+          taosWUnLockLatch(&pMeta->lock);
+          smaDebug("vgId:%d, rsma commit, succeed to commit task:%p, submitReqVer:%" PRIi64 ", fetchResultVer:%" PRIi64
+                   ", table:%" PRIi64 ", level:%d",
+                   TD_VID(pVnode), pTask, pItem->submitReqVer, pItem->fetchResultVer, pRSmaInfo->suid, i + 1);
+        }
+      }
+    }
+    if (pMeta) {
+      taosWLockLatch(&pMeta->lock);
+      if (streamMetaCommit(pMeta)) {
+        taosWUnLockLatch(&pMeta->lock);
+        code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY;
+        TSDB_CHECK_CODE(code, lino, _exit);
+      }
+      taosWUnLockLatch(&pMeta->lock);
+    }
+    if (checkpointBuilt) {
+      smaInfo("vgId:%d, rsma commit, succeed to commit checkpoint:%" PRIi64, TD_VID(pVnode), checkpointId);
+    }
+  } while (0);
 _exit:
   if (code) {
     smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
@@ -1091,7 +1258,7 @@ _exit:
   terrno = code;
   return code;
 }
-#endif
+
 /**
  * @brief trigger to get rsma result in async mode
  *
@@ -1165,14 +1332,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
       smaDebug("vgId:%d, rsma fetch task planned for level:%" PRIi8 " suid:%" PRIi64 " since stat is active",
                SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
       // async process
-      pItem->fetchLevel = pItem->level;
-#if 0
-      // debugging codes
-      SRSmaInfo     *qInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaInfo->suid);
-      SRSmaInfoItem *qItem = RSMA_INFO_ITEM(qInfo, pItem->level - 1);
-      make sure(qItem->level == pItem->level);
-      make sure(qItem->fetchLevel == pItem->fetchLevel);
-#endif
+      atomic_store_8(&pItem->fetchLevel, 1);
+
       if (atomic_load_8(&pRSmaInfo->assigned) == 0) {
         tsem_post(&(pStat->notEmpty));
       }
@@ -1217,15 +1378,15 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) {
   SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
   for (int8_t i = 1; i <= TSDB_RETENTION_L2; ++i) {
     SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, i - 1);
-    if (pItem->fetchLevel) {
-      pItem->fetchLevel = 0;
+
+    if (1 == atomic_val_compare_exchange_8(&pItem->fetchLevel, 1, 0)) {
       qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pInfo, i - 1);
       if (!taskInfo) {
         continue;
       }
 
       if ((++pItem->nScanned * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) {
-        smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi16 " maxDelay:%d, fetch executed",
+        smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi32 " maxDelay:%d, fetch executed",
                  SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay);
       } else {
         int64_t curMs = taosGetTimestampMs();
@@ -1245,14 +1406,15 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) {
       if ((terrno = qSetSMAInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) {
         goto _err;
       }
-      if (tdRSmaExecAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid) < 0) {
+      if (tdRSmaExecAndSubmitResult(pSma, taskInfo, pItem, pInfo, STREAM_GET_ALL, NULL) < 0) {
+        atomic_store_32(&SMA_RSMA_STAT(pSma)->execStat, terrno);
         goto _err;
       }
 
-      smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi16 " maxDelay:%d, fetch finished",
+      smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi32 " maxDelay:%d, fetch finished",
                SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay);
     } else {
-      smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi16
+      smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi32
                " maxDelay:%d, fetch not executed as fetch level is %" PRIi8,
                SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay, pItem->fetchLevel);
     }
@@ -1275,6 +1437,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA
                               .msgStr = POINTER_SHIFT(msg, sizeof(int32_t) + sizeof(int64_t))};
 
       if (!taosArrayPush(pSubmitArr, &packData)) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
         tdFreeRSmaSubmitItems(pSubmitArr);
         goto _err;
       }
@@ -1294,6 +1457,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA
   }
   return TSDB_CODE_SUCCESS;
 _err:
+  atomic_store_32(&SMA_RSMA_STAT(pSma)->execStat, terrno);
   smaError("vgId:%d, batch exec for suid:%" PRIi64 " execType:%d size:%d failed since %s", SMA_VID(pSma), pInfo->suid,
            type, (int32_t)taosArrayGetSize(pSubmitArr), terrstr());
   tdFreeRSmaSubmitItems(pSubmitArr);
@@ -1369,15 +1533,11 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) {
 
                   if (ASSERTS(oldVal >= 0, "oldVal of nFetchAll: %d < 0", oldVal)) {
                     code = TSDB_CODE_APP_ERROR;
+                    taosHashCancelIterate(infoHash, pIter);
                     TSDB_CHECK_CODE(code, lino, _exit);
                   }
 
-                  int8_t curStat = atomic_load_8(RSMA_COMMIT_STAT(pRSmaStat));
-                  if (curStat == 1) {
-                    smaDebug("vgId:%d, fetch all not exec as commit stat is %" PRIi8, SMA_VID(pSma), curStat);
-                  } else {
-                    tdRSmaFetchAllResult(pSma, pInfo);
-                  }
+                  tdRSmaFetchAllResult(pSma, pInfo);
 
                   if (0 == atomic_sub_fetch_32(&pRSmaStat->nFetchAll, 1)) {
                     atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0);
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index bf6f0cf4d6..918d0bd7d0 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -12,11 +12,11 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
+#include "cos.h"
 #include "tsdb.h"
 #include "tsdbDataFileRW.h"
 #include "tsdbReadUtil.h"
 #include "vnd.h"
-#include "vndCos.h"
 
 #define ROCKS_BATCH_SIZE (4096)
 
@@ -3099,7 +3099,7 @@ static int32_t tsdbCacheLoadBlockS3(STsdbFD *pFD, uint8_t **ppBlock) {
   }
   */
   int64_t block_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage;
-  code = s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, ppBlock);
+  code = s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, 0, ppBlock);
   if (code != TSDB_CODE_SUCCESS) {
     // taosMemoryFree(pBlock);
     // code = TSDB_CODE_OUT_OF_MEMORY;
diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
index 5506f61098..b6aa791cf0 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
@@ -28,13 +28,18 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
   // bool    allNullRow = true;
 
   if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) {
+    uint64_t ts = 0;
+    SFirstLastRes* p;
+    col_id_t colId;
     for (int32_t i = 0; i < pReader->numOfCols; ++i) {
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
-      SFirstLastRes*   p = (SFirstLastRes*)varDataVal(pRes[i]);
       int32_t          slotId = slotIds[i];
       SLastCol*        pColVal = (SLastCol*)taosArrayGet(pRow, i);
+      colId = pColVal->colVal.cid;
+      p = (SFirstLastRes*)varDataVal(pRes[i]);
 
       p->ts = pColVal->ts;
+      ts = p->ts;
       p->isNull = !COL_VAL_IS_VALUE(&pColVal->colVal);
       // allNullRow = p->isNull & allNullRow;
 
@@ -55,6 +60,19 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
       varDataSetLen(pRes[i], pColInfoData->info.bytes - VARSTR_HEADER_SIZE);
       colDataSetVal(pColInfoData, numOfRows, (const char*)pRes[i], false);
     }
+    for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) {
+      SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx);
+      if (pCol->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID && pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
+        colDataSetVal(pCol, numOfRows, (const char*)&ts, false);
+        continue;
+      } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && pCol->info.colId == colId) {
+        if (!p->isNull) {
+          colDataSetVal(pCol, numOfRows, p->buf, false);
+        } else {
+          colDataSetNULL(pCol, numOfRows);
+        }
+      }
+    }
 
     // pBlock->info.rows += allNullRow ? 0 : 1;
     ++pBlock->info.rows;
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
index 46e3aff0d4..22fb3b84ec 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c
@@ -192,7 +192,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
       committer->ctx->tbid->uid = record->uid;
 
       if (metaGetInfo(committer->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) {
-        code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid);
+        code = tsdbIterMergerSkipTableData(committer->tombIterMerger, committer->ctx->tbid);
         TSDB_CHECK_CODE(code, lino, _exit);
         continue;
       }
diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c
index 38d221d978..02ef75ae86 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFS2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c
@@ -14,15 +14,15 @@
  */
 
 #include "tsdbFS2.h"
+#include "cos.h"
 #include "tsdbUpgrade.h"
 #include "vnd.h"
-#include "vndCos.h"
 
 #define BLOCK_COMMIT_FACTOR 3
 
 extern int  vnodeScheduleTask(int (*execute)(void *), void *arg);
 extern int  vnodeScheduleTaskEx(int tpid, int (*execute)(void *), void *arg);
-extern void remove_file(const char *fname);
+extern void remove_file(const char *fname, bool last_level);
 
 #define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT
 #define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1)
@@ -532,7 +532,8 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) {
       if (taosIsDir(file->aname)) continue;
 
       if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL) {
-        remove_file(file->aname);
+        int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs);
+        remove_file(file->aname, nlevel > 1 && file->did.level == nlevel - 1);
       }
     }
 
@@ -1282,4 +1283,4 @@ int32_t tsdbFSEnableBgTask(STFileSystem *fs) {
   fs->stop = false;
   taosThreadMutexUnlock(&fs->tsdb->mutex);
   return 0;
-}
\ No newline at end of file
+}
diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c
index 9edb03d35b..cc05b8ee18 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFile2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c
@@ -14,6 +14,7 @@
  */
 
 #include "tsdbFile2.h"
+#include "cos.h"
 
 // to_json
 static int32_t head_to_json(const STFile *file, cJSON *json);
@@ -41,10 +42,20 @@ static const struct {
     [TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json},
 };
 
-void remove_file(const char *fname) {
+void remove_file(const char *fname, bool last_level) {
   int32_t code = taosRemoveFile(fname);
   if (code) {
-    tsdbError("file:%s remove failed", fname);
+    if (tsS3Enabled && last_level) {
+      const char *object_name = taosDirEntryBaseName((char *)fname);
+      long        s3_size = tsS3Enabled ? s3Size(object_name) : 0;
+      if (!strncmp(fname + strlen(fname) - 5, ".data", 5) && s3_size > 0) {
+        s3DeleteObjects(&object_name, 1);
+      } else {
+        tsdbError("file:%s remove failed", fname);
+      }
+    } else {
+      tsdbError("file:%s remove failed", fname);
+    }
   } else {
     tsdbInfo("file:%s is removed", fname);
   }
@@ -224,6 +235,7 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
   fobj[0]->state = TSDB_FSTATE_LIVE;
   fobj[0]->ref = 1;
   tsdbTFileName(pTsdb, f, fobj[0]->fname);
+  fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
   return 0;
 }
 
@@ -245,7 +257,7 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) {
   tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
   if (nRef == 0) {
     if (fobj->state == TSDB_FSTATE_DEAD) {
-      remove_file(fobj->fname);
+      remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1);
     }
     taosMemoryFree(fobj);
   }
@@ -261,7 +273,7 @@ int32_t tsdbTFileObjRemove(STFileObj *fobj) {
   taosThreadMutexUnlock(&fobj->mutex);
   tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
   if (nRef == 0) {
-    remove_file(fobj->fname);
+    remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1);
     taosMemoryFree(fobj);
   }
   return 0;
diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.h b/source/dnode/vnode/src/tsdb/tsdbFile2.h
index 9da198c1f0..b94f7a9fd0 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFile2.h
+++ b/source/dnode/vnode/src/tsdb/tsdbFile2.h
@@ -76,6 +76,7 @@ struct STFileObj {
   STFile        f[1];
   int32_t       state;
   int32_t       ref;
+  int32_t       nlevel;
   char          fname[TSDB_FILENAME_LEN];
 };
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c
index c32b2eedd7..a1f864814f 100644
--- a/source/dnode/vnode/src/tsdb/tsdbOpen.c
+++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c
@@ -89,6 +89,8 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
   return 0;
 
 _err:
+  tsdbCloseFS(&pTsdb->pFS);
+  taosThreadMutexDestroy(&pTsdb->mutex);
   taosMemoryFree(pTsdb);
   return -1;
 }
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c
index 4b5329220b..4682c47bd1 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c
@@ -48,9 +48,9 @@ static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScan
 static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key,
                                      STsdbReader* pReader);
 
-static int32_t       initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost);
-static STsdb*        getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr,
-                                         int8_t* pLevel);
+static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost);
+static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr,
+                                  int8_t* pLevel);
 static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level);
 static bool          hasDataInLastBlock(SLastBlockReader* pLastBlockReader);
 static int32_t       doBuildDataBlock(STsdbReader* pReader);
@@ -384,7 +384,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void
 
   initReaderStatus(&pReader->status);
 
-  pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level);
+  pReader->pTsdb = getTsdbByRetentions(pVnode, pCond, pVnode->config.tsdbCfg.retentions, idstr, &level);
   pReader->info.suid = pCond->suid;
   pReader->info.order = pCond->order;
 
@@ -3152,9 +3152,9 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
   }
 }
 
-static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idStr,
+static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idStr,
                                   int8_t* pLevel) {
-  if (VND_IS_RSMA(pVnode)) {
+  if (VND_IS_RSMA(pVnode) && !pCond->skipRollup) {
     int8_t  level = 0;
     int8_t  precision = pVnode->config.tsdbCfg.precision;
     int64_t now = taosGetTimestamp(precision);
@@ -3170,7 +3170,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
         }
         break;
       }
-      if ((now - pRetention->keep) <= (winSKey + offset)) {
+      if ((now - pRetention->keep) <= (pCond->twindows.skey + offset)) {
         break;
       }
       ++level;
diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index def9a73d10..8b9cae42fc 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "cos.h"
 #include "tsdb.h"
-#include "vndCos.h"
 
 static int32_t tsdbOpenFileImpl(STsdbFD *pFD) {
   int32_t     code = 0;
@@ -338,9 +338,9 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64
     // 2, retrieve pgs from s3
     uint8_t *pBlock = NULL;
     int64_t  retrieve_offset = PAGE_OFFSET(pgno, pFD->szPage);
-    int64_t  pgnoEnd = pgno - 1 + (size - n + szPgCont - 1) / szPgCont;
+    int64_t  pgnoEnd = pgno - 1 + (bOffset + size - n + szPgCont - 1) / szPgCont;
     int64_t  retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage;
-    code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, &pBlock);
+    code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock);
     if (code != TSDB_CODE_SUCCESS) {
       goto _exit;
     }
diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c
index 0fc1e1b64b..194c2784f4 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRetention.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c
@@ -13,9 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "cos.h"
 #include "tsdb.h"
 #include "tsdbFS2.h"
-#include "vndCos.h"
+#include "vnd.h"
 
 typedef struct {
   STsdb  *tsdb;
@@ -291,15 +292,15 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
   if (expLevel < 0) {  // remove the fileset
     for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) {
       if (fobj == NULL) continue;
-
+      /*
       int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
       if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) {
         code = tsdbRemoveFileObjectS3(rtner, fobj);
         TSDB_CHECK_CODE(code, lino, _exit);
-      } else {
-        code = tsdbDoRemoveFileObject(rtner, fobj);
-        TSDB_CHECK_CODE(code, lino, _exit);
-      }
+        } else {*/
+      code = tsdbDoRemoveFileObject(rtner, fobj);
+      TSDB_CHECK_CODE(code, lino, _exit);
+      //}
     }
 
     SSttLvl *lvl;
@@ -387,6 +388,8 @@ _exit:
   return code;
 }
 
+static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); }
+
 static int32_t tsdbDoRetentionSync(void *arg) {
   int32_t code = 0;
   int32_t lino = 0;
@@ -409,6 +412,7 @@ _exit:
     TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
   }
   tsem_post(&((SRtnArg *)arg)->tsdb->pVnode->canCommit);
+  tsdbFreeRtnArg(arg);
   return code;
 }
 
@@ -438,7 +442,7 @@ _exit:
   return code;
 }
 
-static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); }
+
 
 int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
   int32_t code = 0;
diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c
index 1e6526da48..e8be48499b 100644
--- a/source/dnode/vnode/src/tsdb/tsdbWrite.c
+++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c
@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "cos.h"
 #include "tsdb.h"
-#include "vndCos.h"
 
 /**
  * @brief max key by precision
@@ -39,7 +39,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2
   arrSize = taosArrayGetSize(pMsg->aSubmitTbData);
 
   // scan and convert
-  if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) {
+  if ((terrno = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) {
     if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
       tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(terrno));
     }
diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c
index 6ccce5c9d7..df08fb8a2b 100644
--- a/source/dnode/vnode/src/vnd/vnodeModule.c
+++ b/source/dnode/vnode/src/vnd/vnodeModule.c
@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "cos.h"
 #include "vnd.h"
-#include "vndCos.h"
 
 typedef struct SVnodeTask SVnodeTask;
 struct SVnodeTask {
diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index 3bdecee79b..ff79e83d72 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -13,10 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "cos.h"
 #include "sync.h"
 #include "tsdb.h"
 #include "vnd.h"
-#include "vndCos.h"
 
 int32_t vnodeGetPrimaryDir(const char *relPath, int32_t diskPrimary, STfs *pTfs, char *buf, size_t bufLen) {
   if (pTfs) {
diff --git a/source/dnode/vnode/src/vnd/vnodeRetention.c b/source/dnode/vnode/src/vnd/vnodeRetention.c
index f3344d1d7d..c510c0fe92 100644
--- a/source/dnode/vnode/src/vnd/vnodeRetention.c
+++ b/source/dnode/vnode/src/vnd/vnodeRetention.c
@@ -15,8 +15,12 @@
 
 #include "vnd.h"
 
-extern int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync);
-
 int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) {
-  return tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  code = tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
+
+  if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now);
+
+  return code;
 }
\ No newline at end of file
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index b31463ac00..eadfd39d0b 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -18,7 +18,7 @@
 #include "tmsg.h"
 #include "tstrbuild.h"
 #include "vnd.h"
-#include "vndCos.h"
+#include "cos.h"
 #include "vnode.h"
 #include "vnodeInt.h"
 
@@ -928,6 +928,17 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
       goto _exit;
     }
 
+    if(tsEnableAuditCreateTable){
+      char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN);
+      if (str == NULL) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        rcode = -1;
+        goto _exit;
+      }
+      strcpy(str, pCreateReq->name);
+      taosArrayPush(tbNames, &str);
+    }
+
     // validate hash
     sprintf(tbName, "%s.%s", pVnode->config.dbname, pCreateReq->name);
     if (vnodeValidateTableHash(pVnode, tbName) < 0) {
@@ -951,12 +962,6 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
     }
 
     taosArrayPush(rsp.pArray, &cRsp);
-
-    if (tsEnableAuditCreateTable) {
-      char *str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN);
-      strcpy(str, pCreateReq->name);
-      taosArrayPush(tbNames, &str);
-    }
   }
 
   vDebug("vgId:%d, add %d new created tables into query table list", TD_VID(pVnode), (int32_t)taosArrayGetSize(tbUids));
@@ -985,10 +990,10 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
     tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB);
 
     SStringBuilder sb = {0};
-    for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
-      char **key = (char **)taosArrayGet(tbNames, iReq);
+    for(int32_t i = 0; i < tbNames->size; i++){
+      char** key = (char**)taosArrayGet(tbNames, i);
       taosStringBuilderAppendStringLen(&sb, *key, strlen(*key));
-      if (iReq < req.nReqs - 1) {
+      if(i < tbNames->size - 1){
         taosStringBuilderAppendChar(&sb, ',');
       }
       taosMemoryFreeClear(*key);
@@ -1669,7 +1674,7 @@ _exit:
   atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1);
   if (code == 0) {
     atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1);
-    tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT);
+    code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT);
   }
 
   // clear
diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h
index 740ff7b0dc..946d3311a4 100644
--- a/source/libs/executor/inc/executil.h
+++ b/source/libs/executor/inc/executil.h
@@ -40,8 +40,10 @@
 #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
 
 typedef struct SGroupResInfo {
-  int32_t index;
-  SArray* pRows;  // SArray<SResKeyPos>
+  int32_t index;    // rows consumed in func:doCopyToSDataBlockXX
+  int32_t iter;     // relate to index-1, last consumed data's slot id in hash table
+  void*   dataPos;  // relate to index-1, last consumed data's position, in the nodelist of cur slot
+  SArray* pRows;    // SArray<SResKeyPos>
   char*   pBuf;
   bool    freeItem;
 } SGroupResInfo;
@@ -178,7 +180,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow
 SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode);
 SColumn   extractColumnFromColumnNode(SColumnNode* pColNode);
 
-int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode);
+int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode, const SReadHandle* readHandle);
 void    cleanupQueryTableDataCond(SQueryTableDataCond* pCond);
 
 int32_t convertFillType(int32_t mode);
diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index 52acdfaaa8..331ce44366 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -679,6 +679,12 @@ void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows);
 void doBuildResultDatablock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
                             SDiskbasedBuf* pBuf);
 
+/**
+ * @brief copydata from hash table, instead of copying from SGroupResInfo's pRow
+ */
+int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
+                           SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, bool ignoreGroup);
+
 bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
 bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo);
 void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);
diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c
index 29d098494b..6d59698855 100644
--- a/source/libs/executor/src/cachescanoperator.c
+++ b/source/libs/executor/src/cachescanoperator.c
@@ -54,6 +54,19 @@ static int32_t      removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM
 
 #define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW)
 
+static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SNodeList* pTargets) {
+  SNode*  pNode;
+  int32_t idx = 0;
+  FOREACH(pNode, pTargets) {
+    if (nodeType(pNode) == QUERY_NODE_COLUMN) {
+      SColumnNode* pCol = (SColumnNode*)pNode;
+      SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx);
+      pColInfo->info.colId = pCol->colId;
+    }
+    idx++;
+  }
+}
+
 SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
                                            STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
   int32_t             code = TSDB_CODE_SUCCESS;
@@ -114,10 +127,12 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
     capacity = TMIN(totalTables, 4096);
 
     pInfo->pBufferredRes = createOneDataBlock(pInfo->pRes, false);
+    setColIdForCacheReadBlock(pInfo->pBufferredRes, pScanNode->pTargets);
     blockDataEnsureCapacity(pInfo->pBufferredRes, capacity);
   } else {  // by tags
     pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE | SCAN_ROW_TYPE(pScanNode->ignoreNull);
     capacity = 1;  // only one row output
+    setColIdForCacheReadBlock(pInfo->pRes, pScanNode->pTargets);
   }
 
   initResultSizeInfo(&pOperator->resultInfo, capacity);
@@ -191,9 +206,9 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
     SSDataBlock* pRes = pInfo->pRes;
 
     if (pInfo->indexOfBufferedRes < pInfo->pBufferredRes->info.rows) {
-      for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) {
-        SColMatchItem* pMatchInfo = taosArrayGet(pInfo->matchInfo.pList, i);
-        int32_t        slotId = pMatchInfo->dstSlotId;
+      for (int32_t i = 0; i < taosArrayGetSize(pInfo->pBufferredRes->pDataBlock); ++i) {
+        SColumnInfoData* pCol = taosArrayGet(pRes->pDataBlock, i);
+        int32_t        slotId = pCol->info.slotId;
 
         SColumnInfoData* pSrc = taosArrayGet(pInfo->pBufferredRes->pDataBlock, slotId);
         SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, slotId);
@@ -201,8 +216,10 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
         if (colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes)) {
           colDataSetNULL(pDst, 0);
         } else {
-          char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes);
-          colDataSetVal(pDst, 0, p, false);
+          if (pSrc->pData) {
+            char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes);
+            colDataSetVal(pDst, 0, p, false);
+          }
         }
       }
 
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 753d3e680c..39b47504c6 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1713,7 +1713,7 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) {
   return c;
 }
 
-int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) {
+int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode, const SReadHandle* readHandle) {
   pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
   pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols);
 
@@ -1732,6 +1732,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
   pCond->type = TIMEWINDOW_RANGE_CONTAINED;
   pCond->startVersion = -1;
   pCond->endVersion = -1;
+  pCond->skipRollup = readHandle->skipRollup;
 
   int32_t j = 0;
   for (int32_t i = 0; i < pCond->numOfCols; ++i) {
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 9ed7d6e033..a0e35fff87 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -69,12 +69,14 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf
     } else if (type == STREAM_INPUT__DATA_BLOCK) {
       for (int32_t i = 0; i < numOfBlocks; ++i) {
         SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i];
-        SPackedData  tmp = {
-             .pDataBlock = pDataBlock,
-        };
+        SPackedData tmp = {.pDataBlock = pDataBlock};
         taosArrayPush(pInfo->pBlockLists, &tmp);
       }
       pInfo->blockType = STREAM_INPUT__DATA_BLOCK;
+    } else if (type == STREAM_INPUT__CHECKPOINT) {
+      SPackedData tmp = {.pDataBlock = input};
+      taosArrayPush(pInfo->pBlockLists, &tmp);
+      pInfo->blockType = STREAM_INPUT__CHECKPOINT;
     }
 
     return TSDB_CODE_SUCCESS;
@@ -633,7 +635,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
     blockIndex += 1;
 
     current += p->info.rows;
-    ASSERT(p->info.rows > 0);
+    ASSERT(p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT);
     taosArrayPush(pResList, &p);
 
     if (current >= rowsThreshold) {
diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c
index 8ad174f366..138da16324 100644
--- a/source/libs/executor/src/executorInt.c
+++ b/source/libs/executor/src/executorInt.c
@@ -655,6 +655,85 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos
   return 0;
 }
 
+int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
+                                 SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold,
+                                 bool ignoreGroup) {
+  SExprInfo*      pExprInfo = pSup->pExprInfo;
+  int32_t         numOfExprs = pSup->numOfExprs;
+  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;
+  SqlFunctionCtx* pCtx = pSup->pCtx;
+
+  size_t  keyLen = 0;
+  int32_t numOfRows = tSimpleHashGetSize(pHashmap);
+
+  // begin from last iter
+  void*   pData = pGroupResInfo->dataPos;
+  int32_t iter = pGroupResInfo->iter;
+  while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) {
+    void*               key = tSimpleHashGetKey(pData, &keyLen);
+    SResultRowPosition* pos = pData;
+    uint64_t            groupId = *(uint64_t*)key;
+
+    SFilePage* page = getBufPage(pBuf, pos->pageId);
+    if (page == NULL) {
+      qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo));
+      T_LONG_JMP(pTaskInfo->env, terrno);
+    }
+
+    SResultRow* pRow = (SResultRow*)((char*)page + pos->offset);
+
+    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);
+
+    // no results, continue to check the next one
+    if (pRow->numOfRows == 0) {
+      pGroupResInfo->index += 1;
+      pGroupResInfo->iter = iter;
+      pGroupResInfo->dataPos = pData;
+
+      releaseBufPage(pBuf, page);
+      continue;
+    }
+
+    if (!ignoreGroup) {
+      if (pBlock->info.id.groupId == 0) {
+        pBlock->info.id.groupId = groupId;
+      } else {
+        // current value belongs to different group, it can't be packed into one datablock
+        if (pBlock->info.id.groupId != groupId) {
+          releaseBufPage(pBuf, page);
+          break;
+        }
+      }
+    }
+
+    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {
+      uint32_t newSize = pBlock->info.rows + pRow->numOfRows + ((numOfRows - iter) > 1 ? 1 : 0);
+      blockDataEnsureCapacity(pBlock, newSize);
+      qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", newSize,
+             pBlock->info.capacity, GET_TASKID(pTaskInfo));
+      // todo set the pOperator->resultInfo size
+    }
+
+    pGroupResInfo->index += 1;
+    pGroupResInfo->iter = iter;
+    pGroupResInfo->dataPos = pData;
+
+    copyResultrowToDataBlock(pExprInfo, numOfExprs, pRow, pCtx, pBlock, rowEntryOffset, pTaskInfo);
+
+    releaseBufPage(pBuf, page);
+    pBlock->info.rows += pRow->numOfRows;
+    if (pBlock->info.rows >= threshold) {
+      break;
+    }
+  }
+
+  qDebug("%s result generated, rows:%" PRId64 ", groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
+        pBlock->info.id.groupId);
+  pBlock->info.dataLoad = 1;
+  blockDataUpdateTsWindow(pBlock, 0);
+  return 0;
+}
+
 int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,
                            SGroupResInfo* pGroupResInfo, int32_t threshold, bool ignoreGroup) {
   SExprInfo*      pExprInfo = pSup->pExprInfo;
diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c
index 467a49b37a..eb18278870 100644
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@@ -370,6 +370,72 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) {
   return (pRes->info.rows == 0) ? NULL : pRes;
 }
 
+bool hasRemainResultByHash(SOperatorInfo* pOperator) {
+  SGroupbyOperatorInfo* pInfo = pOperator->info;
+  SSHashObj* pHashmap = pInfo->aggSup.pResultRowHashTable;
+  return pInfo->groupResInfo.index < tSimpleHashGetSize(pHashmap);
+}
+
+void doBuildResultDatablockByHash(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
+                                  SDiskbasedBuf* pBuf) {
+  SGroupbyOperatorInfo* pInfo = pOperator->info;
+  SSHashObj*            pHashmap = pInfo->aggSup.pResultRowHashTable;
+  SExecTaskInfo*        pTaskInfo = pOperator->pTaskInfo;
+
+  SSDataBlock* pBlock = pInfo->binfo.pRes;
+
+  // set output datablock version
+  pBlock->info.version = pTaskInfo->version;
+
+  blockDataCleanup(pBlock);
+  if (!hasRemainResultByHash(pOperator)) {
+    return;
+  }
+
+  pBlock->info.id.groupId = 0;
+  if (!pInfo->binfo.mergeResultBlock) {
+    doCopyToSDataBlockByHash(pTaskInfo, pBlock, &pOperator->exprSupp, pInfo->aggSup.pResultBuf, &pInfo->groupResInfo,
+                             pHashmap, pOperator->resultInfo.threshold, false);
+  } else {
+    while (hasRemainResultByHash(pOperator)) {
+      doCopyToSDataBlockByHash(pTaskInfo, pBlock, &pOperator->exprSupp, pInfo->aggSup.pResultBuf, &pInfo->groupResInfo,
+                               pHashmap, pOperator->resultInfo.threshold, true);
+      if (pBlock->info.rows >= pOperator->resultInfo.threshold) {
+        break;
+      }
+      pBlock->info.id.groupId = 0;
+    }
+
+    // clear the group id info in SSDataBlock, since the client does not need it
+    pBlock->info.id.groupId = 0;
+  }
+}
+
+static SSDataBlock* buildGroupResultDataBlockByHash(SOperatorInfo* pOperator) {
+  SGroupbyOperatorInfo* pInfo = pOperator->info;
+  SSDataBlock* pRes = pInfo->binfo.pRes;
+
+  // after filter, if result block turn to null, get next from whole set
+  while (1) {
+    doBuildResultDatablockByHash(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
+
+    doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL);
+    if (!hasRemainResultByHash(pOperator)) {
+      setOperatorCompleted(pOperator);
+      // clean hash after completed
+      tSimpleHashCleanup(pInfo->aggSup.pResultRowHashTable);
+      pInfo->aggSup.pResultRowHashTable = NULL;
+      break;
+    }
+    if (pRes->info.rows > 0) {
+      break;
+    }
+  }
+
+  pOperator->resultInfo.totalRows += pRes->info.rows;
+  return (pRes->info.rows == 0) ? NULL : pRes;
+}
+
 static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
@@ -379,9 +445,10 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
 
   SGroupbyOperatorInfo* pInfo = pOperator->info;
   if (pOperator->status == OP_RES_TO_RETURN) {
-    return buildGroupResultDataBlock(pOperator);
+    return buildGroupResultDataBlockByHash(pOperator);
   }
-
+  SGroupResInfo* pGroupResInfo = &pInfo->groupResInfo;
+  
   int32_t order = pInfo->binfo.inputTsOrder;
   int64_t        st = taosGetTimestampUs();
   SOperatorInfo* downstream = pOperator->pDownstream[0];
@@ -425,10 +492,20 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
     }
   }
 #endif
-  initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, 0);
+  // initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, 0);
+  if (pGroupResInfo->pRows != NULL) {
+    taosArrayDestroy(pGroupResInfo->pRows);
+  }
+  if (pGroupResInfo->pBuf) {
+    taosMemoryFree(pGroupResInfo->pBuf);
+    pGroupResInfo->pBuf = NULL;
+  }
+  pGroupResInfo->index = 0;
+  pGroupResInfo->iter = 0;
+  pGroupResInfo->dataPos = NULL;
 
   pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
-  return buildGroupResultDataBlock(pOperator);
+  return buildGroupResultDataBlockByHash(pOperator);
 }
 
 SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo) {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index efbc978323..c47e14ad0d 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1035,7 +1035,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
   }
 
   initLimitInfo(pScanNode->node.pLimit, pScanNode->node.pSlimit, &pInfo->base.limitInfo);
-  code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode);
+  code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode, readHandle);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -3533,7 +3533,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     goto _error;
   }
 
-  code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode);
+  code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode, readHandle);
   if (code != TSDB_CODE_SUCCESS) {
     taosArrayDestroy(pInfo->base.matchInfo.pList);
     goto _error;
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 344ceea419..fbabef8fd3 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -2772,7 +2772,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_cache_last",
     .type = FUNCTION_TYPE_CACHE_LAST,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC,
     .translateFunc = translateFirstLast,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c
index c97c920a3b..c9b49ee30f 100644
--- a/source/libs/nodes/src/nodesCodeFuncs.c
+++ b/source/libs/nodes/src/nodesCodeFuncs.c
@@ -1773,6 +1773,7 @@ static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) {
 
 static const char* jkLastRowScanPhysiPlanGroupTags = "GroupTags";
 static const char* jkLastRowScanPhysiPlanGroupSort = "GroupSort";
+static const char* jkLastRowScanPhysiPlanTargets = "Targets";
 
 static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) {
   const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj;
@@ -1784,6 +1785,9 @@ static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddBoolToObject(pJson, jkLastRowScanPhysiPlanGroupSort, pNode->groupSort);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodeListToJson(pJson, jkLastRowScanPhysiPlanTargets, pNode->pTargets);
+  }
 
   return code;
 }
@@ -1798,6 +1802,9 @@ static int32_t jsonToPhysiLastRowScanNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetBoolValue(pJson, jkLastRowScanPhysiPlanGroupSort, &pNode->groupSort);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeList(pJson, jkLastRowScanPhysiPlanTargets, &pNode->pTargets);
+  }
 
   return code;
 }
diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c
index 99100b2a1d..ea59d93d7f 100644
--- a/source/libs/nodes/src/nodesMsgFuncs.c
+++ b/source/libs/nodes/src/nodesMsgFuncs.c
@@ -2052,7 +2052,8 @@ enum {
   PHY_LAST_ROW_SCAN_CODE_SCAN = 1,
   PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS,
   PHY_LAST_ROW_SCAN_CODE_GROUP_SORT,
-  PHY_LAST_ROW_SCAN_CODE_IGNULL
+  PHY_LAST_ROW_SCAN_CODE_IGNULL,
+  PHY_LAST_ROW_SCAN_CODE_TARGETS
 };
 
 static int32_t physiLastRowScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@@ -2068,6 +2069,9 @@ static int32_t physiLastRowScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder
   if (TSDB_CODE_SUCCESS == code) {
     code = tlvEncodeBool(pEncoder, PHY_LAST_ROW_SCAN_CODE_IGNULL, pNode->ignoreNull);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tlvEncodeObj(pEncoder, PHY_LAST_ROW_SCAN_CODE_TARGETS, nodeListToMsg, pNode->pTargets);
+  }
 
   return code;
 }
@@ -2091,6 +2095,9 @@ static int32_t msgToPhysiLastRowScanNode(STlvDecoder* pDecoder, void* pObj) {
       case PHY_LAST_ROW_SCAN_CODE_IGNULL:
         code = tlvDecodeBool(pTlv, &pNode->ignoreNull);
         break;
+      case PHY_LAST_ROW_SCAN_CODE_TARGETS:
+        code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets);
+        break;
       default:
         break;
     }
diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c
index 4f6d3d95e1..ee22caf574 100644
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@@ -1285,6 +1285,7 @@ void nodesDestroyNode(SNode* pNode) {
       SLastRowScanPhysiNode* pPhyNode = (SLastRowScanPhysiNode*)pNode;
       destroyScanPhysiNode((SScanPhysiNode*)pNode);
       nodesDestroyList(pPhyNode->pGroupTags);
+      nodesDestroyList(pPhyNode->pTargets);
       break;
     }
     case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN:
diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c
index 0657f1a43d..441f4da3b1 100644
--- a/source/libs/parser/src/parCalcConst.c
+++ b/source/libs/parser/src/parCalcConst.c
@@ -176,12 +176,15 @@ static int32_t calcConstStmtCondition(SCalcConstContext* pCxt, SNode** pCond, bo
 static EDealRes doFindAndReplaceNode(SNode** pNode, void* pContext) {
   SCalcConstContext* pCxt = pContext;
   if (pCxt->replaceCxt.pTarget == *pNode) {
+    char aliasName[TSDB_COL_NAME_LEN] = {0};
+    strcpy(aliasName, ((SExprNode*)*pNode)->aliasName);
     nodesDestroyNode(*pNode);
     *pNode = nodesCloneNode(pCxt->replaceCxt.pNew);
     if (NULL == *pNode) {
       pCxt->code = TSDB_CODE_OUT_OF_MEMORY;
       return DEAL_RES_ERROR;
     }
+    strcpy(((SExprNode*)*pNode)->aliasName, aliasName);
 
     pCxt->replaceCxt.replaced = true;
     return DEAL_RES_END;
@@ -211,7 +214,6 @@ static int32_t calcConstProject(SCalcConstContext* pCxt, SNode* pProject, bool d
   }
 
   char aliasName[TSDB_COL_NAME_LEN] = {0};
-  strcpy(aliasName, ((SExprNode*)pProject)->aliasName);
   int32_t code = TSDB_CODE_SUCCESS;
   if (dual) {
     code = scalarCalculateConstantsFromDual(pProject, pNew);
@@ -219,15 +221,20 @@ static int32_t calcConstProject(SCalcConstContext* pCxt, SNode* pProject, bool d
     code = scalarCalculateConstants(pProject, pNew);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    strcpy(((SExprNode*)*pNew)->aliasName, aliasName);
     if (QUERY_NODE_VALUE == nodeType(*pNew) && NULL != pAssociation) {
       int32_t size = taosArrayGetSize(pAssociation);
       for (int32_t i = 0; i < size; ++i) {
         SAssociationNode* pAssNode = taosArrayGet(pAssociation, i);
         SNode** pCol = pAssNode->pPlace;
         if (*pCol == pAssNode->pAssociationNode) {
+          strcpy(aliasName, ((SExprNode*)*pCol)->aliasName);
+          SArray* pOrigAss = NULL;
+          TSWAP(((SExprNode*)*pCol)->pAssociation, pOrigAss);
           nodesDestroyNode(*pCol);
           *pCol = nodesCloneNode(*pNew);
+          TSWAP(pOrigAss, ((SExprNode*)*pCol)->pAssociation);
+          taosArrayDestroy(pOrigAss);
+          strcpy(((SExprNode*)*pCol)->aliasName, aliasName);
           if (NULL == *pCol) {
             code = TSDB_CODE_OUT_OF_MEMORY;
             break;
diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c
index f50655f8e0..2b8516d37b 100644
--- a/source/libs/parser/src/parInsertSql.c
+++ b/source/libs/parser/src/parInsertSql.c
@@ -1584,8 +1584,7 @@ typedef union SRowsDataContext{
   SStbRowsDataContext* pStbRowsCxt;
 } SRowsDataContext;
 
-static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, 
-                                char* ctbName, bool* pFoundCtbName) {
+static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, bool* pFoundCtbName) {
   *pFoundCtbName = false;
   int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg);
   if (code == TSDB_CODE_SUCCESS){
@@ -1595,7 +1594,13 @@ static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext*
 
     if (pToken->n > 0) {
       if (pToken->n <= TSDB_TABLE_NAME_LEN - 1) {
-        memcpy(pStbRowsCxt->ctbName.tname, pToken->z, pToken->n);
+        for (int i = 0; i < pToken->n; ++i) {
+          if (pToken->z[i] == '.') {
+            return buildInvalidOperationMsg(&pCxt->msg, "tbname can not contain '.'");
+          } else {
+            pStbRowsCxt->ctbName.tname[i] = pToken->z[i]; 
+          }
+        }
         pStbRowsCxt->ctbName.tname[pToken->n] = '\0';
         *pFoundCtbName = true;
       } else {
@@ -1677,8 +1682,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
       }
     }
     else if (pCols->pColIndex[i] == tbnameIdx) {
-      char ctbName[TSDB_TABLE_NAME_LEN];
-      code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, ctbName, bFoundTbName);
+      code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, bFoundTbName);
     }
 
     if (code == TSDB_CODE_SUCCESS && i < pCols->numOfBound - 1) {
diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c
index 222aec9813..e0d154a130 100644
--- a/source/libs/planner/src/planLogicCreater.c
+++ b/source/libs/planner/src/planLogicCreater.c
@@ -266,7 +266,7 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols
 
   if (NULL == pScanCols) {
     if (NULL == pScanPseudoCols) {
-      return SCAN_TYPE_TABLE;
+      return (!tagScan) ? SCAN_TYPE_TABLE : SCAN_TYPE_TAG;
     }
     return FUNCTION_TYPE_BLOCK_DIST_INFO == ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType
                ? SCAN_TYPE_BLOCK_INFO
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index 8d4c042960..706fada4e8 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -2478,6 +2478,27 @@ static bool hasSuitableCache(int8_t cacheLastMode, bool hasLastRow, bool hasLast
   return false;
 }
 
+/// @brief check if we can apply last row scan optimization
+/// @param lastColNum how many distinct last col specified
+/// @param lastColId only used when lastColNum equals 1, the col id of the only one last col
+/// @param selectNonPKColNum num of normal cols
+/// @param selectNonPKColId only used when selectNonPKColNum equals 1, the col id of the only one select col
+static bool lastRowScanOptCheckColNum(int32_t lastColNum, col_id_t lastColId,
+                                      int32_t selectNonPKColNum, col_id_t selectNonPKColId) {
+  // multi select non pk col + last func: select c1, c2, last(c1)
+  if (selectNonPKColNum > 1 && lastColNum > 0) return false;
+
+  if (selectNonPKColNum == 1) {
+    // select last(c1), last(c2), c1 ...
+    // which is not possible currently
+    if (lastColNum > 1) return false;
+
+    // select last(c1), c2 ...
+    if (lastColNum == 1 && lastColId != selectNonPKColId) return false;
+  }
+  return true;
+}
+
 static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
   if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) ||
       QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0))) {
@@ -2493,9 +2514,10 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
     return false;
   }
 
-  bool   hasLastFunc = false;
-  bool   hasSelectFunc = false;
-  SNode* pFunc = NULL;
+  bool     hasNonPKSelectFunc = false;
+  SNode*   pFunc = NULL;
+  int32_t  lastColNum = 0, selectNonPKColNum = 0;
+  col_id_t lastColId = -1, selectNonPKColId = -1;
   FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) {
     SFunctionNode* pAggFunc = (SFunctionNode*)pFunc;
     if (FUNCTION_TYPE_LAST == pAggFunc->funcType) {
@@ -2505,16 +2527,33 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) {
         if (pCol->colType != COLUMN_TYPE_COLUMN) {
           return false;
         }
+        if (lastColId != pCol->colId) {
+          lastColId = pCol->colId;
+          lastColNum++;
+        }
       }
-      if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) {
+      if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) {
         return false;
       }
-      hasLastFunc = true;
+      if (!lastRowScanOptCheckColNum(lastColNum, lastColId, selectNonPKColNum, selectNonPKColId))
+        return false;
     } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType) {
-      if (hasLastFunc) {
+      SNode* pParam = nodesListGetNode(pAggFunc->pParameterList, 0);
+      if (QUERY_NODE_COLUMN == nodeType(pParam)) {
+        SColumnNode* pCol = (SColumnNode*)pParam;
+        if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) {
+          if (selectNonPKColId != pCol->colId) {
+            selectNonPKColId = pCol->colId;
+            selectNonPKColNum++;
+          }
+        } else {
+          continue;
+        }
+      } else if (lastColNum > 0) {
         return false;
       }
-      hasSelectFunc = true;
+      if (!lastRowScanOptCheckColNum(lastColNum, lastColId, selectNonPKColNum, selectNonPKColId))
+        return false;
     } else if (FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) {
       if (!lastRowScanOptLastParaIsTag(nodesListGetNode(pAggFunc->pParameterList, 0))) {
         return false;
@@ -2581,6 +2620,9 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
 
   SLastRowScanOptSetColDataTypeCxt cxt = {.doAgg = true, .pLastCols = NULL};
   SNode*                           pNode = NULL;
+  SColumnNode*                     pPKTsCol = NULL;
+  SColumnNode*                     pNonPKCol = NULL;
+
   FOREACH(pNode, pAgg->pAggFuncs) {
     SFunctionNode* pFunc = (SFunctionNode*)pNode;
     int32_t        funcType = pFunc->funcType;
@@ -2597,6 +2639,16 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
         nodesWalkExpr(nodesListGetNode(pFunc->pParameterList, 0), lastRowScanOptSetColDataType, &cxt);
         nodesListErase(pFunc->pParameterList, nodesListGetCell(pFunc->pParameterList, 1));
       }
+    } else if (FUNCTION_TYPE_SELECT_VALUE == funcType) {
+      pNode = nodesListGetNode(pFunc->pParameterList, 0);
+      if (nodeType(pNode) == QUERY_NODE_COLUMN) {
+        SColumnNode* pCol = (SColumnNode*)pNode;
+        if (pCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+          pPKTsCol = pCol;
+        } else {
+          pNonPKCol = pCol;
+        }
+      }
     }
   }
 
@@ -2608,6 +2660,16 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic
     lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols, true);
     nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt);
     lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, false);
+    if (pPKTsCol && pScan->node.pTargets->length == 1) {
+      // when select last(ts),ts from ..., we add another ts to targets
+      sprintf(pPKTsCol->colName, "#sel_val.%p", pPKTsCol);
+      nodesListAppend(pScan->node.pTargets, nodesCloneNode((SNode*)pPKTsCol));
+    }
+    if (pNonPKCol && cxt.pLastCols->length == 1 && nodesEqualNode((SNode*)pNonPKCol, nodesListGetNode(cxt.pLastCols, 0))) {
+      // when select last(c1), c1 from ..., we add c1 to targets
+      sprintf(pNonPKCol->colName, "#sel_val.%p", pNonPKCol);
+      nodesListAppend(pScan->node.pTargets, nodesCloneNode((SNode*)pNonPKCol));
+    }
     nodesClearList(cxt.pLastCols);
   }
   pAgg->hasLastRow = false;
diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c
index d6799a25a7..5cf3426e6f 100644
--- a/source/libs/planner/src/planPhysiCreater.c
+++ b/source/libs/planner/src/planPhysiCreater.c
@@ -552,6 +552,7 @@ static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSu
   if (NULL == pScan) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pScan->pTargets = nodesCloneList(pScanLogicNode->node.pTargets);
 
   pScan->pGroupTags = nodesCloneList(pScanLogicNode->pGroupTags);
   if (NULL != pScanLogicNode->pGroupTags && NULL == pScan->pGroupTags) {
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index dbdd79cc65..e7c6297f44 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -1230,7 +1230,6 @@ int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam
     code = taosChar2Ts(format, &formats, tsStr, &ts, precision, errMsg, 128);
     if (code) {
       qError("func to_timestamp failed %s", errMsg);
-      code = code == -1 ? TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR : TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR;
       break;
     }
     colDataSetVal(pOutput->columnData, i, (char *)&ts, false);
diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index 4a7d3a2a05..b76a967d0d 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -106,7 +106,6 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
 int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq);
 
 int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId);
-int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
 int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet);
 
 int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t srcTaskId, int32_t index, int64_t checkpointId);
@@ -140,6 +139,18 @@ void*         streamQueueNextItem(SStreamQueue* pQueue);
 void          streamFreeQitem(SStreamQueueItem* data);
 int32_t       streamQueueGetItemSize(const SStreamQueue* pQueue);
 
+typedef enum UPLOAD_TYPE{
+  UPLOAD_DISABLE = -1,
+  UPLOAD_S3 = 0,
+  UPLOAD_RSYNC = 1,
+} UPLOAD_TYPE;
+
+UPLOAD_TYPE getUploadType();
+int uploadCheckpoint(char* id, char* path);
+int downloadCheckpoint(char* id, char* path);
+int deleteCheckpoint(char* id);
+int deleteCheckpointFile(char* id, char* name);
+
 int32_t onNormalTaskReady(SStreamTask* pTask);
 int32_t onScanhistoryTaskReady(SStreamTask* pTask);
 
diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c
index 48b6486e05..5540e3b6fd 100644
--- a/source/libs/stream/src/streamCheckpoint.c
+++ b/source/libs/stream/src/streamCheckpoint.c
@@ -14,6 +14,8 @@
  */
 
 #include "streamInt.h"
+#include "rsync.h"
+#include "cos.h"
 
 int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) {
   if (tStartEncode(pEncoder) < 0) return -1;
@@ -375,3 +377,91 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
 
   return code;
 }
+
+static int uploadCheckpointToS3(char* id, char* path){
+  TdDirPtr pDir = taosOpenDir(path);
+  if (pDir == NULL) return -1;
+
+  TdDirEntryPtr de = NULL;
+  while ((de = taosReadDir(pDir)) != NULL) {
+    char* name = taosGetDirEntryName(de);
+    if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 ||
+        taosDirEntryIsDir(de)) continue;
+
+    char filename[PATH_MAX] = {0};
+    if(path[strlen(path) - 1] == TD_DIRSEP_CHAR){
+      snprintf(filename, sizeof(filename), "%s%s", path, name);
+    }else{
+      snprintf(filename, sizeof(filename), "%s%s%s", path, TD_DIRSEP, name);
+    }
+
+    char object[PATH_MAX] = {0};
+    snprintf(object, sizeof(object), "%s%s%s", id, TD_DIRSEP, name);
+
+    if(s3PutObjectFromFile2(filename, object) != 0){
+      taosCloseDir(&pDir);
+      return -1;
+    }
+    stDebug("[s3] upload checkpoint:%s", filename);
+  }
+  taosCloseDir(&pDir);
+
+  return 0;
+}
+
+UPLOAD_TYPE getUploadType(){
+  if(strlen(tsSnodeAddress) != 0){
+    return UPLOAD_RSYNC;
+  }else if(tsS3StreamEnabled){
+    return UPLOAD_S3;
+  }else{
+    return UPLOAD_DISABLE;
+  }
+}
+
+int uploadCheckpoint(char* id, char* path){
+  if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){
+    stError("uploadCheckpoint parameters invalid");
+    return -1;
+  }
+  if(strlen(tsSnodeAddress) != 0){
+    return uploadRsync(id, path);
+  }else if(tsS3StreamEnabled){
+    return uploadCheckpointToS3(id, path);
+  }
+  return 0;
+}
+
+int downloadCheckpoint(char* id, char* path){
+  if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){
+    stError("downloadCheckpoint parameters invalid");
+    return -1;
+  }
+  if(strlen(tsSnodeAddress) != 0){
+    return downloadRsync(id, path);
+  }else if(tsS3StreamEnabled){
+    return s3GetObjectsByPrefix(id, path);
+  }
+  return 0;
+}
+
+int deleteCheckpoint(char* id){
+  if(id == NULL || strlen(id) == 0){
+    stError("deleteCheckpoint parameters invalid");
+    return -1;
+  }
+  if(strlen(tsSnodeAddress) != 0){
+    return deleteRsync(id);
+  }else if(tsS3StreamEnabled){
+    s3DeleteObjectsByPrefix(id);
+  }
+  return 0;
+}
+
+int deleteCheckpointFile(char* id, char* name){
+  char object[128] = {0};
+  snprintf(object, sizeof(object), "%s/%s", id, name);
+  char *tmp = object;
+  s3DeleteObjects((const char**)&tmp, 1);
+  return 0;
+}
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index f364ed889d..6b82e6683c 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -28,7 +28,6 @@ int32_t streamBackendId = 0;
 int32_t streamBackendCfWrapperId = 0;
 int32_t streamMetaId = 0;
 
-static int64_t streamGetLatestCheckpointId(SStreamMeta* pMeta);
 static void    metaHbToMnode(void* param, void* tmrId);
 static void    streamMetaClear(SStreamMeta* pMeta);
 static int32_t streamMetaBegin(SStreamMeta* pMeta);
@@ -191,11 +190,11 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
       taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
   pMeta->chkpSaved = taosArrayInit(4, sizeof(int64_t));
   pMeta->chkpInUse = taosArrayInit(4, sizeof(int64_t));
-  pMeta->chkpCap = 8;
+  pMeta->chkpCap = 2;
   taosInitRWLatch(&pMeta->chkpDirLock);
 
-  pMeta->chkpId = streamGetLatestCheckpointId(pMeta);
-  pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, vgId);
+  pMeta->chkpId = streamMetaGetLatestCheckpointId(pMeta);
+  pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId);
   while (pMeta->streamBackend == NULL) {
     taosMsleep(100);
     pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, vgId);
@@ -603,7 +602,7 @@ int32_t streamMetaCommit(SStreamMeta* pMeta) {
   return 0;
 }
 
-int64_t streamGetLatestCheckpointId(SStreamMeta* pMeta) {
+int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) {
   int64_t chkpId = 0;
 
   TBC* pCur = NULL;
diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c
index 3de5de9967..103fd9e087 100644
--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@@ -19,7 +19,6 @@
 #include "streamBackendRocksdb.h"
 #include "streamInt.h"
 #include "tcommon.h"
-#include "streamInt.h"
 
 enum SBackendFileType {
   ROCKSDB_OPTIONS_TYPE = 1,
@@ -52,6 +51,7 @@ struct SStreamSnapHandle {
   int8_t         filetype;
   SArray*        pFileList;
   int32_t        currFileIdx;
+  int8_t         delFlag;  // 0 : not del, 1: del
 };
 struct SStreamSnapBlockHdr {
   int8_t  type;
@@ -148,6 +148,7 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk
       taosMemoryFree(tdir);
       return code;
     }
+    pHandle->delFlag = 1;
     chkpId = 0;
   }
 
@@ -193,8 +194,8 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk
       taosArrayPush(pFile->pSst, &sst);
     }
   }
-  {
-    char* buf = taosMemoryCalloc(1, 512);
+  if (qDebugFlag & DEBUG_TRACE) {
+    char* buf = taosMemoryCalloc(1, 128 + taosArrayGetSize(pFile->pSst) * 16);
     sprintf(buf, "[current: %s,", pFile->pCurrent);
     sprintf(buf + strlen(buf), "MANIFEST: %s,", pFile->pMainfest);
     sprintf(buf + strlen(buf), "options: %s,", pFile->pOptions);
@@ -274,7 +275,7 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) {
   if (handle->checkpointId == 0) {
     // del tmp dir
     if (pFile && taosIsDir(pFile->path)) {
-      taosRemoveDir(pFile->path);
+      if (handle->delFlag) taosRemoveDir(pFile->path);
     }
   } else {
     streamBackendDelInUseChkp(handle->handle, handle->checkpointId);
@@ -344,10 +345,10 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si
   stDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 ", file no.%d", STREAM_STATE_TRANSFER,
           item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
   uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize);
-  if(buf == NULL){
+  if (buf == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  int64_t  nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset);
+  int64_t nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset);
   if (nread == -1) {
     taosMemoryFree(buf);
     code = TAOS_SYSTEM_ERROR(terrno);
@@ -423,6 +424,7 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path
   pHandle->pFileList = list;
   pHandle->currFileIdx = 0;
   pHandle->offset = 0;
+  pHandle->delFlag = 0;
 
   *ppWriter = pWriter;
   return 0;
@@ -480,8 +482,8 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa
 }
 int32_t streamSnapWriterClose(SStreamSnapWriter* pWriter, int8_t rollback) {
   SStreamSnapHandle* handle = &pWriter->handle;
-  if (qDebugFlag & DEBUG_DEBUG) {
-    char* buf = (char*)taosMemoryMalloc(1024);
+  if (qDebugFlag & DEBUG_TRACE) {
+    char* buf = (char*)taosMemoryMalloc(128 + taosArrayGetSize(handle->pFileList) * 16);
     int   n = sprintf(buf, "[");
     for (int i = 0; i < taosArrayGetSize(handle->pFileList); i++) {
       SBackendFileItem* item = taosArrayGet(handle->pFileList, i);
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index fb0090ec6d..6ca7bc5e7b 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -221,7 +221,6 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz
   }
 
   pState->pTdbState->pOwner = pTask;
-  pState->checkPointId = 0;
 
   return pState;
 
@@ -274,7 +273,6 @@ int32_t streamStateCommit(SStreamState* pState) {
     SStreamSnapshot* pShot = getSnapshot(pState->pFileState);
     flushSnapshot(pState->pFileState, pShot, true);
   }
-  pState->checkPointId++;
   return 0;
 #else
   if (tdbCommit(pState->pTdbState->db, pState->pTdbState->txn) < 0) {
@@ -288,7 +286,6 @@ int32_t streamStateCommit(SStreamState* pState) {
                TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
     return -1;
   }
-  pState->checkPointId++;
   return 0;
 #endif
 }
diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c
index 584e81fafc..fc47498a3c 100644
--- a/source/libs/stream/src/tstreamFileState.c
+++ b/source/libs/stream/src/tstreamFileState.c
@@ -27,6 +27,9 @@
 #define DEFAULT_MAX_STREAM_BUFFER_SIZE (128 * 1024 * 1024)
 #define MIN_NUM_OF_ROW_BUFF            10240
 
+#define TASK_KEY                       "streamFileState"
+#define STREAM_STATE_INFO_NAME         "StreamStateCheckPoint"
+
 struct SStreamFileState {
   SList*   usedBuffs;
   SList*   freeBuffs;
@@ -113,6 +116,15 @@ void* sessionCreateStateKey(SRowBuffPos* pPos, int64_t num) {
   return pStateKey;
 }
 
+static void streamFileStateDecode(TSKEY* pKey, void* pBuff, int32_t len) { pBuff = taosDecodeFixedI64(pBuff, pKey); }
+
+static void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) {
+  *pLen = sizeof(TSKEY);
+  (*pVal) = taosMemoryCalloc(1, *pLen);
+  void* buff = *pVal;
+  taosEncodeFixedI64(&buff, *pKey);
+}
+
 SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize,
                                       GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId,
                                       int8_t type) {
@@ -181,6 +193,15 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_
     recoverSesssion(pFileState, checkpointId);
   }
 
+  void*   valBuf = NULL;
+  int32_t len = 0;
+  int32_t code = streamDefaultGet_rocksdb(pFileState->pFileStore, STREAM_STATE_INFO_NAME, &valBuf, &len);
+  if (code == TSDB_CODE_SUCCESS) {
+    ASSERT(len == sizeof(TSKEY));
+    streamFileStateDecode(&pFileState->flushMark, valBuf, len);
+    qDebug("===stream===flushMark  read:%" PRId64, pFileState->flushMark);
+  }
+  taosMemoryFreeClear(valBuf);
   return pFileState;
 
 _error:
@@ -506,15 +527,6 @@ SStreamSnapshot* getSnapshot(SStreamFileState* pFileState) {
   return pFileState->usedBuffs;
 }
 
-void streamFileStateDecode(TSKEY* pKey, void* pBuff, int32_t len) { pBuff = taosDecodeFixedI64(pBuff, pKey); }
-
-void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) {
-  *pLen = sizeof(TSKEY);
-  (*pVal) = taosMemoryCalloc(1, *pLen);
-  void* buff = *pVal;
-  taosEncodeFixedI64(&buff, *pKey);
-}
-
 int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState) {
   int32_t   code = TSDB_CODE_SUCCESS;
   SListIter iter = {0};
@@ -538,6 +550,7 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
       continue;
     }
     pPos->beFlushed = true;
+    pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey));
 
     qDebug("===stream===flushed start:%" PRId64, pFileState->getTs(pPos->pKey));
     if (streamStateGetBatchSize(batch) >= BATCH_LIMIT) {
@@ -565,24 +578,12 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
          pFileState->id, numOfElems, BATCH_LIMIT, elapsed);
 
   if (flushState) {
-    const char* taskKey = "streamFileState";
-    {
-      char    keyBuf[128] = {0};
-      void*   valBuf = NULL;
-      int32_t len = 0;
-      sprintf(keyBuf, "%s:%" PRId64 "", taskKey, ((SStreamState*)pFileState->pFileStore)->checkPointId);
-      streamFileStateEncode(&pFileState->flushMark, &valBuf, &len);
-      streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len, 0);
-      taosMemoryFree(valBuf);
-    }
-    {
-      char    keyBuf[128] = {0};
-      char    valBuf[64] = {0};
-      int32_t len = 0;
-      memcpy(keyBuf, taskKey, strlen(taskKey));
-      len = sprintf(valBuf, "%" PRId64 "", ((SStreamState*)pFileState->pFileStore)->checkPointId);
-      code = streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len, 0);
-    }
+    void*   valBuf = NULL;
+    int32_t len = 0;
+    streamFileStateEncode(&pFileState->flushMark, &valBuf, &len);
+    qDebug("===stream===flushMark write:%" PRId64, pFileState->flushMark);
+    streamStatePutBatch(pFileState->pFileStore, "default", batch, STREAM_STATE_INFO_NAME, valBuf, len, 0);
+    taosMemoryFree(valBuf);
     streamStatePutBatch_rocksdb(pFileState->pFileStore, batch);
   }
 
@@ -591,26 +592,23 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
 }
 
 int32_t forceRemoveCheckpoint(SStreamFileState* pFileState, int64_t checkpointId) {
-  const char* taskKey = "streamFileState";
   char        keyBuf[128] = {0};
-  sprintf(keyBuf, "%s:%" PRId64 "", taskKey, checkpointId);
+  sprintf(keyBuf, "%s:%" PRId64 "", TASK_KEY, checkpointId);
   return streamDefaultDel_rocksdb(pFileState->pFileStore, keyBuf);
 }
 
 int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list) {
-  const char* taskKey = "streamFileState";
-  return streamDefaultIterGet_rocksdb(pFileState->pFileStore, taskKey, NULL, list);
+  return streamDefaultIterGet_rocksdb(pFileState->pFileStore, TASK_KEY, NULL, list);
 }
 
 int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) {
   int32_t     code = TSDB_CODE_SUCCESS;
-  const char* taskKey = "streamFileState";
   int64_t     maxCheckPointId = 0;
   {
     char    buf[128] = {0};
     void*   val = NULL;
     int32_t len = 0;
-    memcpy(buf, taskKey, strlen(taskKey));
+    memcpy(buf, TASK_KEY, strlen(TASK_KEY));
     code = streamDefaultGet_rocksdb(pFileState->pFileStore, buf, &val, &len);
     if (code != 0 || len == 0 || val == NULL) {
       return TSDB_CODE_FAILED;
@@ -624,7 +622,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) {
     char    buf[128] = {0};
     void*   val = 0;
     int32_t len = 0;
-    sprintf(buf, "%s:%" PRId64 "", taskKey, i);
+    sprintf(buf, "%s:%" PRId64 "", TASK_KEY, i);
     code = streamDefaultGet_rocksdb(pFileState->pFileStore, buf, &val, &len);
     if (code != 0) {
       return TSDB_CODE_FAILED;
diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt
index 629b04ae51..d756b71e29 100644
--- a/source/libs/stream/test/CMakeLists.txt
+++ b/source/libs/stream/test/CMakeLists.txt
@@ -18,6 +18,17 @@ TARGET_INCLUDE_DIRECTORIES(
   PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
 )
 
+ADD_EXECUTABLE(checkpointTest checkpointTest.cpp)
+TARGET_LINK_LIBRARIES(
+        checkpointTest
+        PUBLIC os common gtest stream executor qcom index transport util
+)
+
+TARGET_INCLUDE_DIRECTORIES(
+        checkpointTest
+        PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
+)
+
 add_test(
   NAME streamUpdateTest
   COMMAND streamUpdateTest
diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp
new file mode 100644
index 0000000000..dca2e97c28
--- /dev/null
+++ b/source/libs/stream/test/checkpointTest.cpp
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+
+#include <taoserror.h>
+#include <tglobal.h>
+#include <iostream>
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+#include "rsync.h"
+#include "streamInt.h"
+#include "cos.h"
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+
+  if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) {
+    printf("error");
+  }
+  if (s3Init() < 0) {
+    return -1;
+  }
+  strcpy(tsSnodeAddress, "127.0.0.1");
+  return RUN_ALL_TESTS();
+}
+
+TEST(testCase, checkpointUpload_Test) {
+  stopRsync();
+  startRsync();
+
+  taosSsleep(5);
+  char* id = "2013892036";
+
+  uploadCheckpoint(id, "/root/offset/");
+}
+
+TEST(testCase, checkpointDownload_Test) {
+  char* id = "2013892036";
+  downloadCheckpoint(id, "/root/offset/download/");
+}
+
+TEST(testCase, checkpointDelete_Test) {
+  char* id = "2013892036";
+  deleteCheckpoint(id);
+}
+
+TEST(testCase, checkpointDeleteFile_Test) {
+  char* id = "2013892036";
+  deleteCheckpointFile(id, "offset-ver0");
+}
diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c
index 8e186af51f..95952c960e 100644
--- a/source/libs/sync/src/syncSnapshot.c
+++ b/source/libs/sync/src/syncSnapshot.c
@@ -865,6 +865,9 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot
   ASSERT(pRcvBuf->start <= pRcvBuf->cursor + 1 && pRcvBuf->cursor < pRcvBuf->end);
 
   if (pMsg->seq > pRcvBuf->cursor) {
+    if (pRcvBuf->entries[pMsg->seq % pRcvBuf->size]) {
+      pRcvBuf->entryDeleteCb(pRcvBuf->entries[pMsg->seq % pRcvBuf->size]);
+    }
     pRcvBuf->entries[pMsg->seq % pRcvBuf->size] = pMsg;
     ppMsg[0] = NULL;
     pRcvBuf->end = TMAX(pMsg->seq + 1, pRcvBuf->end);
@@ -1002,7 +1005,8 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) {
   }
 
   if (pMsg->term < raftStoreGetTerm(pSyncNode)) {
-    syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "reject since small term");
+    sRError(pReceiver, "reject snap replication with smaller term. msg term:%" PRId64 ", seq:%d", pMsg->term,
+            pMsg->seq);
     terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
     return -1;
   }
diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c
index 9e6ea94e78..06847c081c 100644
--- a/source/libs/sync/src/syncUtil.c
+++ b/source/libs/sync/src/syncUtil.c
@@ -268,7 +268,7 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla
 
   taosPrintLog(flags, level, dflag,
                "vgId:%d, %s, sync:%s, snap-sender:%p signature:(%" PRId64 ", %" PRId64 "), {start:%" PRId64
-               " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64
+               " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRId64 " last-cfg:%" PRId64
                ", seq:%d, ack:%d, "
                " buf:[%" PRId64 " %" PRId64 ", %" PRId64
                "), finish:%d, as:%d, to-dnode:%d}"
diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c
index 3843f2cb44..bb282b4ee3 100644
--- a/source/util/src/tconfig.c
+++ b/source/util/src/tconfig.c
@@ -369,6 +369,15 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p
   }
 
   switch (pItem->dtype) {
+    case CFG_DTYPE_BOOL: {
+      int32_t ival = (int32_t)atoi(pVal);
+      if (ival != 0 && ival != 1) {
+        uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name,
+               cfgDtypeStr(pItem->dtype), ival);
+        terrno = TSDB_CODE_OUT_OF_RANGE;
+        return -1;
+      }
+    } break;
     case CFG_DTYPE_INT32: {
       int32_t ival = (int32_t)atoi(pVal);
       if (ival < pItem->imin || ival > pItem->imax) {
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 6c964c3da0..bcdbb3e3ac 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -613,8 +613,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_TYPE,        "Invalid function par
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_VALUE,       "Invalid function para value")
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION,      "Not buildin function")
 TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP,            "Duplicate timestamps not allowed in function")
-TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR, "Func to_timestamp failed, format mismatch")
-TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR, "Func to_timestamp failed, wrong timestamp")
+TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR, "Func to_timestamp failed for format mismatch")
+TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR, "Func to_timestamp failed for wrong timestamp")
+TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED, "Func to_timestamp failed for unsupported timestamp format")
 
 //udf
 TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING,                   "udf is stopping")
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 21dcd16441..efb9d3c667 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -69,6 +69,10 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py
@@ -225,6 +229,7 @@ e
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/ttlChangeOnWrite.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compress_tsz1.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compress_tsz2.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/view/non_marterial_view/test_view.py
 ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py
 ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py
 ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py
@@ -232,6 +237,8 @@ e
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep1.py -N 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3
 ,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py
+
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py
@@ -282,6 +289,7 @@ e
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionUS.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionNS.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4219.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py
@@ -1175,6 +1183,7 @@ e
 ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
 ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
 ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
+,,y,script,./test.sh -f tsim/sync/vnodesnapshot-rsma-test.sim
 ,,n,script,./test.sh -f tsim/valgrind/checkError1.sim
 ,,n,script,./test.sh -f tsim/valgrind/checkError2.sim
 ,,n,script,./test.sh -f tsim/valgrind/checkError3.sim
@@ -1288,6 +1297,7 @@ e
 ,,y,script,./test.sh -f tsim/tagindex/add_index.sim
 ,,n,script,./test.sh -f tsim/tagindex/sma_and_tag_index.sim
 ,,y,script,./test.sh -f tsim/view/view.sim
+,,y,script,./test.sh -f tsim/query/const.sim 
 
 
 #develop test
diff --git a/tests/pytest/util/autogen.py b/tests/pytest/util/autogen.py
index 6f5c89d15b..e1227a680f 100644
--- a/tests/pytest/util/autogen.py
+++ b/tests/pytest/util/autogen.py
@@ -133,8 +133,6 @@ class AutoGen:
             if batch_size == 1 or (i > 0 and i % batch_size == 0) :
                 sql = f"insert into {child_name} values {values}"
                 tdSql.execute(sql)
-                if batch_size > 40:
-                   tdLog.info(f" insert data i={i}")
                 values = ""
 
         # end batch
diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py
index 7dcf6bc3f2..c05df0a852 100644
--- a/tests/pytest/util/sql.py
+++ b/tests/pytest/util/sql.py
@@ -88,7 +88,7 @@ class TDSql:
             expectErrNotOccured = False
             self.errno = e.errno
             error_info = repr(e)
-            self.error_info = error_info[error_info.index('(')+1:-1].split(",")[0].replace("'","")
+            self.error_info = ','.join(error_info[error_info.index('(')+1:-1].split(",")[:-1]).replace("'","")
             # self.error_info = (','.join(error_info.split(",")[:-1]).split("(",1)[1:][0]).replace("'","")
         if expectErrNotOccured:
             tdLog.exit("%s(%d) failed: sql:%s, expect error not occured" % (caller.filename, caller.lineno, sql))
@@ -106,7 +106,7 @@ class TDSql:
               tdLog.info("sql:%s, expect error occured" % (sql))
 
             if expectErrInfo != None:
-                if  expectErrInfo == self.error_info:
+                if  expectErrInfo == self.error_info or expectErrInfo in self.error_info:
                     tdLog.info("sql:%s, expected expectErrInfo %s occured" % (sql, expectErrInfo))
                 else:
                   tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected errno %s" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo))
diff --git a/tests/script/local.supp b/tests/script/local.supp
index 562cddba46..c351846f2c 100644
--- a/tests/script/local.supp
+++ b/tests/script/local.supp
@@ -225,3 +225,1433 @@
    fun:vnodeOpen
    fun:vmOpenVnodeInThread
 }
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_10FileSystemEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_GLOBAL__sub_I_fs_posix.cc
+   fun:__libc_csu_init
+   fun:(below main)
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_
+   fun:_ZZSt9call_onceIZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_22FileChecksumGenFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21DBOptionsConfigurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb19GetDBOptionsFromMapERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_
+   fun:_ZZSt9call_onceIZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_21SstPartitionerFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_18MemTableRepFactoryEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_
+   fun:_ZZSt9call_onceIZN7rocksdb16CompactionFilter16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb16CompactionFilter16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_16CompactionFilterEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_13MergeOperatorEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_ZN7rocksdbL29RegisterBuiltinMergeOperatorsERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb13MergeOperator16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb13MergeOperator16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_13MergeOperatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_Z41__static_initialization_and_destruction_0ii.constprop.0
+   fun:__libc_csu_init
+   fun:(below main)
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb13ObjectLibrary7DefaultEv
+   fun:_GLOBAL__sub_I_fs_posix.cc
+   fun:__libc_csu_init
+   fun:(below main)
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_10FileSystemEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_GLOBAL__sub_I_fs_posix.cc
+   fun:__libc_csu_init
+   fun:(below main)
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb11SystemClock7DefaultEv
+   fun:_ZN7rocksdb3Env7DefaultEv
+   fun:_ZN7rocksdb9DBOptionsC1Ev
+   fun:rocksdb_options_create
+   fun:tsdbOpenRocksCache
+   fun:tsdbOpenCache
+   fun:tsdbOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+   fun:vmProcessMgmtQueue
+   fun:tQWorkerThreadFp
+   fun:start_thread
+   fun:clone
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb10FileSystem7DefaultEv
+   fun:_ZN7rocksdb3Env7DefaultEv
+   fun:_ZN7rocksdb9DBOptionsC1Ev
+   fun:rocksdb_options_create
+   fun:tsdbOpenRocksCache
+   fun:tsdbOpenCache
+   fun:tsdbOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+   fun:vmProcessMgmtQueue
+   fun:tQWorkerThreadFp
+   fun:start_thread
+   fun:clone
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb14ObjectRegistry7DefaultEv
+   fun:_ZN7rocksdb14ObjectRegistry11NewInstanceEv
+   fun:_ZN7rocksdb13ConfigOptionsC1Ev
+   fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS3_INS_19ColumnFamilyOptionsESaIS8_EERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb6DBImpl16WriteOptionsFileEbb
+   fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb
+   fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_
+   fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_
+   fun:rocksdb_open
+   fun:tsdbOpenRocksCache
+   fun:tsdbOpenCache
+   fun:tsdbOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+   fun:vmProcessMgmtQueue
+   fun:tQWorkerThreadFp
+   fun:start_thread
+   fun:clone
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt6vectorISt10shared_ptrIN7rocksdb13ObjectLibraryEESaIS3_EE17_M_realloc_insertIJRKS3_EEEvN9__gnu_cxx17__normal_iteratorIPS3_S5_EEDpOT_
+   fun:_ZN7rocksdb14ObjectRegistryC1ERKSt10shared_ptrINS_13ObjectLibraryEE
+   fun:_ZN7rocksdb14ObjectRegistry7DefaultEv
+   fun:_ZN7rocksdb14ObjectRegistry11NewInstanceEv
+   fun:_ZN7rocksdb13ConfigOptionsC1Ev
+   fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS3_INS_19ColumnFamilyOptionsESaIS8_EERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb6DBImpl16WriteOptionsFileEbb
+   fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb
+   fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_
+   fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_
+   fun:rocksdb_open
+   fun:tsdbOpenRocksCache
+   fun:tsdbOpenCache
+   fun:tsdbOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+   fun:vmProcessMgmtQueue
+   fun:tQWorkerThreadFp
+   fun:start_thread
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryIKNS_14SliceTransformEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS5_St14default_deleteIS5_EEPSsEERKNS0_12PatternEntryESI_
+   fun:_ZN7rocksdbL29RegisterBuiltinSliceTransformERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb14SliceTransform16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIKS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb14SliceTransform16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIKS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrIKNS0_14SliceTransformEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_
+   fun:_ZZSt9call_onceIZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_23FlushBlockPolicyFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb22BlockBasedTableFactory11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb12Customizable18ConfigureNewObjectERKNS_13ConfigOptionsEPS0_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:_ZnwmRKSt9nothrow_t
+   fun:__cxa_thread_atexit
+   fun:_ZN7rocksdb17InstrumentedMutex4LockEv
+   fun:_ZN7rocksdb5TimerD1Ev
+   fun:__run_exit_handlers
+   fun:exit
+   fun:(below main)
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZN7rocksdbL29RegisterBuiltinSliceTransformERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb14SliceTransform16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIKS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb14SliceTransform16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIKS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrIKNS0_14SliceTransformEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_GLOBAL__sub_I_fs_posix.cc
+   fun:__libc_csu_init
+   fun:(below main)
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZN7rocksdbL29RegisterBuiltinMergeOperatorsERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb13MergeOperator16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb13MergeOperator16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_13MergeOperatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZN7rocksdb13ObjectLibrary7DefaultEv
+   fun:_GLOBAL__sub_I_fs_posix.cc
+   fun:__libc_csu_init
+   fun:(below main)
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_
+   fun:_ZZSt9call_onceIZN7rocksdb10Comparator16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb10Comparator16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_10ComparatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_
+   fun:_ZZSt9call_onceIZN7rocksdbL22RegisterTableFactoriesERKSsEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_10FileSystemEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_GLOBAL__sub_I_fs_posix.cc
+   fun:__libc_csu_init
+   fun:(below main)
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZZSt9call_onceIZN7rocksdb10Comparator16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb10Comparator16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_10ComparatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZZSt9call_onceIZN7rocksdbL22RegisterTableFactoriesERKSsEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_13MergeOperatorEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_ZN7rocksdbL29RegisterBuiltinMergeOperatorsERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb13MergeOperator16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb13MergeOperator16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_13MergeOperatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE9_M_rehashEmRKm
+   fun:_ZN7rocksdb19ThreadStatusUpdater14RegisterThreadENS_12ThreadStatus10ThreadTypeEm
+   fun:_ZN7rocksdb14ThreadPoolImpl4Impl15BGThreadWrapperEPv
+   fun:execute_native_thread_routine
+   fun:start_thread
+   fun:clone
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb24CacheEntryStatsCollectorINS_13InternalStats19CacheEntryRoleStatsEE9GetSharedEPNS_5CacheEPNS_11SystemClockEPSt10shared_ptrIS3_E
+   fun:_ZN7rocksdb13InternalStatsC1EiPNS_11SystemClockEPNS_16ColumnFamilyDataE
+   fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKSsjPNS_7VersionERKNS_19ColumnFamilyOptionsE
+   fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE
+   fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE
+   fun:_ZN7rocksdb18VersionEditHandler10InitializeEv
+   fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE
+   fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPSsb
+   fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPmPNS0_15RecoveryContextE
+   fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb
+   fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_
+   fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_
+   fun:rocksdb_open
+   fun:tsdbOpenRocksCache
+   fun:tsdbOpenCache
+   fun:tsdbOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZZSt9call_onceIZN7rocksdb16CompactionFilter16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb16CompactionFilter16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_16CompactionFilterEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_18MemTableRepFactoryEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZZSt9call_onceIZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_21SstPartitionerFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZZSt9call_onceIZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_22FileChecksumGenFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21DBOptionsConfigurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb19GetDBOptionsFromMapERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_10FileSystemEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_GLOBAL__sub_I_fs_posix.cc
+   fun:__libc_csu_init
+   fun:(below main)
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb18BytewiseComparatorEv
+   fun:_ZN7rocksdb19ColumnFamilyOptionsC1Ev
+   fun:rocksdb_options_create
+   fun:tsdbOpenRocksCache
+   fun:tsdbOpenCache
+   fun:tsdbOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+   fun:vmProcessMgmtQueue
+   fun:tQWorkerThreadFp
+   fun:start_thread
+   fun:clone
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZZSt9call_onceIZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_22FileChecksumGenFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21DBOptionsConfigurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb19GetDBOptionsFromMapERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS6_INS_19ColumnFamilyOptionsESaISB_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs
+   fun:_ZZSt9call_onceIZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_22FileChecksumGenFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21DBOptionsConfigurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb19GetDBOptionsFromMapERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdbL29RegisterBuiltinSliceTransformERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb14SliceTransform16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIKS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb14SliceTransform16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIKS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrIKNS0_14SliceTransformEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs
+   fun:_ZN7rocksdbL29RegisterBuiltinSliceTransformERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb14SliceTransform16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIKS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb14SliceTransform16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIKS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrIKNS0_14SliceTransformEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZZSt9call_onceIZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_21SstPartitionerFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS6_INS_19ColumnFamilyOptionsESaISB_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs
+   fun:_ZZSt9call_onceIZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_21SstPartitionerFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_18MemTableRepFactoryEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZZSt9call_onceIZN7rocksdb10Comparator16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb10Comparator16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_10ComparatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS6_INS_19ColumnFamilyOptionsESaISB_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs
+   fun:_ZZSt9call_onceIZN7rocksdb10Comparator16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb10Comparator16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_10ComparatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZZSt9call_onceIZN7rocksdbL22RegisterTableFactoriesERKSsEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS6_INS_19ColumnFamilyOptionsESaISB_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs
+   fun:_ZZSt9call_onceIZN7rocksdbL22RegisterTableFactoriesERKSsEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZZSt9call_onceIZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_23FlushBlockPolicyFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb22BlockBasedTableFactory11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb12Customizable18ConfigureNewObjectERKNS_13ConfigOptionsEPS0_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSs4_Rep9_S_createEmmRKSaIcE
+   fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag
+   fun:_ZNSsC1EPKcRKSaIcE
+   fun:_ZZSt9call_onceIZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_23FlushBlockPolicyFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb22BlockBasedTableFactory11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb12Customizable18ConfigureNewObjectERKNS_13ConfigOptionsEPS0_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs
+   fun:_ZZSt9call_onceIZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_23FlushBlockPolicyFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb22BlockBasedTableFactory11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb12Customizable18ConfigureNewObjectERKNS_13ConfigOptionsEPS0_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZZSt9call_onceIZN7rocksdb16CompactionFilter16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb16CompactionFilter16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_16CompactionFilterEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS6_INS_19ColumnFamilyOptionsESaISB_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs
+   fun:_ZZSt9call_onceIZN7rocksdb16CompactionFilter16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb16CompactionFilter16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_16CompactionFilterEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+   fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE
+   fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_13MergeOperatorEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_ZN7rocksdbL29RegisterBuiltinMergeOperatorsERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb13MergeOperator16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb13MergeOperator16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_13MergeOperatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_
+   fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE
+   fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb15ColumnFamilySetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb10VersionSetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb6DBImplC1ERKNS_9DBOptionsERKSsbbb
+   fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb
+   fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_
+   fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_
+   fun:rocksdb_open
+   fun:tsdbOpenRocksCache
+   fun:tsdbOpenCache
+   fun:tsdbOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+   fun:vmProcessMgmtQueue
+   fun:tQWorkerThreadFp
+   fun:start_thread
+   fun:clone
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_
+   fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE
+   fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKSsjPNS_7VersionERKNS_19ColumnFamilyOptionsE
+   fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE
+   fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE
+   fun:_ZN7rocksdb18VersionEditHandler10InitializeEv
+   fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE
+   fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPSsb
+   fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPmPNS0_15RecoveryContextE
+   fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb
+   fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_
+   fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_
+   fun:rocksdb_open
+   fun:tsdbOpenRocksCache
+   fun:tsdbOpenCache
+   fun:tsdbOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_
+   fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE
+   fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb15ColumnFamilySetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb10VersionSetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb6DBImplC1ERKNS_9DBOptionsERKSsbbb
+   fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb
+   fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_
+   fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_
+   fun:rocksdb_open
+   fun:streamBackendInit
+   fun:streamMetaOpen
+   fun:tqInitialize
+   fun:tqOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+   fun:vmProcessMgmtQueue
+   fun:tQWorkerThreadFp
+   fun:start_thread
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_
+   fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE
+   fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKSsjPNS_7VersionERKNS_19ColumnFamilyOptionsE
+   fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE
+   fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE
+   fun:_ZN7rocksdb18VersionEditHandler10InitializeEv
+   fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE
+   fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPSsb
+   fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPmPNS0_15RecoveryContextE
+   fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb
+   fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_
+   fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_
+   fun:rocksdb_open
+   fun:streamBackendInit
+   fun:streamMetaOpen
+   fun:tqInitialize
+   fun:tqOpen
+   fun:vnodeOpen
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryIKNS_14SliceTransformEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS5_St14default_deleteIS5_EEPSsEERKNS0_12PatternEntryESI_
+   fun:_ZN7rocksdbL29RegisterBuiltinSliceTransformERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb14SliceTransform16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIKS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb14SliceTransform16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIKS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrIKNS0_14SliceTransformEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_
+   fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_13MergeOperatorEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_ZN7rocksdbL29RegisterBuiltinMergeOperatorsERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb13MergeOperator16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb13MergeOperator16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_13MergeOperatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+   fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_
+   fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE9_M_rehashEmRKm
+   fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_
+   fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE
+   fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb15ColumnFamilySetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb10VersionSetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_
+   fun:_ZN7rocksdb6DBImplC1ERKNS_9DBOptionsERKSsbbb
+   fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb
+   fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_
+   fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_
+   fun:rocksdb_open
+   fun:tsdbOpenRocksCache
+   fun:tsdbOpenCache
+   fun:tsdbOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+   fun:vmProcessMgmtQueue
+   fun:tQWorkerThreadFp
+   fun:start_thread
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZNSt10_HashtableISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS6_EESaIS9_EEESaISC_ENSt8__detail10_Select1stESt8equal_toISsESt4hashISsENSE_18_Mod_range_hashingENSE_20_Default_ranged_hashENSE_20_Prime_rehash_policyENSE_17_Hashtable_traitsILb1ELb0ELb1EEEE9_M_rehashEmRKm
+   fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs
+   fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_18MemTableRepFactoryEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_
+   fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0
+   fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv
+   fun:__pthread_once_slow
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE
+   fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E
+   fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_
+   fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv
+   fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv
+   fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv
+   fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_
+   fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_
+   fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb3Env7DefaultEv
+   fun:_ZN7rocksdb9DBOptionsC1Ev
+   fun:rocksdb_options_create
+   fun:tsdbOpenRocksCache
+   fun:tsdbOpenCache
+   fun:tsdbOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+   fun:vmProcessMgmtQueue
+   fun:tQWorkerThreadFp
+   fun:start_thread
+   fun:clone
+}
+{
+   <insert_a_suppression_name_here>
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN7rocksdb14ThreadLocalPtr8InstanceEv
+   fun:_ZN7rocksdb3Env7DefaultEv
+   fun:_ZN7rocksdb9DBOptionsC1Ev
+   fun:rocksdb_options_create
+   fun:tsdbOpenRocksCache
+   fun:tsdbOpenCache
+   fun:tsdbOpen
+   fun:vnodeOpen
+   fun:vmProcessCreateVnodeReq
+   fun:vmProcessMgmtQueue
+   fun:tQWorkerThreadFp
+   fun:start_thread
+   fun:clone
+}
diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim
index 5d9e244565..4c0797e2a7 100644
--- a/tests/script/tsim/insert/insert_stb.sim
+++ b/tests/script/tsim/insert/insert_stb.sim
@@ -70,6 +70,7 @@ sql_error insert into d2.st values(now, 1, 1)
 sql_error insert into d2.st(ts, f) values(now, 1);
 sql_error insert into d2.st(ts, f, tbname) values(now, 1);
 sql_error insert into d2.st(ts, f, tbname) values(now, 1, '');
+sql_error insert into d2.st(ts, f, tbname) values(now, 1, 'd2.ct2');
 sql_error insert into d2.st(ts, tbname) values(now, 1, 34)
 sql_error insert into st using st2 tags(2) values(now,1);
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/query/const.sim b/tests/script/tsim/query/const.sim
new file mode 100644
index 0000000000..08f2b909c1
--- /dev/null
+++ b/tests/script/tsim/query/const.sim
@@ -0,0 +1,11 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql select b.z from (select c.a as z from (select 'a' as a) c) b;
+if $rows != 1 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/query/tag_scan.sim b/tests/script/tsim/query/tag_scan.sim
index 7404bd37fe..ee56a214a5 100644
--- a/tests/script/tsim/query/tag_scan.sim
+++ b/tests/script/tsim/query/tag_scan.sim
@@ -164,4 +164,9 @@ sql select count(*) from (select tags ts from stb34)
 if $data00 != 2 then
   return -1
 endi
+
+sql select tags 2 from stb34
+if $rows != 1 then 
+  return -1
+endi
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim
index 6f78829db7..c70f2dc20a 100644
--- a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim
+++ b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim
@@ -5,7 +5,7 @@ sleep 50
 sql connect
 
 #todo wait for streamState checkpoint
-return 1
+#return 1
 
 print =============== create database with retentions
 sql create database d0 retentions -:7d,5m:21d,15m:365d;
diff --git a/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim b/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim
index 3b3cd01521..8b1720d213 100644
--- a/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim
+++ b/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim
@@ -114,7 +114,7 @@ endi
 
 vg_ready:
 print ====>  create stable/child table
-sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) rollup(sum) watermark 3s,3s max_delay 3s,3s
+sql create table stb (ts timestamp, c1 float, c2 float, c3 double) tags (t1 int) rollup(sum) watermark 3s,3s max_delay 3s,3s
 
 sql show stables
 if $rows != 1 then 
@@ -167,9 +167,6 @@ system sh/exec.sh -n dnode4 -s start
 
 sleep 3000
 
-
-
-
 print =============== query data of level 1
 sql connect
 sql use db
@@ -181,12 +178,21 @@ if $rows != 100 then
   return -1
 endi
 
+print =============== sleep 5s to wait the result
+sleep 5000
+
 print =============== query data of level 2
 sql select * from ct1 where ts > now - 10d
-print rows of level 2: $rows
 print $data00 $data01 $data02
+print $data10 $data11 $data12
+if $rows != 100 then
+  print rows of level 2: $rows
+endi
 
 print =============== query data of level 3
 sql select * from ct1
-print rows of level 3: $rows
 print $data00 $data01 $data02
+print $data10 $data11 $data12
+if $rows != 100 then
+  print rows of level 3: $rows
+endi
\ No newline at end of file
diff --git a/tests/script/win-test-file b/tests/script/win-test-file
index 4ff4b52f7e..fe5f5c39e3 100644
--- a/tests/script/win-test-file
+++ b/tests/script/win-test-file
@@ -320,6 +320,7 @@
 ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim
 ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim
 ./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim
+./test.sh -f tsim/sync/vnodesnapshot-rsma-test.sim
 ./test.sh -f tsim/valgrind/checkError1.sim
 ./test.sh -f tsim/valgrind/checkError2.sim
 ./test.sh -f tsim/valgrind/checkError3.sim
diff --git a/tests/system-test/0-others/delete_check.py b/tests/system-test/0-others/delete_check.py
new file mode 100644
index 0000000000..ec589295e4
--- /dev/null
+++ b/tests/system-test/0-others/delete_check.py
@@ -0,0 +1,154 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+from util.sqlset import *
+from util.autogen import *
+
+import random
+import time
+import traceback
+import os
+from   os import path
+
+
+class TDTestCase:
+    # init 
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), True)
+        
+        # autoGen
+        self.autoGen = AutoGen()
+        # init cluster path
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+        self.projDir = f"{projPath}sim/"
+        tdLog.info(f" init projPath={self.projDir}")
+
+    def compactDatbase(self):
+        # compact database
+        tdSql.execute(f"compact database {self.dbname}", show=True)
+        waitSeconds = 20
+        if self.waitTranslation(waitSeconds) == False:
+            tdLog.exit(f"translation can not finish after wait {waitSeconds} seconds")
+            return          
+
+    # check tsdb folder empty
+    def check_tsdb_dir(self, tsdbDir):
+        for vfile in os.listdir(tsdbDir):
+            fileName, ext = os.path.splitext(vfile)
+            pathFile = os.path.join(tsdbDir, vfile)
+            ext = ext.lower()
+            tdLog.info(f"check exist file {pathFile} ...")
+            if ext == ".head" or ext == ".data" or ext == ".stt" or ext == ".sma":
+                tdLog.info(f"found {pathFile} not to be deleted ...")
+                real = True
+                for i in range(50):
+                    tdLog.info(f"i={i} compact and try again ...")
+                    self.compactDatbase()
+                    if os.path.exists(pathFile) is False:
+                        real = False
+                        break
+                    else:
+                        time.sleep(0.5)
+                        tdLog.info(f"file real exist {pathFile} , sleep 500ms and try")
+
+                if real is False:
+                    continue
+                fileStat = os.stat(pathFile)
+                tdLog.exit(f" check file can not be deleted. file={pathFile} file size={fileStat.st_size}")
+
+        return True
+
+    # check vnode tsdb folder empty
+    def check_filedelete(self):     
+        # put all vnode to list
+        for dnode in os.listdir(self.projDir):
+            vnodesDir = self.projDir + f"{dnode}/data/vnode/"
+            if os.path.isdir(vnodesDir) == False or dnode[:5] != "dnode":
+                continue
+            print(f"vnodesDir={vnodesDir}")
+            # enum all vnode
+            for vnode in os.listdir(vnodesDir):
+                vnodeDir = path.join(vnodesDir, vnode)
+                print(f"vnodeDir={vnodeDir}")
+                if os.path.isdir(vnodesDir):
+                    tsdbDir = path.join(vnodeDir, "tsdb")
+                    if path.exists(tsdbDir) :
+                        self.check_tsdb_dir(tsdbDir)
+                        
+    def waitTranslation(self, waitSeconds):
+        # wait end
+        for i in range(waitSeconds):
+            sql ="show transactions;"
+            rows = tdSql.query(sql)
+            if rows == 0:
+               return True
+            tdLog.info(f"i={i} wait for translation finish ...")
+            time.sleep(1)
+
+        return False
+
+    # run
+    def run(self):
+        # seed
+        random.seed(int(time.time()))
+        self.dbname = "deletecheck"
+        stbname = "meters"
+        childname= "d"
+        child_cnt   = 2
+        batch_size  = 8000
+        insert_rows = 100015
+        start_ts = 1600000000000
+
+        self.autoGen.create_db(self.dbname)
+
+        loop = 3
+        for i in range(loop):
+            self.autoGen.create_stable(stbname, 4, 10, 4, 8)
+            self.autoGen.create_child(stbname, childname, child_cnt)
+            self.autoGen.set_batch_size(batch_size)
+            self.autoGen.insert_data(insert_rows)
+            self.autoGen.set_start_ts(start_ts)
+            
+            if i % 2 == 1:
+              tdSql.execute(f"flush database {self.dbname}", show=True)
+
+            # drop stable
+            tdSql.execute(f"drop table {self.dbname}.{stbname} ", show = True)
+
+            self.compactDatbase()
+
+            # check file delete
+            self.check_filedelete()
+            tdLog.info(f"loop = {i+1} / {loop} check file delete ok after drop table successfully.")
+
+            start_ts += i*100000000
+
+        
+    # stop
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file
diff --git a/tests/system-test/0-others/view/non_marterial_view/test_view.py b/tests/system-test/0-others/view/non_marterial_view/test_view.py
new file mode 100644
index 0000000000..afb2476305
--- /dev/null
+++ b/tests/system-test/0-others/view/non_marterial_view/test_view.py
@@ -0,0 +1,591 @@
+
+import taos
+import os
+import sys
+import time
+from pathlib import Path
+sys.path.append(os.path.dirname(Path(__file__).resolve().parent.parent.parent) + "/7-tmq")
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+from util.sqlset import *
+from tmqCommon import *
+
+class TDTestCase:
+    """This test case is used to veirfy the tmq consume data from non marterial view
+    """
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+        self.setsql = TDSetSql()
+        
+        # db info
+        self.dbname = "view_db"
+        self.stbname = 'stb'
+        self.ctbname_list = ["ct1", "ct2"]
+        self.stable_column_dict = {
+            'ts': 'timestamp',
+            'col1': 'float',
+            'col2': 'int',
+        }
+        self.tag_dict = {
+            'ctbname': 'binary(10)'
+        }
+
+    def prepare_data(self, conn=None):
+        """Create the db and data for test
+        """
+        tdLog.debug("Start to prepare the data")
+        if not conn:
+            conn = tdSql
+        # create datebase
+        conn.execute(f"create database {self.dbname}")
+        conn.execute(f"use {self.dbname}")
+        time.sleep(2)
+
+        # create stable
+        conn.execute(self.setsql.set_create_stable_sql(self.stbname, self.stable_column_dict, self.tag_dict))
+        tdLog.debug("Create stable {} successfully".format(self.stbname))
+
+        # create child tables
+        for ctname in self.ctbname_list:
+            conn.execute(f"create table {ctname} using {self.stbname} tags('{ctname}');")
+            tdLog.debug("Create child table {} successfully".format(ctname))
+
+            # insert data into child tables
+            conn.execute(f"insert into {ctname} values(now, 1.1, 1)(now+1s, 2.2, 2)(now+2s, 3.3, 3)(now+3s, 4.4, 4)(now+4s, 5.5, 5)(now+5s, 6.6, 6)(now+6s, 7.7, 7)(now+7s, 8.8, 8)(now+8s, 9.9, 9)(now+9s, 10.1, 10);)")
+            tdLog.debug(f"Insert into data to {ctname} successfully")
+
+    def prepare_tmq_data(self, para_dic):
+        tdLog.debug("Start to prepare the tmq data")
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, para_dic["dbName"], para_dic["dropFlag"], vgroups=para_dic["vgroups"], replica=1)
+        tdLog.info("create stb")
+        tdCom.create_stable(tdSql, dbname=para_dic["dbName"], stbname=para_dic["stbName"], column_elm_list=para_dic['colSchema'], tag_elm_list=para_dic['tagSchema'])
+        tdLog.info("create ctb")
+        tdCom.create_ctable(tdSql, dbname=para_dic["dbName"], stbname=para_dic["stbName"],tag_elm_list=para_dic['tagSchema'], count=para_dic["ctbNum"], default_ctbname_prefix=para_dic['ctbPrefix'])
+        tdLog.info("insert data")
+        tmqCom.insert_data(tdSql, para_dic["dbName"], para_dic["ctbPrefix"], para_dic["ctbNum"], para_dic["rowsPerTbl"], para_dic["batchNum"], para_dic["startTs"])
+        tdLog.debug("Finish to prepare the tmq data")
+        
+    def check_view_num(self, num):
+        tdSql.query("show views;")
+        rows = tdSql.queryRows
+        assert(rows == num)
+        tdLog.debug(f"Verify the view number successfully")
+        
+    def create_user(self, username, password):
+        tdSql.execute(f"create user {username} pass '{password}';")
+        tdLog.debug("Create user {} with password {} successfully".format(username, password))
+        
+    def check_permissions(self, username, db_name, permission_dict, view_name=None):
+        """
+        :param permission_dict: {'db': ["read", "write], 'view': ["read", "write", "alter"]}
+        """
+        tdSql.query("select * from information_schema.ins_user_privileges;")
+        for item in permission_dict.keys():
+            if item == "db":
+                for permission in permission_dict[item]:
+                    assert((username, permission, db_name, "", "", "") in tdSql.queryResult)
+                    tdLog.debug(f"Verify the {item} {db_name} {permission} permission successfully")
+            elif item == "view":
+                for permission in permission_dict[item]:
+                    assert((username, permission, db_name, view_name, "", "view") in tdSql.queryResult)
+                    tdLog.debug(f"Verify the {item} {db_name} {view_name} {permission} permission successfully")
+            else:
+                raise Exception(f"Invalid permission type: {item}")
+
+    def test_create_view_from_one_database(self):
+        """This test case is used to verify the create view from one database
+        """
+        self.prepare_data()
+        tdSql.execute(f"create view v1 as select * from {self.stbname};")
+        self.check_view_num(1)
+        tdSql.error(f'create view v1 as select * from {self.stbname};', expectErrInfo='view already exists in db')
+        tdSql.error(f'create view db2.v2 as select * from {self.stbname};', expectErrInfo='Fail to get table info, error: Database not exist')
+        tdSql.error(f'create view v2 as select c2 from {self.stbname};', expectErrInfo='Invalid column name: c2')
+        tdSql.error(f'create view v2 as select ts, col1 from tt1;', expectErrInfo='Fail to get table info, error: Table does not exist')
+
+        tdSql.execute(f"drop database {self.dbname}")
+        tdLog.debug("Finish test case 'test_create_view_from_one_database'")
+
+    def test_create_view_from_multi_database(self):
+        """This test case is used to verify the create view from multi database
+        """
+        self.prepare_data()
+        tdSql.execute(f"create view v1 as select * from view_db.{self.stbname};")
+        self.check_view_num(1)
+
+        self.dbname = "view_db2"
+        self.prepare_data()
+        tdSql.execute(f"create view v1 as select * from view_db2.{self.stbname};")
+        tdSql.execute(f"create view v2 as select * from view_db.v1;")
+        self.check_view_num(2)
+
+        self.dbname = "view_db"
+        tdSql.execute(f"drop database view_db;")
+        tdSql.execute(f"drop database view_db2;")
+        tdLog.debug("Finish test case 'test_create_view_from_multi_database'")
+
+    def test_create_view_name_params(self):
+        """This test case is used to verify the create view with different view name params
+        """
+        self.prepare_data()
+        tdSql.execute(f"create view v1 as select * from {self.stbname};")
+        self.check_view_num(1)
+        tdSql.error(f"create view v/2 as select * from {self.stbname};", expectErrInfo='syntax error near "/2 as select * from stb;"')
+        tdSql.execute(f"create view v2 as select ts, col1 from {self.stbname};")
+        self.check_view_num(2)
+        view_name_192_characters = "rzuoxoIXilAGgzNjYActiQwgzZK7PZYpDuaOe1lSJMFMVYXaexh1OfMmk3LvJcQbTeXXW7uGJY8IHuweHF73VHgoZgf0waO33YpZiTKfDQbdWtN4YmR2eWjL84ZtkfjM4huCP6lCysbDMj8YNwWksTdUq70LIyNhHp2V8HhhxyYSkREYFLJ1kOE78v61MQT6"
+        tdSql.execute(f"create view {view_name_192_characters} as select * from {self.stbname};")
+        self.check_view_num(3)
+        tdSql.error(f"create view {view_name_192_characters}1 as select * from {self.stbname};", expectErrInfo='Invalid identifier name: rzuoxoixilaggznjyactiqwgzzk7pzypduaoe1lsjmfmvyxaexh1ofmmk3lvjcqbtexxw7ugjy8ihuwehf73vhgozgf0wao33ypzitkfdqbdwtn4ymr2ewjl84ztkfjm4hucp6lcysbdmj8ynwwkstduq70liynhhp2v8hhhxyyskreyflj1koe78v61mqt61 as select * from stb;')
+        tdSql.execute(f"drop database {self.dbname}")
+        tdLog.debug("Finish test case 'test_create_view_name_params'")
+
+    def test_create_view_query(self):
+        """This test case is used to verify the create view with different data type in query
+        """
+        self.prepare_data()
+        # add different data type table
+        tdSql.execute(f"create table tb (ts timestamp, c1 int, c2 int unsigned, c3 bigint, c4 bigint unsigned, c5 float, c6 double, c7 binary(16), c8 smallint, c9 smallint unsigned, c10 tinyint, c11 tinyint unsigned, c12 bool, c13 varchar(16), c14 nchar(8), c15 geometry(21), c16 varbinary(16));")
+        tdSql.execute(f"create view v1 as select ts, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16 from tb;")
+        # check data type in create view sql
+        tdSql.query("desc v1;")
+        res = tdSql.queryResult
+        data_type_list = [res[index][1] for index in range(len(res))]
+        tdLog.debug(data_type_list)
+        assert('TIMESTAMP' in data_type_list and 'INT' in data_type_list and 'INT UNSIGNED' in data_type_list and 'BIGINT' in data_type_list and 'BIGINT UNSIGNED' in data_type_list and 'FLOAT' in data_type_list and 'DOUBLE' in data_type_list and 'VARCHAR' in data_type_list and 'SMALLINT' in data_type_list and 'SMALLINT UNSIGNED' in data_type_list and 'TINYINT' in data_type_list and 'TINYINT UNSIGNED' in data_type_list and 'BOOL' in data_type_list and 'VARCHAR' in data_type_list and 'NCHAR' in data_type_list and 'GEOMETRY' in data_type_list and 'VARBINARY' in data_type_list)
+        tdSql.execute("create view v2 as select * from tb where c1 >5 and c7 like '%ab%';")
+        self.check_view_num(2)
+        tdSql.error("create view v3 as select * from tb where c1 like '%ab%';", expectErrInfo='Invalid value type')
+        tdSql.execute("create view v3 as select first(ts), sum(c1) from tb group by c2 having avg(c4) > 0;")
+        tdSql.execute("create view v4 as select _wstart,sum(c6) from tb interval(10s);")
+        tdSql.execute("create view v5 as select * from tb join v2 on tb.ts = v2.ts;")
+        tdSql.execute("create view v6 as select * from (select ts, c1, c2 from (select * from v2));")
+        self.check_view_num(6)
+        for v in ['v1', 'v2', 'v3', 'v4', 'v5', 'v6']:
+            tdSql.execute(f"drop view {v};")
+        tdSql.execute(f"drop database {self.dbname}")
+        tdLog.debug("Finish test case 'test_create_view_query'")
+
+    def test_show_view(self):
+        """This test case is used to verify the show view
+        """
+        self.prepare_data()
+        tdSql.execute(f"create view v1 as select * from {self.ctbname_list[0]};")
+
+        # query from show sql
+        tdSql.query("show views;")
+        res = tdSql.queryResult
+        assert(res[0][0] == 'v1' and res[0][1] == 'view_db' and res[0][2] == 'root' and res[0][4] == 'NORMAL' and res[0][5] == 'select * from ct1;')
+
+        # show create sql
+        tdSql.query("show create view v1;")
+        res = tdSql.queryResult
+        assert(res[0][1] == 'CREATE VIEW `view_db`.`v1` AS select * from ct1;')
+
+        # query from desc results
+        tdSql.query("desc view_db.v1;")
+        res = tdSql.queryResult
+        assert(res[0][1] == 'TIMESTAMP' and res[1][1] == 'FLOAT' and res[2][1] == 'INT')
+
+        # query from system table
+        tdSql.query("select * from information_schema.ins_views;")
+        res = tdSql.queryResult
+        assert(res[0][0] == 'v1' and res[0][1] == 'view_db' and res[0][2] == 'root' and res[0][4] == 'NORMAL' and res[0][5] == 'select * from ct1;')
+        tdSql.error("show db3.views;", expectErrInfo='Database not exist')
+        tdSql.error("desc viewx;", expectErrInfo='Table does not exist')
+        tdSql.error(f"show create view {self.dbname}.viewx;", expectErrInfo='view not exists in db')
+        tdSql.execute(f"drop database {self.dbname}")
+        tdSql.error("show views;", expectErrInfo='Database not exist')
+        tdLog.debug("Finish test case 'test_show_view'")
+
+    def test_drop_view(self):
+        """This test case is used to verify the drop view
+        """
+        self.prepare_data()
+        self.dbname = "view_db2"
+        self.prepare_data()
+        tdSql.execute("create view view_db.v1 as select * from view_db.stb;")
+        tdSql.execute("create view view_db2.v1 as select * from view_db2.stb;")
+        # delete view without database name
+        tdSql.execute("drop view v1;")
+        # delete view with database name
+        tdSql.execute("drop view view_db.v1;")
+        # delete non exist view
+        tdSql.error("drop view view_db.v11;", expectErrInfo='view not exists in db')
+        tdSql.execute("drop database view_db")
+        tdSql.execute("drop database view_db2;")
+        self.dbname = "view_db"
+        tdLog.debug("Finish test case 'test_drop_view'")
+
+    def test_view_permission_db_all_view_all(self):
+        """This test case is used to verify the view permission with db all and view all,
+        the time sleep to wait the permission take effect
+        """
+        self.prepare_data()
+        username = "view_test"
+        password = "test"
+        self.create_user(username, password)
+        # grant all db permission to user
+        tdSql.execute("grant all on view_db.* to view_test;")
+        
+        conn = taos.connect(user=username, password=password)
+        conn.execute(f"use {self.dbname};")
+        conn.execute("create view v1 as select * from stb;")
+        res = conn.query("show views;")
+        assert(len(res.fetch_all()) == 1)
+        tdLog.debug(f"Verify the show view permission of user '{username}' with db all and view all successfully")
+        self.check_permissions("view_test", "view_db", {"db": ["read", "write"], "view": ["read", "write", "alter"]}, "v1")
+        tdLog.debug(f"Verify the view permission from system table successfully")
+        time.sleep(2)
+        conn.execute("drop view v1;")
+        tdSql.execute("revoke all on view_db.* from view_test;")
+        tdSql.execute(f"drop database {self.dbname};")
+        time.sleep(1)
+
+        # prepare data by user 'view_test'
+        self.prepare_data(conn)
+
+        conn.execute("create view v1 as select * from stb;")
+        res = conn.query("show views;")
+        assert(len(res.fetch_all()) == 1)
+        tdLog.debug(f"Verify the view permission of user '{username}' with db all and view all successfully")
+        self.check_permissions("view_test", "view_db", {"db": ["read", "write"], "view": ["read", "write", "alter"]}, "v1")
+        tdLog.debug(f"Verify the view permission from system table successfully")
+        time.sleep(2)
+        conn.execute("drop view v1;")
+        tdSql.execute("revoke all on view_db.* from view_test;")
+        tdSql.execute("revoke all on view_db.v1 from view_test;")
+        tdSql.execute(f"drop database {self.dbname}")
+        tdSql.execute("drop user view_test;")
+        tdLog.debug("Finish test case 'test_view_permission_db_all_view_all'")
+
+    def test_view_permission_db_write_view_all(self):
+        """This test case is used to verify the view permission with db write and view all
+        """
+        username = "view_test"
+        password = "test"
+        self.create_user(username, password)
+        conn = taos.connect(user=username, password=password)
+        self.prepare_data(conn)
+        conn.execute("create view v1 as select * from stb;")
+        tdSql.execute("revoke read on view_db.* from view_test;")
+        self.check_permissions("view_test", "view_db", {"db": ["write"], "view": ["read", "write", "alter"]}, "v1")
+        # create view permission error
+        try:
+            conn.execute("create view v2 as select * from v1;")
+        except Exception as ex:
+            assert("[0x2644]: Permission denied or target object not exist" in str(ex))
+        # query from view permission error
+        try:
+            conn.query("select * from v1;")
+        except Exception as ex:
+            assert("[0x2644]: Permission denied or target object not exist" in str(ex))
+        # view query permission
+        res = conn.query("show views;")
+        assert(len(res.fetch_all()) == 1)
+        time.sleep(2)
+        conn.execute("drop view v1;")
+        tdSql.execute("revoke write on view_db.* from view_test;")
+        tdSql.execute(f"drop database {self.dbname}")
+        tdSql.execute("drop user view_test;")
+        tdLog.debug("Finish test case 'test_view_permission_db_write_view_all'")
+
+    def test_view_permission_db_write_view_read(self):
+        """This test case is used to verify the view permission with db write and view read
+        """
+        username = "view_test"
+        password = "test"
+        self.create_user(username, password)
+        conn = taos.connect(user=username, password=password)
+        self.prepare_data()
+        
+        tdSql.execute("create view v1 as select * from stb;")
+        tdSql.execute("grant write on view_db.* to view_test;")
+        tdSql.execute("grant read on view_db.v1 to view_test;")
+        
+        conn.execute(f"use {self.dbname};")
+        time.sleep(2)
+        res = conn.query("select * from v1;")
+        assert(len(res.fetch_all()) == 20)
+        
+        conn.execute("create view v2 as select * from v1;")
+        # create view from super table of database
+        try:
+            conn.execute("create view v3 as select * from stb;")
+        except Exception as ex:
+            assert("[0x2644]: Permission denied or target object not exist" in str(ex))
+        time.sleep(2)
+        conn.execute("drop view v2;")
+        try:
+            conn.execute("drop view v1;")
+        except Exception as ex:
+            assert("[0x2644]: Permission denied or target object not exist" in str(ex))
+        tdSql.execute("revoke read on view_db.v1 from view_test;")
+        tdSql.execute("revoke write on view_db.* from view_test;")
+        tdSql.execute(f"drop database {self.dbname}")
+        tdSql.execute("drop user view_test;")
+        tdLog.debug("Finish test case 'test_view_permission_db_write_view_read'")
+
+    def test_view_permission_db_write_view_alter(self):
+        """This test case is used to verify the view permission with db write and view alter
+        """
+        username = "view_test"
+        password = "test"
+        self.create_user(username, password)
+        conn = taos.connect(user=username, password=password)
+        self.prepare_data()
+        
+        tdSql.execute("create view v1 as select * from stb;")
+        tdSql.execute("grant write on view_db.* to view_test;")
+        tdSql.execute("grant alter on view_db.v1 to view_test;")
+        try:
+            conn.execute(f"use {self.dbname};")
+            conn.execute("select * from v1;")
+        except Exception as ex:
+            assert("[0x2644]: Permission denied or target object not exist" in str(ex))
+        time.sleep(2)
+        conn.execute("drop view v1;")
+        tdSql.execute("revoke write on view_db.* from view_test;")
+        tdSql.execute(f"drop database {self.dbname}")
+        tdSql.execute("drop user view_test;")
+        tdLog.debug("Finish test case 'test_view_permission_db_write_view_alter'")
+
+    def test_view_permission_db_read_view_all(self):
+        """This test case is used to verify the view permission with db read and view all
+        """
+        username = "view_test"
+        password = "test"
+        self.create_user(username, password)
+        conn = taos.connect(user=username, password=password)
+        self.prepare_data()
+        
+        tdSql.execute("create view v1 as select * from stb;")
+        tdSql.execute("grant read on view_db.* to view_test;")
+        tdSql.execute("grant all on view_db.v1 to view_test;")
+        try:
+            conn.execute(f"use {self.dbname};")
+            conn.execute("create view v2 as select * from v1;")
+        except Exception as ex:
+            assert("[0x2644]: Permission denied or target object not exist" in str(ex))
+        time.sleep(2)
+        res = conn.query("select * from v1;")
+        assert(len(res.fetch_all()) == 20)
+        conn.execute("drop view v1;")
+        tdSql.execute("revoke read on view_db.* from view_test;")
+        tdSql.execute(f"drop database {self.dbname}")
+        tdSql.execute("drop user view_test;")
+        tdLog.debug("Finish test case 'test_view_permission_db_read_view_all'")
+
+    def test_view_permission_db_read_view_alter(self):
+        """This test case is used to verify the view permission with db read and view alter
+        """
+        username = "view_test"
+        password = "test"
+        self.create_user(username, password)
+        conn = taos.connect(user=username, password=password)
+        self.prepare_data()
+        
+        tdSql.execute("create view v1 as select * from stb;")
+        tdSql.execute("grant read on view_db.* to view_test;")
+        tdSql.execute("grant alter on view_db.v1 to view_test;")
+        try:
+            conn.execute(f"use {self.dbname};")
+            conn.execute("select * from v1;")
+        except Exception as ex:
+            assert("[0x2644]: Permission denied or target object not exist" in str(ex))
+            
+        time.sleep(2)
+        conn.execute("drop view v1;")
+        tdSql.execute("revoke read on view_db.* from view_test;")
+        tdSql.execute(f"drop database {self.dbname}")
+        tdSql.execute("drop user view_test;")
+        tdLog.debug("Finish test case 'test_view_permission_db_read_view_alter'")
+
+    def test_view_permission_db_read_view_read(self):
+        """This test case is used to verify the view permission with db read and view read
+        """
+        username = "view_test"
+        password = "test"
+        self.create_user(username, password)
+        conn = taos.connect(user=username, password=password)
+        self.prepare_data()
+        
+        tdSql.execute("create view v1 as select * from stb;")
+        tdSql.execute("grant read on view_db.* to view_test;")
+        tdSql.execute("grant read on view_db.v1 to view_test;")
+        conn.execute(f"use {self.dbname};")
+        time.sleep(2)
+        res = conn.query("select * from v1;")
+        assert(len(res.fetch_all()) == 20)
+        try:
+            conn.execute("drop view v1;")
+        except Exception as ex:
+            assert("[0x2644]: Permission denied or target object not exist" in str(ex))
+        tdSql.execute("revoke read on view_db.* from view_test;")
+        tdSql.execute("revoke read on view_db.v1 from view_test;")
+        tdSql.execute(f"drop database {self.dbname}")
+        tdSql.execute("drop user view_test;")
+        tdLog.debug("Finish test case 'test_view_permission_db_read_view_read'")
+
+    def test_query_from_view(self):
+        """This test case is used to verify the query from view
+        """
+        self.prepare_data()
+        view_name_list = []
+
+        # common query from super table
+        tdSql.execute(f"create view v1 as select * from {self.stbname};")
+        tdSql.query(f"select * from v1;")
+        rows = tdSql.queryRows
+        assert(rows == 20)
+        view_name_list.append("v1")
+        tdLog.debug("Verify the query from super table successfully")
+
+        # common query from child table
+        tdSql.execute(f"create view v2 as select * from {self.ctbname_list[0]};")
+        tdSql.query(f"select * from v2;")
+        rows = tdSql.queryRows
+        assert(rows == 10)
+        view_name_list.append("v2")
+        tdLog.debug("Verify the query from child table successfully")
+
+        # join query
+        tdSql.execute(f"create view v3 as select * from {self.stbname} join {self.ctbname_list[1]} on {self.ctbname_list[1]}.ts = {self.stbname}.ts;")
+        tdSql.query(f"select * from v3;")
+        rows = tdSql.queryRows
+        assert(rows == 10)
+        view_name_list.append("v3")
+        tdLog.debug("Verify the join query successfully")
+
+        # group by query
+        tdSql.execute(f"create view v4 as select count(*) from {self.stbname} group by tbname;")
+        tdSql.query(f"select * from v4;")
+        rows = tdSql.queryRows
+        assert(rows == 2)
+        res = tdSql.queryResult
+        assert(res[0][0] == 10)
+        view_name_list.append("v4")
+        tdLog.debug("Verify the group by query successfully")
+
+        # partition by query
+        tdSql.execute(f"create view v5 as select sum(col1) from {self.stbname} where col2 > 4 partition by tbname interval(3s);")
+        tdSql.query(f"select * from v5;")
+        rows = tdSql.queryRows
+        assert(rows >= 4)
+        view_name_list.append("v5")
+        tdLog.debug("Verify the partition by query successfully")
+
+        # query from nested view
+        tdSql.execute(f"create view v6 as select * from v5;")
+        tdSql.query(f"select * from v6;")
+        rows = tdSql.queryRows
+        assert(rows >= 4)
+        view_name_list.append("v6")
+        tdLog.debug("Verify the query from nested view successfully")
+
+        # delete view
+        for view in view_name_list:
+            tdSql.execute(f"drop view {view};")
+            tdLog.debug(f"Drop view {view} successfully")
+        tdSql.execute(f"drop database {self.dbname}")
+        tdLog.debug("Finish test case 'test_query_from_view'")
+
+    def test_tmq_from_view(self):
+        """This test case is used to verify the tmq consume data from view
+        """
+        # params for db
+        paraDict = {'dbName':     'view_db',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbNum':     1,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1}
+        # topic info
+        topic_name_list = ['topic1']
+        view_name_list = ['view1']
+        expectRowsList = []
+        
+        self.prepare_tmq_data(paraDict)
+        
+        # init consume info, and start tmq_sim, then check consume result
+        tmqCom.initConsumerTable()
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        tdSql.execute(f"create view {view_name_list[0]} as {queryString}")
+        sqlString = "create topic %s as %s" %(topic_name_list[0], "select * from %s"%view_name_list[0])
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query(queryString)
+        expectRowsList.append(tdSql.getRows())
+
+        consumerId   = 1
+        topicList    = topic_name_list[0]
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+        keyList      = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt, topicList, keyList, ifcheckdata, ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(paraDict['pollDelay'], paraDict["dbName"], paraDict['showMsg'], paraDict['showRow'])
+
+        tdLog.info("wait the consume result")
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        if expectRowsList[0] != resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
+            tdLog.exit("1 tmq consume rows error!")
+
+        tmqCom.checkFileContent(consumerId, queryString)
+
+        time.sleep(10)
+        for i in range(len(topic_name_list)):
+            tdSql.query("drop topic %s"%topic_name_list[i])
+        for i in range(len(view_name_list)):
+            tdSql.query("drop view %s"%view_name_list[i])
+
+        # drop database
+        tdSql.execute(f"drop database {paraDict['dbName']}")
+        tdSql.execute("drop database cdb;")
+        tdLog.debug("Finish test case 'test_tmq_from_view'")
+
+    def run(self):
+        self.test_create_view_from_one_database()
+        self.test_create_view_from_multi_database()
+        self.test_create_view_name_params()
+        self.test_create_view_query()
+        self.test_show_view()
+        self.test_drop_view()
+        self.test_view_permission_db_all_view_all()
+        self.test_view_permission_db_write_view_all()
+        self.test_view_permission_db_write_view_read()
+        self.test_view_permission_db_write_view_alter()
+        self.test_view_permission_db_read_view_all()
+        self.test_view_permission_db_read_view_alter()
+        self.test_view_permission_db_read_view_read()
+        self.test_query_from_view()
+        self.test_tmq_from_view()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/1-insert/test_ts4219.py b/tests/system-test/1-insert/test_ts4219.py
new file mode 100644
index 0000000000..e6447d77ae
--- /dev/null
+++ b/tests/system-test/1-insert/test_ts4219.py
@@ -0,0 +1,27 @@
+import sys 
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+from math import inf
+
+
+class TDTestCase:
+    def init(self, conn, logSql, replicaVer=1):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), True)
+
+    def prepare_data(self):
+        tdSql.execute("create database db;")
+        tdSql.execute("use db;")
+        tdSql.execute("create stable st(ts timestamp, c1 int, c2 float) tags(groupname binary(32));")
+
+    def run(self):
+        tdSql.error("insert into ct1 using st tags('group name 1') values(now, 1, 1.1)(now+1s, 2, 2.2)  ct1 using st tags('group 1) values(now+2s, 3, 3.3); ")
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py
new file mode 100644
index 0000000000..0f0936ebab
--- /dev/null
+++ b/tests/system-test/2-query/last_cache_scan.py
@@ -0,0 +1,334 @@
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+from datetime import datetime
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+# from tmqCommon import *
+
+COMPARE_DATA = 0
+COMPARE_LEN = 1
+
+class TDTestCase:
+    def __init__(self):
+        self.vgroups    = 4
+        self.ctbNum     = 10
+        self.rowsPerTbl = 10000
+        self.duraion = '1h'
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'):
+        if dropFlag == 1:
+            tsql.execute("drop database if exists %s"%(dbName))
+
+        tsql.execute("create database if not exists %s vgroups %d replica %d duration %s  CACHEMODEL 'both'"%(dbName, vgroups, replica, duration))
+        tdLog.debug("complete to create database %s"%(dbName))
+        return
+
+    def create_stable(self,tsql, paraDict):
+        colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"])
+        tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"])
+        sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString)
+        tdLog.debug("%s"%(sqlString))
+        tsql.execute(sqlString)
+        return
+
+    def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0):
+        for i in range(ctbNum):
+            sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \
+                    (dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx)
+            tsql.execute(sqlString)
+
+        tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName))
+        return
+
+    def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep):
+        tdLog.debug("start to insert data ............")
+        tsql.execute("use %s" %dbName)
+        pre_insert = "insert into "
+        sql = pre_insert
+
+        for i in range(ctbNum):
+            rowsBatched = 0
+            sql += " %s%d values "%(ctbPrefix,i)
+            for j in range(rowsPerTbl):
+                if (i < ctbNum/2):
+                    sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d', %d) "%(startTs + j*tsStep, j%1000, j%500, j%1000, j%5000, j%5400, j%128, j%10000, j%1000, startTs+j*tsStep+1000)
+                else:
+                    sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d', %d) "%(startTs + j*tsStep, j%1000, j%500, j%1000, j%128, j%10000, j%1000, startTs + j*tsStep + 1000)
+                rowsBatched += 1
+                if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
+                    tsql.execute(sql)
+                    rowsBatched = 0
+                    if j < rowsPerTbl - 1:
+                        sql = "insert into %s%d values " %(ctbPrefix,i)
+                    else:
+                        sql = "insert into "
+        if sql != pre_insert:
+            tsql.execute(sql)
+        tdLog.debug("insert data ............ [OK]")
+        return
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'test',
+                    'dropFlag':   1,
+                    'vgroups':    2,
+                    'stbName':    'meters',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},
+                                    {'type': 'BIGINT', 'count':1},
+                                    {'type': 'FLOAT', 'count':1},
+                                    {'type': 'DOUBLE', 'count':1},
+                                    {'type': 'smallint', 'count':1},
+                                    {'type': 'tinyint', 'count':1},
+                                    {'type': 'bool', 'count':1},
+                                    {'type': 'binary', 'len':10, 'count':1},
+                                    {'type': 'nchar', 'len':10, 'count':1},
+                                    {'type': 'timestamp', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}],
+                    'ctbPrefix':  't',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     100,
+                    'rowsPerTbl': 10000,
+                    'batchNum':   3000,
+                    'startTs':    1537146000000,
+                    'tsStep':     600000}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        tdLog.info("create database")
+        self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion)
+
+        tdLog.info("create stb")
+        self.create_stable(tsql=tdSql, paraDict=paraDict)
+
+        tdLog.info("create child tables")
+        self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \
+                stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\
+                ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"])
+        self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\
+                ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\
+                rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\
+                startTs=paraDict["startTs"],tsStep=paraDict["tsStep"])
+        return
+
+    def check_explain_res_has_row(self, plan_str_expect: str, rows, sql):
+        plan_found = False
+        for row in rows:
+            if str(row).find(plan_str_expect) >= 0:
+                tdLog.debug("plan: [%s] found in: [%s]" % (plan_str_expect, str(row)))
+                plan_found = True
+                break
+        if not plan_found:
+            tdLog.exit("plan: %s not found in res: [%s] in sql: %s" % (plan_str_expect, str(rows), sql))
+
+    def check_explain_res_no_row(self, plan_str_not_expect: str, res, sql):
+        for row in res:
+            if str(row).find(plan_str_not_expect) >= 0:
+                tdLog.exit('plan: [%s] found in: [%s] for sql: %s' % (plan_str_not_expect, str(row), sql))
+
+    def explain_sql(self, sql: str):
+        sql = "explain verbose true " + sql
+        tdSql.query(sql, queryTimes=1)
+        return tdSql.queryResult
+
+    def explain_and_check_res(self, sqls, hasLastRowScanRes):
+        for sql, has_last in zip(sqls, hasLastRowScanRes):
+            res = self.explain_sql(sql)
+            if has_last == 1:
+                self.check_explain_res_has_row("Last Row Scan", res, sql)
+            else:
+                self.check_explain_res_no_row("Last Row Scan", res, sql)
+
+    def format_sqls(self, sql_template, select_items):
+        sqls = []
+        for item in select_items:
+            sqls.append(sql_template % item)
+        return sqls
+
+    def query_check_one(self, sql, res_expect):
+        if res_expect is not None:
+            tdSql.query(sql, queryTimes=1)
+            tdSql.checkRows(1)
+            for i in range(0, tdSql.queryCols):
+                tdSql.checkData(0, i, res_expect[i])
+                tdLog.info('%s check res col: %d succeed value: %s' % (sql, i, str(res_expect[i])))
+
+    def query_check_sqls(self, sqls, has_last_row_scan_res, res_expect):
+        for sql, has_last, res in zip(sqls, has_last_row_scan_res, res_expect):
+            if has_last == 1:
+                self.query_check_one(sql, res)
+
+    def test_last_cache_scan(self):
+        sql_template = 'select %s from meters'
+        select_items = [
+                "last(ts), ts", "last(ts), c1", "last(ts), c2", "last(ts), c3",\
+                "last(ts), c4", "last(ts), tbname", "last(ts), t1", "last(ts), ts, ts"]
+        has_last_row_scan_res = [1, 0, 0, 0, 0, 0, 0, 1]
+        res_expect = [
+                ["2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000"],
+                None, None, None, None, None, None,
+                ["2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000"]
+                ]
+        sqls = self.format_sqls(sql_template, select_items)
+        self.explain_and_check_res(sqls, has_last_row_scan_res)
+        self.query_check_sqls(sqls, has_last_row_scan_res, res_expect)
+
+        select_items = ["last(c1),ts", "last(c1), c1", "last(c1), c2", "last(c1), c3",\
+                "last(c1), c4", "last(c1), tbname", "last(c1), t1", "last(c1), ts, ts", "last(c1), c1, c1"]
+        has_last_row_scan_res = [1, 1, 0, 0, 0, 0, 0, 1, 1]
+        res_expect = [
+                [999, "2018-11-25 19:30:00.000"],
+                [999, 999], None, None, None, None, None,
+                [999, "2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000"],
+                [999,999,999]
+                ]
+        sqls = self.format_sqls(sql_template, select_items)
+        self.explain_and_check_res(sqls, has_last_row_scan_res)
+        self.query_check_sqls(sqls, has_last_row_scan_res, res_expect)
+
+        sql_template = 'select %s from t1'
+        select_items = ["last(c4),ts", "last(c4), c1", "last(c4), c2", "last(c4), c3",\
+                "last(c4), c4", "last(c4), tbname", "last(c4), t1"]
+        has_last_row_scan_res = [1, 0, 0, 0, 1, 0, 0]
+        res_expect = [
+                [4999.000000000000000, "2018-11-25 19:30:00.000"],
+                None,None,None,
+                [4999.000000000000000, 4999.000000000000000]
+                ]
+        sqls = self.format_sqls(sql_template, select_items)
+        self.explain_and_check_res(sqls, has_last_row_scan_res)
+        self.query_check_sqls(sqls, has_last_row_scan_res, res_expect)
+
+        sql_template = 'select %s from meters'
+        select_items = ["last(c8), ts", "last(c8), c1", "last(c8), c8", "last(c8), tbname", \
+                "last(c8), t1", "last(c8), c8, c8", "last(c8), ts, ts"]
+        has_last_row_scan_res = [1, 0, 1, 0, 0, 1, 1]
+        res_expect = [
+                ["binary9999", "2018-11-25 19:30:00.000"],
+                None,
+                ["binary9999", "binary9999"],
+                None, None,
+                ["binary9999", "binary9999", "binary9999"],
+                ["binary9999", "2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000"]
+                ]
+        sqls = self.format_sqls(sql_template, select_items)
+        self.explain_and_check_res(sqls, has_last_row_scan_res)
+        self.query_check_sqls(sqls, has_last_row_scan_res, res_expect)
+
+        # c2, c4 in last row of t5,t6,t7,t8,t9 will always be NULL
+        sql_template = 'select %s from t5'
+        select_items = ["last(c4), ts", "last(c4), c4", "last(c4), c4, c4", "last(c4), ts, ts"]
+        has_last_row_scan_res = [1,1,1,1]
+
+        sqls = self.format_sqls(sql_template, select_items)
+        self.explain_and_check_res(sqls, has_last_row_scan_res)
+        #for sql in sqls:
+            #tdSql.query(sql, queryTimes=1)
+            #tdSql.checkRows(0)
+
+        sql_template = 'select %s from meters'
+        select_items = [
+                "last_row(ts), last(ts)",
+                "last_row(c1), last(c1)",
+                "last_row(c1), c1,c3, ts"
+                ]
+        has_last_row_scan_res = [0,0,1]
+        sqls = self.format_sqls(sql_template, select_items)
+        self.explain_and_check_res(sqls, has_last_row_scan_res)
+        #res_expect = [None, None, [999, 999, 499, "2018-11-25 19:30:00.000"]]
+        #self.query_check_sqls(sqls, has_last_row_scan_res, res_expect)
+
+        select_items = ["last(c10), c10",
+                        "last(c10), ts",
+                        "last(c10), c10, ts",
+                        "last(c10), c10, ts, c10,ts",
+                        "last(c10), ts, c1"]
+        has_last_row_scan_res = [1,1,1,1,0]
+        sqls = self.format_sqls(sql_template, select_items)
+        self.explain_and_check_res(sqls, has_last_row_scan_res)
+        res_expect = [
+                ["2018-11-25 19:30:01.000", "2018-11-25 19:30:01.000"],
+                ["2018-11-25 19:30:01.000", "2018-11-25 19:30:00.000"],
+                ["2018-11-25 19:30:01.000", "2018-11-25 19:30:01.000", "2018-11-25 19:30:00.000"],
+                ["2018-11-25 19:30:01.000", "2018-11-25 19:30:01.000", "2018-11-25 19:30:00.000", "2018-11-25 19:30:01.000", "2018-11-25 19:30:00.000"]
+                ]
+        self.query_check_sqls(sqls, has_last_row_scan_res, res_expect)
+
+        sql = "select last(c1), c1, c1+1, c1+2, ts from meters"
+        res = self.explain_sql(sql)
+        self.check_explain_res_has_row("Last Row Scan", res, sql)
+
+        tdSql.query(sql)
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 999)
+        tdSql.checkData(0, 1, 999)
+        tdSql.checkData(0, 2, 1000)
+        tdSql.checkData(0, 3, 1001)
+        tdSql.checkData(0, 4, "2018-11-25 19:30:00.000")
+
+        sql_template = 'select %s from meters partition by tbname'
+        select_items = ["ts, last(c10), c10, ts", "ts, ts, last(c10), c10, tbname", "last(c10), c10, ts"]
+        has_last_row_scan_res = [1,1,1]
+        sqls = self.format_sqls(sql_template, select_items)
+        self.explain_and_check_res(sqls, has_last_row_scan_res)
+        tdSql.query(sqls[0], queryTimes=1)
+        tdSql.checkRows(10)
+        tdSql.checkData(0,0, '2018-11-25 19:30:00.000')
+        tdSql.checkData(0,1, '2018-11-25 19:30:01.000')
+        tdSql.checkData(0,2, '2018-11-25 19:30:01.000')
+        tdSql.checkData(0,3, '2018-11-25 19:30:00.000')
+
+        tdSql.query(sqls[1], queryTimes=1)
+        tdSql.checkRows(10)
+        tdSql.checkData(0,0, '2018-11-25 19:30:00.000')
+        tdSql.checkData(0,1, '2018-11-25 19:30:00.000')
+        tdSql.checkData(0,2, '2018-11-25 19:30:01.000')
+        tdSql.checkData(0,3, '2018-11-25 19:30:01.000')
+
+        sql_template = 'select %s from meters partition by t1'
+        select_items = ["ts, last(c10), c10, ts", "ts, ts, last(c10), c10, t1", "last(c10), c10, ts"]
+        has_last_row_scan_res = [1,1,1]
+        sqls = self.format_sqls(sql_template, select_items)
+        self.explain_and_check_res(sqls, has_last_row_scan_res)
+        tdSql.query(sqls[0], queryTimes=1)
+        tdSql.checkRows(5)
+        tdSql.checkData(0,0, '2018-11-25 19:30:00.000')
+        tdSql.checkData(0,1, '2018-11-25 19:30:01.000')
+        tdSql.checkData(0,2, '2018-11-25 19:30:01.000')
+        tdSql.checkData(0,3, '2018-11-25 19:30:00.000')
+
+        tdSql.query("select ts, last(c10), t1, t2 from meters partition by t1, t2")
+        tdSql.checkRows(10)
+        tdSql.checkData(0, 0, '2018-11-25 19:30:00.000')
+        tdSql.checkData(0, 1, '2018-11-25 19:30:01.000')
+
+    def run(self):
+        self.prepareTestEnv()
+        #time.sleep(99999999)
+        self.test_last_cache_scan()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())