other: merge 3.0

.gitignore

@@ -16,7 +16,6 @@ debug/
 release/
 target/
 debs/
-deps/
 rpms/
 mac/
 *.pyc

cmake/cmake.define

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.0)
 
-set(CMAKE_VERBOSE_MAKEFILE OFF)
+set(CMAKE_VERBOSE_MAKEFILE ON)
 set(TD_BUILD_TAOSA_INTERNAL FALSE)
 
 #set output directory

cmake/cmake.platform

@@ -172,5 +172,15 @@ ENDIF()
 
 MESSAGE(STATUS "Platform arch:" ${PLATFORM_ARCH_STR})
 
+set(TD_DEPS_DIR "x86")
+if (TD_LINUX)
+  IF (TD_ARM_64 OR TD_ARM_32)
+    set(TD_DEPS_DIR "arm")
+  ELSE() 
+    set(TD_DEPS_DIR "x86")
+  ENDIF()
+endif()
+MESSAGE(STATUS "DEPS_DIR: " ${TD_DEPS_DIR})
+
 MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})")
 MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})")

cmake/rocksdb_CMakeLists.txt.in

@@ -1,5 +1,6 @@
 
 # rocksdb
+IF (NOT ${TD_LINUX})
 ExternalProject_Add(rocksdb
         GIT_REPOSITORY https://github.com/facebook/rocksdb.git
         GIT_TAG v8.1.1
@@ -9,3 +10,5 @@ ExternalProject_Add(rocksdb
         INSTALL_COMMAND ""
         TEST_COMMAND ""
         )
+ENDIF(NOT ${TD_LINUX})
+

contrib/CMakeLists.txt

@@ -78,10 +78,18 @@ if(${BUILD_WITH_LEVELDB})
 endif(${BUILD_WITH_LEVELDB})
 
 # rocksdb
+IF (NOT ${TD_LINUX})
 if(${BUILD_WITH_ROCKSDB})
     cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     add_definitions(-DUSE_ROCKSDB)
 endif(${BUILD_WITH_ROCKSDB})
+ELSE()
+if(${BUILD_WITH_ROCKSDB})
+    #cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    add_definitions(-DUSE_ROCKSDB)
+endif(${BUILD_WITH_ROCKSDB})
+    
+ENDIF(NOT ${TD_LINUX})
 
 # canonical-raft
 if(${BUILD_WITH_CRAFT})
@@ -227,6 +235,8 @@ endif(${BUILD_WITH_LEVELDB})
 
 # rocksdb
 # To support rocksdb build on ubuntu: sudo apt-get install libgflags-dev
+IF (NOT ${TD_LINUX})
+
 if(${BUILD_WITH_ROCKSDB})
     if(${TD_LINUX})
         SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL} -Wno-error=maybe-uninitialized -Wno-error=unused-but-set-variable -Wno-error=unused-variable -Wno-error=unused-function -Wno-errno=unused-private-field -Wno-error=unused-result")
@@ -288,6 +298,7 @@ if(${BUILD_WITH_ROCKSDB})
     )
 endif(${BUILD_WITH_ROCKSDB})
 
+ENDIF(NOT ${TD_LINUX})
 # lucene
 # To support build on ubuntu: sudo apt-get install libboost-all-dev
 if(${BUILD_WITH_LUCENE})
@@ -497,6 +508,7 @@ if(${BUILD_GEOS})
     endif(${TD_LINUX})
     option(BUILD_SHARED_LIBS "Build GEOS with shared libraries" OFF)
     add_subdirectory(geos EXCLUDE_FROM_ALL)
+    unset(CMAKE_CXX_STANDARD CACHE) # undo libgeos's setting of global CMAKE_CXX_STANDARD
     target_include_directories(
         geos_c
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/geos/include>

docs/en/14-reference/03-connector/07-python.mdx

@@ -667,6 +667,137 @@ Insert with req_id argument
 </TabItem>
 </Tabs>
 
+### Parameter Binding
+
+The Python connector provides a parameter binding api for inserting data. Similar to most databases, TDengine currently only supports the question mark `?` to indicate the parameters to be bound.
+
+<Tabs>
+<TabItem value="native" label="native connection">
+
+#### Create Stmt
+
+Call the `statement` method in `Connection` to create the `stmt` for parameter binding.
+
+```
+import taos
+
+conn = taos.connect()
+stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
+```
+
+#### parameter binding
+
+Call the `new_multi_binds` function to create the parameter list for parameter bindings.
+
+```
+params = new_multi_binds(16)
+params[0].timestamp((1626861392589, 1626861392590, 1626861392591))
+params[1].bool((True, None, False))
+params[2].tinyint([-128, -128, None])  # -128 is tinyint null
+params[3].tinyint([0, 127, None])
+params[4].smallint([3, None, 2])
+params[5].int([3, 4, None])
+params[6].bigint([3, 4, None])
+params[7].tinyint_unsigned([3, 4, None])
+params[8].smallint_unsigned([3, 4, None])
+params[9].int_unsigned([3, 4, None])
+params[10].bigint_unsigned([3, 4, None])
+params[11].float([3, None, 1])
+params[12].double([3, None, 1.2])
+params[13].binary(["abc", "dddafadfadfadfadfa", None])
+params[14].nchar(["涛思数据", None, "a long string with 中文字符"])
+params[15].timestamp([None, None, 1626861392591])
+```
+
+Call the `bind_param` (for a single row) method or the `bind_param_batch` (for multiple rows) method to set the values.
+
+```
+stmt.bind_param_batch(params)
+```
+
+#### execute sql
+
+Call `execute` method to execute sql.
+
+```
+stmt.execute()
+```
+
+#### Close Stmt
+
+```
+stmt.close()
+```
+
+#### Example
+
+```python
+{{#include docs/examples/python/stmt_example.py}}
+```
+</TabItem>
+
+<TabItem value="websocket" label="WebSocket connection">
+
+#### Create Stmt
+
+Call the `statement` method in `Connection` to create the `stmt` for parameter binding.
+
+```
+import taosws
+
+conn = taosws.connect('taosws://localhost:6041/test')
+stmt = conn.statement()
+```
+
+#### Prepare sql
+
+Call `prepare` method in stmt to prepare sql.
+
+```
+stmt.prepare("insert into t1 values (?, ?, ?, ?)")
+```
+
+#### parameter binding
+
+Call the `bind_param` method to bind parameters.
+
+```
+stmt.bind_param([
+    taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+    taosws.ints_to_column([1, 2, 3, 4]),
+    taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+    taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+])
+```
+
+Call the `add_batch` method to add parameters to the batch.
+
+```
+stmt.add_batch()
+```
+
+#### execute sql
+
+Call `execute` method to execute sql.
+
+```
+stmt.execute()
+```
+
+#### Close Stmt
+
+```
+stmt.close()
+```
+
+#### Example
+
+```python
+{{#include docs/examples/python/stmt_websocket_example.py}}
+```
+</TabItem>
+</Tabs>
+
 ### Other sample programs
 
 | Example program links | Example program content |

docs/examples/python/stmt_example.py

@@ -0,0 +1,82 @@
+#!
+
+import taosws
+
+import taos
+
+db_name = 'test_ws_stmt'
+
+
+def before():
+    taos_conn = taos.connect()
+    taos_conn.execute("drop database if exists %s" % db_name)
+    taos_conn.execute("create database %s" % db_name)
+    taos_conn.select_db(db_name)
+    taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))")
+    taos_conn.execute(
+        "create table stb1 (ts timestamp, a int, b float, c varchar(10)) tags (t1 int, t2 binary(10))")
+    taos_conn.close()
+
+
+def stmt_insert():
+    before()
+
+    conn = taosws.connect('taosws://root:taosdata@localhost:6041/%s' % db_name)
+
+    while True:
+        try:
+            stmt = conn.statement()
+            stmt.prepare("insert into t1 values (?, ?, ?, ?)")
+
+            stmt.bind_param([
+                taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+                taosws.ints_to_column([1, 2, 3, 4]),
+                taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+                taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+            ])
+
+            stmt.add_batch()
+            rows = stmt.execute()
+            print(rows)
+            stmt.close()
+        except Exception as e:
+            if 'Retry needed' in e.args[0]:  # deal with [0x0125] Retry needed
+                continue
+            else:
+                raise e
+
+        break
+
+
+def stmt_insert_into_stable():
+    before()
+
+    conn = taosws.connect("taosws://root:taosdata@localhost:6041/%s" % db_name)
+
+    while True:
+        try:
+            stmt = conn.statement()
+            stmt.prepare("insert into ? using stb1 tags (?, ?) values (?, ?, ?, ?)")
+            stmt.set_tbname('stb1_1')
+            stmt.set_tags([
+                taosws.int_to_tag(1),
+                taosws.varchar_to_tag('aaa'),
+            ])
+            stmt.bind_param([
+                taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+                taosws.ints_to_column([1, 2, 3, 4]),
+                taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+                taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+            ])
+
+            stmt.add_batch()
+            rows = stmt.execute()
+            print(rows)
+            stmt.close()
+        except Exception as e:
+            if 'Retry needed' in e.args[0]:  # deal with [0x0125] Retry needed
+                continue
+            else:
+                raise e
+
+        break

docs/examples/python/stmt_websocket_example.py

@@ -0,0 +1,78 @@
+#!
+import time
+
+import taosws
+
+import taos
+
+
+def before_test(db_name):
+    taos_conn = taos.connect()
+    taos_conn.execute("drop database if exists %s" % db_name)
+    taos_conn.execute("create database %s" % db_name)
+    taos_conn.select_db(db_name)
+    taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))")
+    taos_conn.execute(
+        "create table stb1 (ts timestamp, a int, b float, c varchar(10)) tags (t1 int, t2 binary(10))")
+    taos_conn.close()
+
+
+def after_test(db_name):
+    taos_conn = taos.connect()
+    taos_conn.execute("drop database if exists %s" % db_name)
+    taos_conn.close()
+
+
+def stmt_insert():
+    db_name = 'test_ws_stmt_{}'.format(int(time.time()))
+    before_test(db_name)
+
+    conn = taosws.connect('taosws://root:taosdata@localhost:6041/%s' % db_name)
+
+    stmt = conn.statement()
+    stmt.prepare("insert into t1 values (?, ?, ?, ?)")
+
+    stmt.bind_param([
+        taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+        taosws.ints_to_column([1, 2, 3, 4]),
+        taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+        taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+    ])
+
+    stmt.add_batch()
+    rows = stmt.execute()
+    assert rows == 4
+    stmt.close()
+    after_test(db_name)
+
+
+def stmt_insert_into_stable():
+    db_name = 'test_ws_stmt_{}'.format(int(time.time()))
+    before_test(db_name)
+
+    conn = taosws.connect("taosws://root:taosdata@localhost:6041/%s" % db_name)
+
+    stmt = conn.statement()
+    stmt.prepare("insert into ? using stb1 tags (?, ?) values (?, ?, ?, ?)")
+    stmt.set_tbname('stb1_1')
+    stmt.set_tags([
+        taosws.int_to_tag(1),
+        taosws.varchar_to_tag('aaa'),
+    ])
+    stmt.bind_param([
+        taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+        taosws.ints_to_column([1, 2, 3, 4]),
+        taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+        taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+    ])
+
+    stmt.add_batch()
+    rows = stmt.execute()
+    assert rows == 4
+    stmt.close()
+    after_test(db_name)
+
+
+if __name__ == '__main__':
+    stmt_insert()
+    stmt_insert_into_stable()

docs/zh/08-connector/30-python.mdx

@@ -672,6 +672,141 @@ consumer.close()
 </TabItem>
 </Tabs>
 
+### 通过参数绑定写入数据
+
+TDengine 的 Python 连接器支持参数绑定风格的 Prepare API 方式写入数据，和大多数数据库类似，目前仅支持用 `?` 来代表待绑定的参数。
+
+<Tabs>
+<TabItem value="native" label="原生连接">
+
+#### 创建 stmt
+
+Python 连接器的 `Connection` 提供了 `statement` 方法用于创建参数绑定对象 stmt，该方法接收 sql 字符串作为参数，sql 字符串目前仅支持用 `?` 来代表绑定的参数。
+
+```
+import taos
+
+conn = taos.connect()
+stmt = conn.statement("insert into log values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
+```
+
+#### 参数绑定
+
+调用 `new_multi_binds` 函数创建 params 列表，用于参数绑定。
+
+```
+params = new_multi_binds(16)
+params[0].timestamp((1626861392589, 1626861392590, 1626861392591))
+params[1].bool((True, None, False))
+params[2].tinyint([-128, -128, None])  # -128 is tinyint null
+params[3].tinyint([0, 127, None])
+params[4].smallint([3, None, 2])
+params[5].int([3, 4, None])
+params[6].bigint([3, 4, None])
+params[7].tinyint_unsigned([3, 4, None])
+params[8].smallint_unsigned([3, 4, None])
+params[9].int_unsigned([3, 4, None])
+params[10].bigint_unsigned([3, 4, None])
+params[11].float([3, None, 1])
+params[12].double([3, None, 1.2])
+params[13].binary(["abc", "dddafadfadfadfadfa", None])
+params[14].nchar(["涛思数据", None, "a long string with 中文字符"])
+params[15].timestamp([None, None, 1626861392591])
+```
+
+调用 stmt 的 `bind_param` 以单行的方式设置 values 或 `bind_param_batch` 以多行的方式设置 values 方法绑定参数。
+
+```
+stmt.bind_param_batch(params)
+```
+
+#### 执行 sql
+
+调用 stmt 的 `execute` 方法执行 sql
+
+```
+stmt.execute()
+```
+
+#### 关闭 stmt
+
+最后需要关闭 stmt。
+
+```
+stmt.close()
+```
+
+#### 示例代码
+
+```python
+{{#include docs/examples/python/stmt_example.py}}
+```
+</TabItem>
+
+<TabItem value="websocket" label="WebSocket 连接">
+
+#### 创建 stmt
+
+Python WebSocket 连接器的 `Connection` 提供了 `statement` 方法用于创建参数绑定对象 stmt，该方法接收 sql 字符串作为参数，sql 字符串目前仅支持用 `?` 来代表绑定的参数。
+
+```
+import taosws
+
+conn = taosws.connect('taosws://localhost:6041/test')
+stmt = conn.statement()
+```
+
+#### 解析 sql
+
+调用 stmt 的 `prepare` 方法来解析 insert 语句。
+
+```
+stmt.prepare("insert into t1 values (?, ?, ?, ?)")
+```
+
+#### 参数绑定
+
+调用 stmt 的 `bind_param` 方法绑定参数。
+
+```
+stmt.bind_param([
+    taosws.millis_timestamps_to_column([1686844800000, 1686844801000, 1686844802000, 1686844803000]),
+    taosws.ints_to_column([1, 2, 3, 4]),
+    taosws.floats_to_column([1.1, 2.2, 3.3, 4.4]),
+    taosws.varchar_to_column(['a', 'b', 'c', 'd']),
+])
+```
+
+调用 stmt 的 `add_batch` 方法，将参数加入批处理。
+
+```
+stmt.add_batch()
+```
+
+#### 执行 sql
+
+调用 stmt 的 `execute` 方法执行 sql
+
+```
+stmt.execute()
+```
+
+#### 关闭 stmt
+
+最后需要关闭 stmt。
+
+```
+stmt.close()
+```
+
+#### 示例代码
+
+```python
+{{#include docs/examples/python/stmt_websocket_example.py}}
+```
+</TabItem>
+</Tabs>
+
 ### 其它示例程序
 
 | 示例程序链接                                                                                                  | 示例程序内容            |

include/common/tglobal.h

@@ -120,6 +120,7 @@ extern bool    tsQueryUseNodeAllocator;
 extern bool    tsKeepColumnName;
 extern bool    tsEnableQueryHb;
 extern bool    tsEnableScience;
+extern bool    tsTtlChangeOnWrite;
 extern int32_t tsRedirectPeriod;
 extern int32_t tsRedirectFactor;
 extern int32_t tsRedirectMaxPeriod;

include/common/tmsg.h

@@ -945,7 +945,7 @@ int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
 int32_t tDeserializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
 
 typedef struct {
-  int32_t timestamp;
+  int32_t timestampSec;
 } SVDropTtlTableReq;
 
 int32_t tSerializeSVDropTtlTableReq(void* buf, int32_t bufLen, SVDropTtlTableReq* pReq);
@@ -2278,7 +2278,7 @@ typedef struct SVCreateTbReq {
   int32_t  flags;
   char*    name;
   tb_uid_t uid;
-  int64_t  ctime;
+  int64_t  btime;
   int32_t  ttl;
   int32_t  commentLen;
   char*    comment;
@@ -2415,10 +2415,12 @@ typedef struct {
   int32_t newTTL;
   int32_t newCommentLen;
   char*   newComment;
+  int64_t ctimeMs; // fill by vnode
 } SVAlterTbReq;
 
 int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
 int32_t tDecodeSVAlterTbReq(SDecoder* pDecoder, SVAlterTbReq* pReq);
+int32_t tDecodeSVAlterTbReqSetCtime(SDecoder* pDecoder, SVAlterTbReq* pReq, int64_t ctimeMs);
 
 typedef struct {
   int32_t        code;
@@ -3436,6 +3438,7 @@ typedef struct SDeleteRes {
   int64_t  affectedRows;
   char     tableFName[TSDB_TABLE_NAME_LEN];
   char     tsColName[TSDB_COL_NAME_LEN];
+  int64_t  ctimeMs; // fill by vnode
 } SDeleteRes;
 
 int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes);
@@ -3454,10 +3457,12 @@ int32_t tDecodeSSingleDeleteReq(SDecoder* pCoder, SSingleDeleteReq* pReq);
 typedef struct {
   int64_t suid;
   SArray* deleteReqs;  // SArray<SSingleDeleteReq>
+  int64_t ctimeMs;     // fill by vnode
 } SBatchDeleteReq;
 
 int32_t tEncodeSBatchDeleteReq(SEncoder* pCoder, const SBatchDeleteReq* pReq);
 int32_t tDecodeSBatchDeleteReq(SDecoder* pCoder, SBatchDeleteReq* pReq);
+int32_t tDecodeSBatchDeleteReqSetCtime(SDecoder* pDecoder, SBatchDeleteReq* pReq, int64_t ctimeMs);
 
 typedef struct {
   int32_t msgIdx;
@@ -3525,6 +3530,7 @@ typedef struct {
     SArray* aRowP;
     SArray* aCol;
   };
+  int64_t        ctimeMs;
 } SSubmitTbData;
 
 typedef struct {

include/libs/executor/storageapi.h

@@ -54,7 +54,7 @@ typedef struct SMetaEntry {
       SRSmaParam     rsmaParam;
     } stbEntry;
     struct {
-      int64_t  ctime;
+      int64_t  btime;
       int32_t  ttlDays;
       int32_t  commentLen;
       char*    comment;
@@ -62,7 +62,7 @@ typedef struct SMetaEntry {
       uint8_t* pTags;
     } ctbEntry;
     struct {
-      int64_t        ctime;
+      int64_t        btime;
       int32_t        ttlDays;
       int32_t        commentLen;
       char*          comment;

source/common/src/tglobal.c

@@ -110,6 +110,7 @@ int32_t tsQueryRspPolicy = 0;
 int64_t tsQueryMaxConcurrentTables = 200;  // unit is TSDB_TABLE_NUM_UNIT
 bool    tsEnableQueryHb = false;
 bool    tsEnableScience = false;  // on taos-cli show float and doulbe with scientific notation if true
+bool    tsTtlChangeOnWrite = false; // ttl delete time changes on last write if true
 int32_t tsQuerySmaOptimize = 0;
 int32_t tsQueryRsmaTolerance = 1000;  // the tolerance time (ms) to judge from which level to query rsma data.
 bool    tsQueryPlannerTrace = false;
@@ -513,6 +514,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 100000, 1) != 0) return -1;
+  if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, 0) != 0) return -1;
   if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, 0) != 0) return -1;
 
@@ -873,6 +875,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
 
   tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
   tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval;
+  tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval;
   tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32;
   tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN);
   tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32;
@@ -978,6 +981,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) {
         taosSetCoreDump(enableCore);
       } else if (strcasecmp("enableQueryHb", name) == 0) {
         tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval;
+      }  else if (strcasecmp("ttlChangeOnWrite", name) == 0) {
+        tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval;
       }
       break;
     }

source/common/src/tmsg.c

@@ -30,6 +30,9 @@
 
 #include "tlog.h"
 
+static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq);
+static int32_t tDecodeSBatchDeleteReqCommon(SDecoder *pDecoder, SBatchDeleteReq *pReq);
+
 int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
   if (pMsg == NULL) {
     terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
@@ -3161,7 +3164,7 @@ int32_t tSerializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableReq
   tEncoderInit(&encoder, buf, bufLen);
 
   if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->timestamp) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->timestampSec) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -3174,7 +3177,7 @@ int32_t tDeserializeSVDropTtlTableReq(void *buf, int32_t bufLen, SVDropTtlTableR
   tDecoderInit(&decoder, buf, bufLen);
 
   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->timestamp) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->timestampSec) < 0) return -1;
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
@@ -6409,7 +6412,7 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) {
   if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1;
   if (tEncodeCStr(pCoder, pReq->name) < 0) return -1;
   if (tEncodeI64(pCoder, pReq->uid) < 0) return -1;
-  if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1;
+  if (tEncodeI64(pCoder, pReq->btime) < 0) return -1;
   if (tEncodeI32(pCoder, pReq->ttl) < 0) return -1;
   if (tEncodeI8(pCoder, pReq->type) < 0) return -1;
   if (tEncodeI32(pCoder, pReq->commentLen) < 0) return -1;
@@ -6444,7 +6447,7 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) {
   if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1;
   if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1;
   if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1;
-  if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1;
+  if (tDecodeI64(pCoder, &pReq->btime) < 0) return -1;
   if (tDecodeI32(pCoder, &pReq->ttl) < 0) return -1;
   if (tDecodeI8(pCoder, &pReq->type) < 0) return -1;
   if (tDecodeI32(pCoder, &pReq->commentLen) < 0) return -1;
@@ -6909,14 +6912,13 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
     default:
       break;
   }
+  if (tEncodeI64(pEncoder, pReq->ctimeMs) < 0) return -1;
 
   tEndEncode(pEncoder);
   return 0;
 }
 
-int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
+static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq) {
-  if (tStartDecode(pDecoder) < 0) return -1;
-
   if (tDecodeCStr(pDecoder, &pReq->tbName) < 0) return -1;
   if (tDecodeI8(pDecoder, &pReq->action) < 0) return -1;
   if (tDecodeI32(pDecoder, &pReq->colId) < 0) return -1;
@@ -6960,6 +6962,28 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
     default:
       break;
   }
+  return 0;
+}
+
+int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+  if (tDecodeSVAlterTbReqCommon(pDecoder, pReq) < 0) return -1;
+
+  pReq->ctimeMs = 0;
+  if (!tDecodeIsEnd(pDecoder)) {
+    if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1;
+  }
+
+  tEndDecode(pDecoder);
+  return 0;
+}
+
+int32_t tDecodeSVAlterTbReqSetCtime(SDecoder* pDecoder, SVAlterTbReq* pReq, int64_t ctimeMs) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+  if (tDecodeSVAlterTbReqCommon(pDecoder, pReq) < 0) return -1;
+
+  *(int64_t *)(pDecoder->data + pDecoder->pos) = ctimeMs;
+  if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1;
 
   tEndDecode(pDecoder);
   return 0;
@@ -7238,6 +7262,7 @@ int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) {
 
   if (tEncodeCStr(pCoder, pRes->tableFName) < 0) return -1;
   if (tEncodeCStr(pCoder, pRes->tsColName) < 0) return -1;
+  if (tEncodeI64(pCoder, pRes->ctimeMs) < 0) return -1;
   return 0;
 }
 
@@ -7257,6 +7282,11 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) {
 
   if (tDecodeCStrTo(pCoder, pRes->tableFName) < 0) return -1;
   if (tDecodeCStrTo(pCoder, pRes->tsColName) < 0) return -1;
+
+  pRes->ctimeMs = 0;
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI64(pCoder, &pRes->ctimeMs) < 0) return -1;
+  }
   return 0;
 }
 
@@ -7480,10 +7510,11 @@ int32_t tEncodeSBatchDeleteReq(SEncoder *pEncoder, const SBatchDeleteReq *pReq)
     SSingleDeleteReq *pOneReq = taosArrayGet(pReq->deleteReqs, i);
     if (tEncodeSSingleDeleteReq(pEncoder, pOneReq) < 0) return -1;
   }
+  if (tEncodeI64(pEncoder, pReq->ctimeMs) < 0) return -1;
   return 0;
 }
 
-int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) {
+static int32_t tDecodeSBatchDeleteReqCommon(SDecoder *pDecoder, SBatchDeleteReq *pReq) {
   if (tDecodeI64(pDecoder, &pReq->suid) < 0) return -1;
   int32_t sz;
   if (tDecodeI32(pDecoder, &sz) < 0) return -1;
@@ -7497,6 +7528,24 @@ int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) {
   return 0;
 }
 
+int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) {
+  if (tDecodeSBatchDeleteReqCommon(pDecoder, pReq)) return -1;
+
+  pReq->ctimeMs = 0;
+  if (!tDecodeIsEnd(pDecoder)) {
+    if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1;
+  }
+  return 0;
+}
+
+int32_t tDecodeSBatchDeleteReqSetCtime(SDecoder *pDecoder, SBatchDeleteReq *pReq, int64_t ctimeMs) {
+  if (tDecodeSBatchDeleteReqCommon(pDecoder, pReq)) return -1;
+
+  *(int64_t *)(pDecoder->data + pDecoder->pos) = ctimeMs;
+  if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1;
+  return 0;
+}
+
 static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubmitTbData) {
   if (tStartEncode(pCoder) < 0) return -1;
 
@@ -7531,6 +7580,7 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm
       pCoder->pos += rows[iRow]->len;
     }
   }
+  if (tEncodeI64(pCoder, pSubmitTbData->ctimeMs) < 0) return -1;
 
   tEndEncode(pCoder);
   return 0;
@@ -7611,6 +7661,14 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa
     }
   }
 
+  pSubmitTbData->ctimeMs = 0;
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI64(pCoder, &pSubmitTbData->ctimeMs) < 0) {
+      code = TSDB_CODE_INVALID_MSG;
+      goto _exit;
+    }
+  }
+
   tEndDecode(pCoder);
 
 _exit:

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -46,6 +46,7 @@ typedef struct {
   int32_t vgId;
   int32_t vgVersion;
   int8_t  dropped;
+  int32_t toVgId;
   char    path[PATH_MAX + 20];
 } SWrapperCfg;
 
@@ -55,6 +56,7 @@ typedef struct {
   int32_t       refCount;
   int8_t        dropped;
   int8_t        disable;
+  int32_t       toVgId;
   char         *path;
   SVnode       *pImpl;
   SMultiWorker  pWriteW;
@@ -70,6 +72,7 @@ typedef struct {
   int32_t      vnodeNum;
   int32_t      opened;
   int32_t      failed;
+  bool         updateVnodesList;
   int32_t      threadIndex;
   TdThread     thread;
   SVnodeMgmt  *pMgmt;

source/dnode/mgmt/mgmt_vnode/src/vmFile.c

@@ -71,6 +71,8 @@ static int32_t vmDecodeVnodeList(SJson *pJson, SVnodeMgmt *pMgmt, SWrapperCfg **
     if (code < 0) goto _OVER;
     tjsonGetInt32ValueFromDouble(vnode, "vgVersion", pCfg->vgVersion, code);
     if (code < 0) goto _OVER;
+    tjsonGetInt32ValueFromDouble(vnode, "toVgId", pCfg->toVgId, code);
+    if (code < 0) goto _OVER;
 
     snprintf(pCfg->path, sizeof(pCfg->path), "%s%svnode%d", pMgmt->path, TD_DIRSEP, pCfg->vgId);
   }
@@ -165,6 +167,7 @@ static int32_t vmEncodeVnodeList(SJson *pJson, SVnodeObj **ppVnodes, int32_t num
     if (tjsonAddDoubleToObject(vnode, "vgId", pVnode->vgId) < 0) return -1;
     if (tjsonAddDoubleToObject(vnode, "dropped", pVnode->dropped) < 0) return -1;
     if (tjsonAddDoubleToObject(vnode, "vgVersion", pVnode->vgVersion) < 0) return -1;
+    if (pVnode->toVgId && tjsonAddDoubleToObject(vnode, "toVgId", pVnode->toVgId) < 0) return -1;
     if (tjsonAddItemToArray(vnodes, vnode) < 0) return -1;
   }
 
@@ -179,7 +182,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
   SVnodeObj **ppVnodes = NULL;
   char        file[PATH_MAX] = {0};
   char        realfile[PATH_MAX] = {0};
-  snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP);
+  snprintf(file, sizeof(file), "%s%svnodes_tmp.json", pMgmt->path, TD_DIRSEP);
   snprintf(realfile, sizeof(realfile), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
 
   int32_t numOfVnodes = 0;

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -484,10 +484,18 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
 
   int32_t srcVgId = req.srcVgId;
   int32_t dstVgId = req.dstVgId;
+
+  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, dstVgId);
+  if (pVnode != NULL) {
+    dError("vgId:%d, vnode already exist", dstVgId);
+    vmReleaseVnode(pMgmt, pVnode);
+    terrno = TSDB_CODE_VND_ALREADY_EXIST;
+    return -1;
+  }
+
   dInfo("vgId:%d, start to alter vnode hashrange:[%u, %u], dstVgId:%d", req.srcVgId, req.hashBegin, req.hashEnd,
         req.dstVgId);
-
+  pVnode = vmAcquireVnode(pMgmt, srcVgId);
-  SVnodeObj *pVnode = vmAcquireVnode(pMgmt, srcVgId);
   if (pVnode == NULL) {
     dError("vgId:%d, failed to alter hashrange since %s", srcVgId, terrstr());
     terrno = TSDB_CODE_VND_NOT_EXIST;
@@ -501,6 +509,13 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   };
   tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path));
 
+  // prepare alter
+  pVnode->toVgId = dstVgId;
+  if (vmWriteVnodeListToFile(pMgmt) != 0) {
+    dError("vgId:%d, failed to write vnode list since %s", dstVgId, terrstr());
+    return -1;
+  }
+
   dInfo("vgId:%d, close vnode", srcVgId);
   vmCloseVnode(pMgmt, pVnode, true);
 
@@ -532,6 +547,7 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     return -1;
   }
 
+  // complete alter
   if (vmWriteVnodeListToFile(pMgmt) != 0) {
     dError("vgId:%d, failed to write vnode list since %s", dstVgId, terrstr());
     return -1;

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -158,6 +158,28 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
   taosMemoryFree(pVnode);
 }
 
+static int32_t vmRestoreVgroupId(SWrapperCfg *pCfg, STfs *pTfs) {
+  int32_t srcVgId = pCfg->vgId;
+  int32_t dstVgId = pCfg->toVgId;
+  if (dstVgId == 0) return 0;
+
+  char srcPath[TSDB_FILENAME_LEN];
+  char dstPath[TSDB_FILENAME_LEN];
+
+  snprintf(srcPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, srcVgId);
+  snprintf(dstPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, dstVgId);
+
+  int32_t vgId = vnodeRestoreVgroupId(srcPath, dstPath, srcVgId, dstVgId, pTfs);
+  if (vgId <= 0) {
+    dError("vgId:%d, failed to restore vgroup id. srcPath: %s", pCfg->vgId, srcPath);
+    return -1;
+  }
+
+  pCfg->vgId = vgId;
+  pCfg->toVgId = 0;
+  return 0;
+}
+
 static void *vmOpenVnodeInThread(void *param) {
   SVnodeThread *pThread = param;
   SVnodeMgmt   *pMgmt = pThread->pMgmt;
@@ -174,18 +196,34 @@ static void *vmOpenVnodeInThread(void *param) {
              pMgmt->state.openVnodes, pMgmt->state.totalVnodes);
     tmsgReportStartup("vnode-open", stepDesc);
 
+    if (pCfg->toVgId) {
+      if (vmRestoreVgroupId(pCfg, pMgmt->pTfs) != 0) {
+        dError("vgId:%d, failed to restore vgroup id by thread:%d", pCfg->vgId, pThread->threadIndex);
+        pThread->failed++;
+        continue;
+      }
+      pThread->updateVnodesList = true;
+    }
+
     snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId);
+
     SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb);
     if (pImpl == NULL) {
       dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
       pThread->failed++;
-    } else {
+      continue;
-      vmOpenVnode(pMgmt, pCfg, pImpl);
+    }
+
+    if (vmOpenVnode(pMgmt, pCfg, pImpl) != 0) {
+      dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
+      pThread->failed++;
+      continue;
+    }
+
     dInfo("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
     pThread->opened++;
     atomic_add_fetch_32(&pMgmt->state.openVnodes, 1);
   }
-  }
 
   dInfo("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
         pThread->failed);
@@ -242,6 +280,8 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
     taosThreadAttrDestroy(&thAttr);
   }
 
+  bool updateVnodesList = false;
+
   for (int32_t t = 0; t < threadNum; ++t) {
     SVnodeThread *pThread = &threads[t];
     if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) {
@@ -249,6 +289,7 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
       taosThreadClear(&pThread->thread);
     }
     taosMemoryFree(pThread->pCfgs);
+    if (pThread->updateVnodesList) updateVnodesList = true;
   }
   taosMemoryFree(threads);
   taosMemoryFree(pCfgs);
@@ -256,10 +297,15 @@ static int32_t vmOpenVnodes(SVnodeMgmt *pMgmt) {
   if (pMgmt->state.openVnodes != pMgmt->state.totalVnodes) {
     dError("there are total vnodes:%d, opened:%d", pMgmt->state.totalVnodes, pMgmt->state.openVnodes);
     return -1;
-  } else {
+  }
+
+  if (updateVnodesList && vmWriteVnodeListToFile(pMgmt) != 0) {
+    dError("failed to write vnode list since %s", terrstr());
+    return -1;
+  }
+
   dInfo("successfully opened %d vnodes", pMgmt->state.totalVnodes);
   return 0;
-  }
 }
 
 static void *vmCloseVnodeInThread(void *param) {

source/dnode/mnode/impl/src/mndStb.c

@@ -888,7 +888,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
   SSdb             *pSdb = pMnode->pSdb;
   SVgObj           *pVgroup = NULL;
   void             *pIter = NULL;
-  SVDropTtlTableReq ttlReq = {.timestamp = taosGetTimestampSec()};
+  SVDropTtlTableReq ttlReq = {.timestampSec = taosGetTimestampSec()};
   int32_t           reqLen = tSerializeSVDropTtlTableReq(NULL, 0, &ttlReq);
   int32_t           contLen = reqLen + sizeof(SMsgHead);
 
@@ -914,7 +914,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
     if (code != 0) {
       mError("vgId:%d, failed to send drop ttl table request to vnode since 0x%x", pVgroup->vgId, code);
     } else {
-      mInfo("vgId:%d, send drop ttl table request to vnode, time:%d", pVgroup->vgId, ttlReq.timestamp);
+      mInfo("vgId:%d, send drop ttl table request to vnode, time:%" PRId32, pVgroup->vgId, ttlReq.timestampSec);
     }
     sdbRelease(pSdb, pVgroup);
   }

source/dnode/mnode/impl/src/mndVgroup.c

@@ -1217,6 +1217,7 @@ static int32_t mndAddAlterVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, i
   action.pCont = pReq;
   action.contLen = contLen;
   action.msgType = TDMT_VND_ALTER_HASHRANGE;
+  action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST;
 
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(pReq);

source/dnode/vnode/CMakeLists.txt

@@ -24,6 +24,7 @@ set(
     "src/meta/metaEntry.c"
     "src/meta/metaSnapshot.c"
     "src/meta/metaCache.c"
+    "src/meta/metaTtl.c"
 
   # sma
   "src/sma/smaEnv.c"
@@ -90,6 +91,7 @@ IF (TD_VNODE_PLUGINS)
   )
 ENDIF ()
 
+IF (NOT ${TD_LINUX})
 target_include_directories(
     vnode
     PUBLIC "inc"
@@ -97,7 +99,25 @@ target_include_directories(
     PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
     PUBLIC "${TD_SOURCE_DIR}/contrib/rocksdb/include"
 )
+ELSE()
+target_include_directories(
+    vnode
+    PUBLIC "inc"
+    PUBLIC "src/inc"
+    PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
+)
+ENDIF (NOT ${TD_LINUX})
+
 IF (TD_LINUX)
+target_include_directories(
+      vnode
+      PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static"
+    )
+
+    target_link_directories(
+        vnode
+        PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static"
+    )
 target_link_libraries(
     vnode
     PUBLIC os

source/dnode/vnode/inc/vnode.h

@@ -54,6 +54,7 @@ void    vnodeCleanup();
 int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs);
 int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs);
 int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, STfs *pTfs);
+int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs);
 void    vnodeDestroy(const char *path, STfs *pTfs);
 SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb);
 void    vnodePreClose(SVnode *pVnode);
@@ -114,6 +115,7 @@ int         metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
 int      metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName);
 int      metaGetTableUidByName(void *pVnode, char *tbName, uint64_t *uid);
 int      metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType);
+int      metaGetTableTtlByUid(void *meta, uint64_t uid, int64_t *ttlDays);
 bool     metaIsTableExist(void* pVnode, tb_uid_t uid);
 int32_t  metaGetCachedTableUidList(void *pVnode, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList,
                                    bool *acquired);

source/dnode/vnode/src/inc/meta.h

@@ -17,6 +17,7 @@
 #define _TD_VNODE_META_H_
 
 #include "index.h"
+#include "metaTtl.h"
 #include "vnodeInt.h"
 
 #ifdef __cplusplus
@@ -90,9 +91,9 @@ struct SMeta {
   void* pTagIvtIdx;
 
   TTB*        pTagIdx;
-  TTB* pTtlIdx;
+  STtlManger* pTtlMgr;
 
-  TTB* pCtimeIdx;  // table created time idx
+  TTB* pBtimeIdx;  // table created time idx
   TTB* pNcolIdx;   // ncol of table idx, normal table only
 
   TTB* pSmaIdx;
@@ -138,20 +139,15 @@ typedef struct {
 } STagIdxKey;
 #pragma pack(pop)
 
-typedef struct {
-  int64_t  dtime;
-  tb_uid_t uid;
-} STtlIdxKey;
-
 typedef struct {
   tb_uid_t uid;
   int64_t  smaUid;
 } SSmaIdxKey;
 
 typedef struct {
-  int64_t  ctime;
+  int64_t  btime;
   tb_uid_t uid;
-} SCtimeIdxKey;
+} SBtimeIdxKey;
 
 typedef struct {
   int64_t  ncol;

source/dnode/vnode/src/inc/metaTtl.h

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_TTL_H_
+#define _TD_VNODE_TTL_H_
+
+#include "taosdef.h"
+#include "thash.h"
+
+#include "tdb.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum DirtyEntryType {
+  ENTRY_TYPE_DEL = 1,
+  ENTRY_TYPE_UPSERT = 2,
+} DirtyEntryType;
+
+typedef struct STtlManger {
+  TdThreadRwlock lock;
+
+  TTB* pOldTtlIdx;       // btree<{deleteTime, tuid}, NULL>
+
+  SHashObj* pTtlCache;   // key: tuid, value: {ttl, ctime}
+  SHashObj* pDirtyUids;  // dirty tuid
+  TTB*      pTtlIdx;     // btree<{deleteTime, tuid}, ttl>
+} STtlManger;
+
+typedef struct {
+  int64_t ttlDays;
+  int64_t changeTimeMs;
+} STtlCacheEntry;
+
+typedef struct {
+  DirtyEntryType type;
+} STtlDirtyEntry;
+
+typedef struct {
+  int64_t  deleteTimeSec;
+  tb_uid_t uid;
+} STtlIdxKey;
+
+typedef struct {
+  int64_t  deleteTimeMs;
+  tb_uid_t uid;
+} STtlIdxKeyV1;
+
+typedef struct {
+  int64_t ttlDays;
+} STtlIdxValue;
+
+typedef struct {
+  tb_uid_t uid;
+  int64_t  changeTimeMs;
+} STtlUpdCtimeCtx;
+
+typedef struct {
+  tb_uid_t uid;
+  int64_t  changeTimeMs;
+  int64_t  ttlDays;
+} STtlUpdTtlCtx;
+
+typedef struct {
+  tb_uid_t uid;
+  TXN*     pTxn;
+} STtlDelTtlCtx;
+
+int ttlMgrOpen(STtlManger** ppTtlMgr, TDB* pEnv, int8_t rollback);
+int ttlMgrClose(STtlManger* pTtlMgr);
+int ttlMgrBegin(STtlManger* pTtlMgr, void* pMeta);
+
+int ttlMgrConvert(TTB* pOldTtlIdx, TTB* pNewTtlIdx, void* pMeta);
+int ttlMgrFlush(STtlManger* pTtlMgr, TXN* pTxn);
+
+int ttlMgrInsertTtl(STtlManger* pTtlMgr, const STtlUpdTtlCtx* pUpdCtx);
+int ttlMgrDeleteTtl(STtlManger* pTtlMgr, const STtlDelTtlCtx* pDelCtx);
+int ttlMgrUpdateChangeTime(STtlManger* pTtlMgr, const STtlUpdCtimeCtx* pUpdCtimeCtx);
+int ttlMgrFindExpired(STtlManger* pTtlMgr, int64_t timePointMs, SArray* pTbUids);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_TTL_H_*/

source/dnode/vnode/src/inc/vnodeInt.h

@@ -149,8 +149,9 @@ int             metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq
 int             metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp);
 int             metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid);
 int32_t         metaTrimTables(SMeta* pMeta);
-int             metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids);
+int             metaTtlDropTable(SMeta* pMeta, int64_t timePointMs, SArray* tbUids);
 int             metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
+int             metaUpdateChangeTime(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs);
 SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);
 STSchema*       metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);
 int32_t         metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sver, STSchema** ppTSchema);
@@ -176,7 +177,6 @@ SArray*       metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid);
 SArray*       metaGetSmaTbUids(SMeta* pMeta);
 void*         metaGetIdx(SMeta* pMeta);
 void*         metaGetIvtIdx(SMeta* pMeta);
-int           metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList);
 
 void metaReaderInit(SMetaReader* pReader, SMeta* pMeta, int32_t flags);
 

source/dnode/vnode/src/meta/metaCommit.c

@@ -40,6 +40,12 @@ int metaBegin(SMeta *pMeta, int8_t heap) {
     return -1;
   }
 
+  if (ttlMgrBegin(pMeta->pTtlMgr, pMeta) < 0) {
+    return -1;
+  }
+
+  tdbCommit(pMeta->pEnv, pMeta->txn);
+
   return 0;
 }
 
@@ -50,6 +56,7 @@ int  metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv
 int  metaPrepareAsyncCommit(SMeta *pMeta) {
    // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn);
   int code = 0;
+  code = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn);
   code = tdbCommit(pMeta->pEnv, pMeta->txn);
 
   return code;

source/dnode/vnode/src/meta/metaEntry.c

@@ -31,7 +31,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
       if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
     }
   } else if (pME->type == TSDB_CHILD_TABLE) {
-    if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1;
+    if (tEncodeI64(pCoder, pME->ctbEntry.btime) < 0) return -1;
     if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1;
     if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1;
     if (pME->ctbEntry.commentLen > 0) {
@@ -40,7 +40,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
     if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1;
     if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1;
   } else if (pME->type == TSDB_NORMAL_TABLE) {
-    if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
+    if (tEncodeI64(pCoder, pME->ntbEntry.btime) < 0) return -1;
     if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
     if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1;
     if (pME->ntbEntry.commentLen > 0) {
@@ -76,7 +76,7 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
       if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
     }
   } else if (pME->type == TSDB_CHILD_TABLE) {
-    if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1;
+    if (tDecodeI64(pCoder, &pME->ctbEntry.btime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
     if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1;
     if (pME->ctbEntry.commentLen > 0) {
@@ -85,7 +85,7 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
     if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1;
     if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1;  // (TODO)
   } else if (pME->type == TSDB_NORMAL_TABLE) {
-    if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
+    if (tDecodeI64(pCoder, &pME->ntbEntry.btime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
     if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1;
     if (pME->ntbEntry.commentLen > 0) {

source/dnode/vnode/src/meta/metaOpen.c

@@ -19,12 +19,11 @@ static int tbDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen
 static int skmDbKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int ctbIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
-static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int uidIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int taskIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 
-static int ctimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static int btimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int ncolIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 
 static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); }
@@ -128,8 +127,8 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
     goto _err;
   }
 
-  // open pTtlIdx
+  // open pTtlMgr ("ttlv1.idx")
-  ret = tdbTbOpen("ttl.idx", sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pMeta->pEnv, &pMeta->pTtlIdx, 0);
+  ret = ttlMgrOpen(&pMeta->pTtlMgr, pMeta->pEnv, 0);
   if (ret < 0) {
     metaError("vgId:%d, failed to open meta ttl index since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
@@ -143,7 +142,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
   }
 
   // idx table create time
-  ret = tdbTbOpen("ctime.idx", sizeof(SCtimeIdxKey), 0, ctimeIdxCmpr, pMeta->pEnv, &pMeta->pCtimeIdx, 0);
+  ret = tdbTbOpen("ctime.idx", sizeof(SBtimeIdxKey), 0, btimeIdxCmpr, pMeta->pEnv, &pMeta->pBtimeIdx, 0);
   if (ret < 0) {
     metaError("vgId:%d, failed to open meta ctime index since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
@@ -184,9 +183,9 @@ _err:
   if (pMeta->pIdx) metaCloseIdx(pMeta);
   if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb);
   if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx);
-  if (pMeta->pCtimeIdx) tdbTbClose(pMeta->pCtimeIdx);
+  if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx);
   if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
-  if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
+  if (pMeta->pTtlMgr) ttlMgrClose(pMeta->pTtlMgr);
   if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
   if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
   if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
@@ -209,9 +208,9 @@ int metaClose(SMeta **ppMeta) {
     if (pMeta->pIdx) metaCloseIdx(pMeta);
     if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb);
     if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx);
-    if (pMeta->pCtimeIdx) tdbTbClose(pMeta->pCtimeIdx);
+    if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx);
     if (pMeta->pSmaIdx) tdbTbClose(pMeta->pSmaIdx);
-    if (pMeta->pTtlIdx) tdbTbClose(pMeta->pTtlIdx);
+    if (pMeta->pTtlMgr) ttlMgrClose(pMeta->pTtlMgr);
     if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx);
     if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx);
     if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx);
@@ -399,37 +398,18 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL
   return 0;
 }
 
-static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+static int btimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
-  STtlIdxKey *pTtlIdxKey1 = (STtlIdxKey *)pKey1;
+  SBtimeIdxKey *pBtimeIdxKey1 = (SBtimeIdxKey *)pKey1;
-  STtlIdxKey *pTtlIdxKey2 = (STtlIdxKey *)pKey2;
+  SBtimeIdxKey *pBtimeIdxKey2 = (SBtimeIdxKey *)pKey2;
-
+  if (pBtimeIdxKey1->btime > pBtimeIdxKey2->btime) {
-  if (pTtlIdxKey1->dtime > pTtlIdxKey2->dtime) {
     return 1;
-  } else if (pTtlIdxKey1->dtime < pTtlIdxKey2->dtime) {
+  } else if (pBtimeIdxKey1->btime < pBtimeIdxKey2->btime) {
     return -1;
   }
 
-  if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) {
+  if (pBtimeIdxKey1->uid > pBtimeIdxKey2->uid) {
     return 1;
-  } else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) {
+  } else if (pBtimeIdxKey1->uid < pBtimeIdxKey2->uid) {
-    return -1;
-  }
-
-  return 0;
-}
-
-static int ctimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
-  SCtimeIdxKey *pCtimeIdxKey1 = (SCtimeIdxKey *)pKey1;
-  SCtimeIdxKey *pCtimeIdxKey2 = (SCtimeIdxKey *)pKey2;
-  if (pCtimeIdxKey1->ctime > pCtimeIdxKey2->ctime) {
-    return 1;
-  } else if (pCtimeIdxKey1->ctime < pCtimeIdxKey2->ctime) {
-    return -1;
-  }
-
-  if (pCtimeIdxKey1->uid > pCtimeIdxKey2->uid) {
-    return 1;
-  } else if (pCtimeIdxKey1->uid < pCtimeIdxKey2->uid) {
     return -1;
   }
 

source/dnode/vnode/src/meta/metaQuery.c

@@ -212,6 +212,29 @@ int metaReadNext(SMetaReader *pReader) {
   return 0;
 }
 
+int metaGetTableTtlByUid(void *meta, uint64_t uid, int64_t *ttlDays) {
+  int         code = -1;
+  SMetaReader mr = {0};
+  metaReaderInit(&mr, (SMeta *)meta, 0);
+  code = metaReaderGetTableEntryByUid(&mr, uid);
+  if (code < 0) {
+    goto _exit;
+  }
+  if (mr.me.type == TSDB_CHILD_TABLE) {
+    *ttlDays = mr.me.ctbEntry.ttlDays;
+  } else if (mr.me.type == TSDB_NORMAL_TABLE) {
+    *ttlDays = mr.me.ntbEntry.ttlDays;
+  } else {
+    goto _exit;
+  }
+
+  code = 0;
+
+_exit:
+  metaReaderClear(&mr);
+  return code;
+}
+
 #if 1  // ===================================================
 SMTbCursor *metaOpenTbCursor(void *pVnode) {
   SMTbCursor *pTbCur = NULL;
@@ -387,37 +410,6 @@ _err:
   return NULL;
 }
 
-int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList) {
-  TBC *pCur;
-  int  ret = tdbTbcOpen(pMeta->pTtlIdx, &pCur, NULL);
-  if (ret < 0) {
-    return ret;
-  }
-
-  STtlIdxKey ttlKey = {0};
-  ttlKey.dtime = ttl;
-  ttlKey.uid = INT64_MAX;
-  int c = 0;
-  tdbTbcMoveTo(pCur, &ttlKey, sizeof(ttlKey), &c);
-  if (c < 0) {
-    tdbTbcMoveToPrev(pCur);
-  }
-
-  void *pKey = NULL;
-  int   kLen = 0;
-  while (1) {
-    ret = tdbTbcPrev(pCur, &pKey, &kLen, NULL, NULL);
-    if (ret < 0) {
-      break;
-    }
-    ttlKey = *(STtlIdxKey *)pKey;
-    taosArrayPush(uidList, &ttlKey.uid);
-  }
-  tdbFree(pKey);
-  tdbTbcClose(pCur);
-  return 0;
-}
-
 struct SMCtbCursor {
   SMeta   *pMeta;
   TBC     *pCur;
@@ -1018,17 +1010,17 @@ int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
   pCursor->type = param->type;
 
   metaRLock(pMeta);
-  ret = tdbTbcOpen(pMeta->pCtimeIdx, &pCursor->pCur, NULL);
+  ret = tdbTbcOpen(pMeta->pBtimeIdx, &pCursor->pCur, NULL);
   if (ret != 0) {
     goto END;
   }
   int64_t uidLimit = param->reverse ? INT64_MAX : 0;
 
-  SCtimeIdxKey  ctimeKey = {.ctime = *(int64_t *)(param->val), .uid = uidLimit};
+  SBtimeIdxKey  btimeKey = {.btime = *(int64_t *)(param->val), .uid = uidLimit};
-  SCtimeIdxKey *pCtimeKey = &ctimeKey;
+  SBtimeIdxKey *pBtimeKey = &btimeKey;
 
   int cmp = 0;
-  if (tdbTbcMoveTo(pCursor->pCur, &ctimeKey, sizeof(ctimeKey), &cmp) < 0) {
+  if (tdbTbcMoveTo(pCursor->pCur, &btimeKey, sizeof(btimeKey), &cmp) < 0) {
     goto END;
   }
 
@@ -1042,10 +1034,10 @@ int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
     valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, NULL, NULL);
     if (valid < 0) break;
 
-    SCtimeIdxKey *p = entryKey;
+    SBtimeIdxKey *p = entryKey;
     if (count > TRY_ERROR_LIMIT) break;
 
-    int32_t cmp = (*param->filterFunc)((void *)&p->ctime, (void *)&pCtimeKey->ctime, param->type);
+    int32_t cmp = (*param->filterFunc)((void *)&p->btime, (void *)&pBtimeKey->btime, param->type);
     if (cmp == 0)
       taosArrayPush(pUids, &p->uid);
     else {
@@ -1149,7 +1141,7 @@ int32_t metaFilterTtl(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
   pCursor->type = param->type;
 
   metaRLock(pMeta);
-  ret = tdbTbcOpen(pMeta->pTtlIdx, &pCursor->pCur, NULL);
+  //ret = tdbTbcOpen(pMeta->pTtlIdx, &pCursor->pCur, NULL);
 
 END:
   if (pCursor->pMeta) metaULock(pCursor->pMeta);

source/dnode/vnode/src/meta/metaTable.c

@@ -20,7 +20,7 @@ static int  metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, con
 static int  metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME);
-static int  metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME);
+static int  metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME);
@@ -28,8 +28,8 @@ static int  metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry);
 static int  metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type);
 static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey);
 // opt ins_tables query
-static int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
+static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
-static int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
+static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
 
@@ -734,7 +734,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
   me.uid = pReq->uid;
   me.name = pReq->name;
   if (me.type == TSDB_CHILD_TABLE) {
-    me.ctbEntry.ctime = pReq->ctime;
+    me.ctbEntry.btime = pReq->btime;
     me.ctbEntry.ttlDays = pReq->ttl;
     me.ctbEntry.commentLen = pReq->commentLen;
     me.ctbEntry.comment = pReq->comment;
@@ -770,7 +770,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
     metaTbGroupCacheClear(pMeta, me.ctbEntry.suid);
     metaULock(pMeta);
   } else {
-    me.ntbEntry.ctime = pReq->ctime;
+    me.ntbEntry.btime = pReq->btime;
     me.ntbEntry.ttlDays = pReq->ttl;
     me.ntbEntry.commentLen = pReq->commentLen;
     me.ntbEntry.comment = pReq->comment;
@@ -923,50 +923,40 @@ end:
   return code;
 }
 
-int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) {
+int metaTtlDropTable(SMeta *pMeta, int64_t timePointMs, SArray *tbUids) {
-  int ret = metaTtlSmaller(pMeta, ttl, tbUids);
+  int ret = ttlMgrFlush(pMeta->pTtlMgr, pMeta->txn);
   if (ret != 0) {
+    metaError("ttl failed to flush, ret:%d", ret);
+    return ret;
+  }
+
+  ret = ttlMgrFindExpired(pMeta->pTtlMgr, timePointMs, tbUids);
+  if (ret != 0) {
+    metaError("ttl failed to find expired table, ret:%d", ret);
     return ret;
   }
   if (TARRAY_SIZE(tbUids) == 0) {
     return 0;
   }
 
+  metaInfo("ttl find expired table count: %zu" , TARRAY_SIZE(tbUids));
+
   metaDropTables(pMeta, tbUids);
   return 0;
 }
 
-static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME) {
+static int metaBuildBtimeIdxKey(SBtimeIdxKey *btimeKey, const SMetaEntry *pME) {
-  int64_t ttlDays = 0;
+  int64_t btime;
-  int64_t ctime = 0;
   if (pME->type == TSDB_CHILD_TABLE) {
-    ctime = pME->ctbEntry.ctime;
+    btime = pME->ctbEntry.btime;
-    ttlDays = pME->ctbEntry.ttlDays;
   } else if (pME->type == TSDB_NORMAL_TABLE) {
-    ctime = pME->ntbEntry.ctime;
+    btime = pME->ntbEntry.btime;
-    ttlDays = pME->ntbEntry.ttlDays;
-  } else {
-    metaError("meta/table: invalide table type: %" PRId8 " build ttl idx key failed.", pME->type);
-    return;
-  }
-
-  if (ttlDays <= 0) return;
-
-  ttlKey->dtime = ctime / 1000 + ttlDays * tsTtlUnit;
-  ttlKey->uid = pME->uid;
-}
-static int metaBuildCtimeIdxKey(SCtimeIdxKey *ctimeKey, const SMetaEntry *pME) {
-  int64_t ctime;
-  if (pME->type == TSDB_CHILD_TABLE) {
-    ctime = pME->ctbEntry.ctime;
-  } else if (pME->type == TSDB_NORMAL_TABLE) {
-    ctime = pME->ntbEntry.ctime;
   } else {
     return -1;
   }
 
-  ctimeKey->ctime = ctime;
+  btimeKey->btime = btime;
-  ctimeKey->uid = pME->uid;
+  btimeKey->uid = pME->uid;
   return 0;
 }
 
@@ -980,11 +970,9 @@ static int metaBuildNColIdxKey(SNcolIdxKey *ncolKey, const SMetaEntry *pME) {
   return 0;
 }
 
-static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
+static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) {
-  STtlIdxKey ttlKey = {0};
+  STtlDelTtlCtx ctx = {.uid = pME->uid, .pTxn = pMeta->txn};
-  metaBuildTtlIdxKey(&ttlKey, pME);
+  return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx);
-  if (ttlKey.dtime == 0) return 0;
-  return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), pMeta->txn);
 }
 
 static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
@@ -1066,10 +1054,10 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
   tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, pMeta->txn);
   tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), pMeta->txn);
 
-  if (e.type == TSDB_CHILD_TABLE || e.type == TSDB_NORMAL_TABLE) metaDeleteCtimeIdx(pMeta, &e);
+  if (e.type == TSDB_CHILD_TABLE || e.type == TSDB_NORMAL_TABLE) metaDeleteBtimeIdx(pMeta, &e);
   if (e.type == TSDB_NORMAL_TABLE) metaDeleteNcolIdx(pMeta, &e);
 
-  if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e);
+  if (e.type != TSDB_SUPER_TABLE) metaDeleteTtl(pMeta, &e);
 
   if (e.type == TSDB_CHILD_TABLE) {
     tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), pMeta->txn);
@@ -1102,23 +1090,23 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
   return 0;
 }
 // opt ins_tables
-int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
+int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  SCtimeIdxKey ctimeKey = {0};
+  SBtimeIdxKey btimeKey = {0};
-  if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) {
+  if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) {
     return 0;
   }
-  metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " ctime:%" PRId64, TD_VID(pMeta->pVnode),
+  metaTrace("vgId:%d, start to save version:%" PRId64 " uid:%" PRId64 " btime:%" PRId64, TD_VID(pMeta->pVnode),
-            pME->version, pME->uid, ctimeKey.ctime);
+            pME->version, pME->uid, btimeKey.btime);
 
-  return tdbTbUpsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn);
+  return tdbTbUpsert(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), NULL, 0, pMeta->txn);
 }
 
-int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
+int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  SCtimeIdxKey ctimeKey = {0};
+  SBtimeIdxKey btimeKey = {0};
-  if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) {
+  if (metaBuildBtimeIdxKey(&btimeKey, pME) < 0) {
     return 0;
   }
-  return tdbTbDelete(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), pMeta->txn);
+  return tdbTbDelete(pMeta->pBtimeIdx, &btimeKey, sizeof(btimeKey), pMeta->txn);
 }
 int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SNcolIdxKey ncolKey = {0};
@@ -1328,6 +1316,8 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
 
   metaULock(pMeta);
 
+  metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs);
+
   metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp);
 
   if (entry.pBuf) taosMemoryFree(entry.pBuf);
@@ -1515,6 +1505,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
 
   metaULock(pMeta);
 
+  metaUpdateChangeTime(pMeta, ctbEntry.uid, pAlterTbReq->ctimeMs);
+
   tDecoderClear(&dc1);
   tDecoderClear(&dc2);
   taosMemoryFree((void *)ctbEntry.ctbEntry.pTags);
@@ -1603,9 +1595,9 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
   // build SMetaEntry
   if (entry.type == TSDB_CHILD_TABLE) {
     if (pAlterTbReq->updateTTL) {
-      metaDeleteTtlIdx(pMeta, &entry);
+      metaDeleteTtl(pMeta, &entry);
       entry.ctbEntry.ttlDays = pAlterTbReq->newTTL;
-      metaUpdateTtlIdx(pMeta, &entry);
+      metaUpdateTtl(pMeta, &entry);
     }
     if (pAlterTbReq->newCommentLen >= 0) {
       entry.ctbEntry.commentLen = pAlterTbReq->newCommentLen;
@@ -1613,9 +1605,9 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
     }
   } else {
     if (pAlterTbReq->updateTTL) {
-      metaDeleteTtlIdx(pMeta, &entry);
+      metaDeleteTtl(pMeta, &entry);
       entry.ntbEntry.ttlDays = pAlterTbReq->newTTL;
-      metaUpdateTtlIdx(pMeta, &entry);
+      metaUpdateTtl(pMeta, &entry);
     }
     if (pAlterTbReq->newCommentLen >= 0) {
       entry.ntbEntry.commentLen = pAlterTbReq->newCommentLen;
@@ -1628,6 +1620,8 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
   metaUpdateUidIdx(pMeta, &entry);
   metaULock(pMeta);
 
+  metaUpdateChangeTime(pMeta, entry.uid, pAlterTbReq->ctimeMs);
+
   tdbTbcClose(pTbDbc);
   tdbTbcClose(pUidIdxc);
   tDecoderClear(&dc);
@@ -1967,11 +1961,28 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
   return tdbTbUpsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn);
 }
 
-static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
+static int metaUpdateTtl(SMeta *pMeta, const SMetaEntry *pME) {
-  STtlIdxKey ttlKey = {0};
+  if (pME->type != TSDB_CHILD_TABLE && pME->type != TSDB_NORMAL_TABLE) return 0;
-  metaBuildTtlIdxKey(&ttlKey, pME);
+
-  if (ttlKey.dtime == 0) return 0;
+  STtlUpdTtlCtx ctx = {.uid = pME->uid};
-  return tdbTbUpsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn);
+
+  if (pME->type == TSDB_CHILD_TABLE) {
+    ctx.ttlDays = pME->ctbEntry.ttlDays;
+    ctx.changeTimeMs = pME->ctbEntry.btime;
+  } else {
+    ctx.ttlDays = pME->ntbEntry.ttlDays;
+    ctx.changeTimeMs = pME->ntbEntry.btime;
+  }
+
+  return ttlMgrInsertTtl(pMeta->pTtlMgr, &ctx);
+}
+
+int metaUpdateChangeTime(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeMs) {
+  if (!tsTtlChangeOnWrite) return 0;
+
+  STtlUpdCtimeCtx ctx = {.uid = uid, .changeTimeMs = changeTimeMs};
+
+  return ttlMgrUpdateChangeTime(pMeta->pTtlMgr, &ctx);
 }
 
 static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -2182,7 +2193,7 @@ int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
     }
   }
 
-  code = metaUpdateCtimeIdx(pMeta, pME);
+  code = metaUpdateBtimeIdx(pMeta, pME);
   VND_CHECK_CODE(code, line, _err);
 
   if (pME->type == TSDB_NORMAL_TABLE) {
@@ -2191,7 +2202,7 @@ int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
   }
 
   if (pME->type != TSDB_SUPER_TABLE) {
-    code = metaUpdateTtlIdx(pMeta, pME);
+    code = metaUpdateTtl(pMeta, pME);
     VND_CHECK_CODE(code, line, _err);
   }
 

source/dnode/vnode/src/meta/metaTtl.c

@@ -0,0 +1,434 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "metaTtl.h"
+#include "meta.h"
+
+typedef struct {
+  TTB   *pNewTtlIdx;
+  SMeta *pMeta;
+} SConvertData;
+
+static void    ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid);
+static int     ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static int     ttlIdxKeyV1Cmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
+static int     ttlMgrFillCache(STtlManger *pTtlMgr);
+static int32_t ttlMgrFillCacheOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pTtlCache);
+static int32_t ttlMgrConvertOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pConvertData);
+
+static int32_t ttlMgrWLock(STtlManger *pTtlMgr);
+static int32_t ttlMgrRLock(STtlManger *pTtlMgr);
+static int32_t ttlMgrULock(STtlManger *pTtlMgr);
+
+const char *ttlTbname = "ttl.idx";
+const char *ttlV1Tbname = "ttlv1.idx";
+
+int ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback) {
+  int ret;
+
+  *ppTtlMgr = NULL;
+
+  STtlManger *pTtlMgr = (STtlManger *)tdbOsCalloc(1, sizeof(*pTtlMgr));
+  if (pTtlMgr == NULL) {
+    return -1;
+  }
+
+  if (tdbTbExist(ttlTbname, pEnv)) {
+    ret = tdbTbOpen(ttlTbname, sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, pEnv, &pTtlMgr->pOldTtlIdx, rollback);
+    if (ret < 0) {
+      metaError("failed to open %s index since %s", ttlTbname, tstrerror(terrno));
+      return ret;
+    }
+  }
+
+  ret = tdbTbOpen(ttlV1Tbname, TDB_VARIANT_LEN, TDB_VARIANT_LEN, ttlIdxKeyV1Cmpr, pEnv, &pTtlMgr->pTtlIdx, rollback);
+  if (ret < 0) {
+    metaError("failed to open %s since %s", ttlV1Tbname, tstrerror(terrno));
+
+    tdbOsFree(pTtlMgr);
+    return ret;
+  }
+
+  pTtlMgr->pTtlCache = taosHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+  pTtlMgr->pDirtyUids = taosHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+
+  taosThreadRwlockInit(&pTtlMgr->lock, NULL);
+
+  *ppTtlMgr = pTtlMgr;
+  return 0;
+}
+
+int ttlMgrClose(STtlManger *pTtlMgr) {
+  taosHashCleanup(pTtlMgr->pTtlCache);
+  taosHashCleanup(pTtlMgr->pDirtyUids);
+  tdbTbClose(pTtlMgr->pTtlIdx);
+  taosThreadRwlockDestroy(&pTtlMgr->lock);
+  tdbOsFree(pTtlMgr);
+  return 0;
+}
+
+int ttlMgrBegin(STtlManger *pTtlMgr, void *pMeta) {
+  metaInfo("ttl mgr start open");
+  int ret;
+
+  int64_t startNs = taosGetTimestampNs();
+
+  SMeta *meta = (SMeta *)pMeta;
+
+  if (pTtlMgr->pOldTtlIdx) {
+    ret = ttlMgrConvert(pTtlMgr->pOldTtlIdx, pTtlMgr->pTtlIdx, pMeta);
+    if (ret < 0) {
+      metaError("failed to convert ttl index since %s", tstrerror(terrno));
+      goto _out;
+    }
+
+    ret = tdbTbDropByName(ttlTbname, meta->pEnv, meta->txn);
+    if (ret < 0) {
+      metaError("failed to drop old ttl index since %s", tstrerror(terrno));
+      goto _out;
+    }
+
+    tdbTbClose(pTtlMgr->pOldTtlIdx);
+    pTtlMgr->pOldTtlIdx = NULL;
+  }
+
+  ret = ttlMgrFillCache(pTtlMgr);
+  if (ret < 0) {
+    metaError("failed to fill hash since %s", tstrerror(terrno));
+    goto _out;
+  }
+
+  int64_t endNs = taosGetTimestampNs();
+
+  metaInfo("ttl mgr open end, hash size: %d, time consumed: %" PRId64 " ns", taosHashGetSize(pTtlMgr->pTtlCache),
+           endNs - startNs);
+_out:
+  return ret;
+}
+
+static void ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid) {
+  if (ttlDays <= 0) return;
+
+  pTtlKey->deleteTimeMs = changeTimeMs + ttlDays * tsTtlUnit * 1000;
+  pTtlKey->uid = uid;
+}
+
+static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+  STtlIdxKey *pTtlIdxKey1 = (STtlIdxKey *)pKey1;
+  STtlIdxKey *pTtlIdxKey2 = (STtlIdxKey *)pKey2;
+
+  if (pTtlIdxKey1->deleteTimeSec > pTtlIdxKey2->deleteTimeSec) {
+    return 1;
+  } else if (pTtlIdxKey1->deleteTimeSec < pTtlIdxKey2->deleteTimeSec) {
+    return -1;
+  }
+
+  if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) {
+    return 1;
+  } else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) {
+    return -1;
+  }
+
+  return 0;
+}
+
+static int ttlIdxKeyV1Cmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+  STtlIdxKeyV1 *pTtlIdxKey1 = (STtlIdxKeyV1 *)pKey1;
+  STtlIdxKeyV1 *pTtlIdxKey2 = (STtlIdxKeyV1 *)pKey2;
+
+  if (pTtlIdxKey1->deleteTimeMs > pTtlIdxKey2->deleteTimeMs) {
+    return 1;
+  } else if (pTtlIdxKey1->deleteTimeMs < pTtlIdxKey2->deleteTimeMs) {
+    return -1;
+  }
+
+  if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) {
+    return 1;
+  } else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) {
+    return -1;
+  }
+
+  return 0;
+}
+
+static int ttlMgrFillCache(STtlManger *pTtlMgr) {
+  return tdbTbTraversal(pTtlMgr->pTtlIdx, pTtlMgr->pTtlCache, ttlMgrFillCacheOneEntry);
+}
+
+static int32_t ttlMgrFillCacheOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pTtlCache) {
+  SHashObj *pCache = (SHashObj *)pTtlCache;
+
+  STtlIdxKeyV1 *ttlKey = (STtlIdxKeyV1 *)pKey;
+  tb_uid_t      uid = ttlKey->uid;
+  int64_t       ttlDays = *(int64_t *)pVal;
+  int64_t       changeTimeMs = ttlKey->deleteTimeMs - ttlDays * tsTtlUnit * 1000;
+
+  STtlCacheEntry data = {.ttlDays = ttlDays, .changeTimeMs = changeTimeMs};
+
+  return taosHashPut(pCache, &uid, sizeof(uid), &data, sizeof(data));
+}
+
+static int ttlMgrConvertOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pConvertData) {
+  SConvertData *pData = (SConvertData *)pConvertData;
+
+  STtlIdxKey *ttlKey = (STtlIdxKey *)pKey;
+  tb_uid_t    uid = ttlKey->uid;
+  int64_t     ttlDays = 0;
+
+  int ret = metaGetTableTtlByUid(pData->pMeta, uid, &ttlDays);
+  if (ret < 0) {
+    metaError("ttlMgr convert failed to get ttl since %s", tstrerror(terrno));
+    goto _out;
+  }
+
+  STtlIdxKeyV1 ttlKeyV1 = {.deleteTimeMs = ttlKey->deleteTimeSec * 1000, .uid = uid};
+  ret = tdbTbUpsert(pData->pNewTtlIdx, &ttlKeyV1, sizeof(ttlKeyV1), &ttlDays, sizeof(ttlDays), pData->pMeta->txn);
+  if (ret < 0) {
+    metaError("ttlMgr convert failed to upsert since %s", tstrerror(terrno));
+    goto _out;
+  }
+
+  ret = 0;
+_out:
+  return ret;
+}
+
+int ttlMgrConvert(TTB *pOldTtlIdx, TTB *pNewTtlIdx, void *pMeta) {
+  SMeta *meta = pMeta;
+
+  metaInfo("ttlMgr convert ttl start.");
+
+  SConvertData cvData = {.pNewTtlIdx = pNewTtlIdx, .pMeta = meta};
+
+  int ret = tdbTbTraversal(pOldTtlIdx, &cvData, ttlMgrConvertOneEntry);
+  if (ret < 0) {
+    metaError("failed to convert ttl since %s", tstrerror(terrno));
+  }
+
+  metaInfo("ttlMgr convert ttl end.");
+  return ret;
+}
+
+int ttlMgrInsertTtl(STtlManger *pTtlMgr, const STtlUpdTtlCtx *updCtx) {
+  if (updCtx->ttlDays == 0) return 0;
+
+  STtlCacheEntry cacheEntry = {.ttlDays = updCtx->ttlDays, .changeTimeMs = updCtx->changeTimeMs};
+  STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_UPSERT};
+
+  ttlMgrWLock(pTtlMgr);
+
+  int ret = taosHashPut(pTtlMgr->pTtlCache, &updCtx->uid, sizeof(updCtx->uid), &cacheEntry, sizeof(cacheEntry));
+  if (ret < 0) {
+    metaError("ttlMgr insert failed to update ttl cache since %s", tstrerror(terrno));
+    goto _out;
+  }
+
+  ret = taosHashPut(pTtlMgr->pDirtyUids, &updCtx->uid, sizeof(updCtx->uid), &dirtryEntry, sizeof(dirtryEntry));
+  if (ret < 0) {
+    metaError("ttlMgr insert failed to update ttl dirty uids since %s", tstrerror(terrno));
+    goto _out;
+  }
+
+  ret = 0;
+_out:
+  ttlMgrULock(pTtlMgr);
+
+  metaDebug("ttl mgr insert ttl, uid: %" PRId64 ", ctime: %" PRId64 ", ttlDays: %" PRId64, updCtx->uid,
+            updCtx->changeTimeMs, updCtx->ttlDays);
+
+  return ret;
+}
+
+int ttlMgrDeleteTtl(STtlManger *pTtlMgr, const STtlDelTtlCtx *delCtx) {
+  ttlMgrWLock(pTtlMgr);
+
+  STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_DEL};
+
+  int ret = taosHashPut(pTtlMgr->pDirtyUids, &delCtx->uid, sizeof(delCtx->uid), &dirtryEntry, sizeof(dirtryEntry));
+  if (ret < 0) {
+    metaError("ttlMgr del failed to update ttl dirty uids since %s", tstrerror(terrno));
+    goto _out;
+  }
+
+  ret = 0;
+_out:
+  ttlMgrULock(pTtlMgr);
+
+  metaDebug("ttl mgr delete ttl, uid: %" PRId64, delCtx->uid);
+
+  return ret;
+}
+
+int ttlMgrUpdateChangeTime(STtlManger *pTtlMgr, const STtlUpdCtimeCtx *pUpdCtimeCtx) {
+  ttlMgrWLock(pTtlMgr);
+
+  STtlCacheEntry *oldData = taosHashGet(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid));
+  if (oldData == NULL) {
+    goto _out;
+  }
+
+  STtlCacheEntry cacheEntry = {.ttlDays = oldData->ttlDays, .changeTimeMs = pUpdCtimeCtx->changeTimeMs};
+  STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_UPSERT};
+
+  int ret =
+      taosHashPut(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &cacheEntry, sizeof(cacheEntry));
+  if (ret < 0) {
+    metaError("ttlMgr update ctime failed to update ttl cache since %s", tstrerror(terrno));
+    goto _out;
+  }
+
+  ret = taosHashPut(pTtlMgr->pDirtyUids, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &dirtryEntry,
+                    sizeof(dirtryEntry));
+  if (ret < 0) {
+    metaError("ttlMgr update ctime failed to update ttl dirty uids since %s", tstrerror(terrno));
+    goto _out;
+  }
+
+  ret = 0;
+_out:
+  ttlMgrULock(pTtlMgr);
+
+  metaDebug("ttl mgr update ctime, uid: %" PRId64 ", ctime: %" PRId64, pUpdCtimeCtx->uid, pUpdCtimeCtx->changeTimeMs);
+
+  return ret;
+}
+
+int ttlMgrFindExpired(STtlManger *pTtlMgr, int64_t timePointMs, SArray *pTbUids) {
+  ttlMgrRLock(pTtlMgr);
+
+  TBC *pCur;
+  int  ret = tdbTbcOpen(pTtlMgr->pTtlIdx, &pCur, NULL);
+  if (ret < 0) {
+    goto _out;
+  }
+
+  STtlIdxKeyV1 ttlKey = {0};
+  ttlKey.deleteTimeMs = timePointMs;
+  ttlKey.uid = INT64_MAX;
+  int c = 0;
+  tdbTbcMoveTo(pCur, &ttlKey, sizeof(ttlKey), &c);
+  if (c < 0) {
+    tdbTbcMoveToPrev(pCur);
+  }
+
+  void *pKey = NULL;
+  int   kLen = 0;
+  while (1) {
+    ret = tdbTbcPrev(pCur, &pKey, &kLen, NULL, NULL);
+    if (ret < 0) {
+      ret = 0;
+      break;
+    }
+    ttlKey = *(STtlIdxKeyV1 *)pKey;
+    taosArrayPush(pTbUids, &ttlKey.uid);
+  }
+
+  tdbFree(pKey);
+  tdbTbcClose(pCur);
+
+  ret = 0;
+_out:
+  ttlMgrULock(pTtlMgr);
+  return ret;
+}
+
+int ttlMgrFlush(STtlManger *pTtlMgr, TXN *pTxn) {
+  ttlMgrWLock(pTtlMgr);
+
+  metaInfo("ttl mgr flush start.");
+
+  int ret = -1;
+
+  void *pIter = taosHashIterate(pTtlMgr->pDirtyUids, NULL);
+  while (pIter != NULL) {
+    STtlDirtyEntry *pEntry = (STtlDirtyEntry *)pIter;
+    tb_uid_t       *pUid = taosHashGetKey(pIter, NULL);
+
+    STtlCacheEntry *cacheEntry = taosHashGet(pTtlMgr->pTtlCache, pUid, sizeof(*pUid));
+    if (cacheEntry == NULL) {
+      metaError("ttlMgr flush failed to get ttl cache since %s", tstrerror(terrno));
+      goto _out;
+    }
+
+    STtlIdxKeyV1 ttlKey;
+    ttlMgrBuildKey(&ttlKey, cacheEntry->ttlDays, cacheEntry->changeTimeMs, *pUid);
+
+    if (pEntry->type == ENTRY_TYPE_UPSERT) {
+      ret = tdbTbUpsert(pTtlMgr->pTtlIdx, &ttlKey, sizeof(ttlKey), &cacheEntry->ttlDays, sizeof(cacheEntry->ttlDays),
+                        pTxn);
+      if (ret < 0) {
+        metaError("ttlMgr flush failed to flush ttl cache upsert since %s", tstrerror(terrno));
+        goto _out;
+      }
+    } else if (pEntry->type == ENTRY_TYPE_DEL) {
+      ret = tdbTbDelete(pTtlMgr->pTtlIdx, &ttlKey, sizeof(ttlKey), pTxn);
+      if (ret < 0) {
+        metaError("ttlMgr flush failed to flush ttl cache del since %s", tstrerror(terrno));
+        goto _out;
+      }
+
+      ret = taosHashRemove(pTtlMgr->pTtlCache, pUid, sizeof(*pUid));
+      if (ret < 0) {
+        metaError("ttlMgr flush failed to delete ttl cache since %s", tstrerror(terrno));
+        goto _out;
+      }
+    } else {
+      metaError("ttlMgr flush failed to flush ttl cache, unknown type: %d", pEntry->type);
+      goto _out;
+    }
+
+    pIter = taosHashIterate(pTtlMgr->pDirtyUids, pIter);
+  }
+
+  taosHashClear(pTtlMgr->pDirtyUids);
+
+  ret = 0;
+_out:
+  ttlMgrULock(pTtlMgr);
+
+  metaInfo("ttl mgr flush end.");
+
+  return ret;
+}
+
+static int32_t ttlMgrRLock(STtlManger *pTtlMgr) {
+  int32_t ret = 0;
+
+  metaTrace("ttlMgr rlock %p", &pTtlMgr->lock);
+
+  ret = taosThreadRwlockRdlock(&pTtlMgr->lock);
+
+  return ret;
+}
+
+static int32_t ttlMgrWLock(STtlManger *pTtlMgr) {
+  int32_t ret = 0;
+
+  metaTrace("ttlMgr wlock %p", &pTtlMgr->lock);
+
+  ret = taosThreadRwlockWrlock(&pTtlMgr->lock);
+
+  return ret;
+}
+
+static int32_t ttlMgrULock(STtlManger *pTtlMgr) {
+  int32_t ret = 0;
+
+  metaTrace("ttlMgr ulock %p", &pTtlMgr->lock);
+
+  ret = taosThreadRwlockUnlock(&pTtlMgr->lock);
+
+  return ret;
+}

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -136,14 +136,13 @@ static int32_t vnodeVgroupIdLen(int32_t vgId) {
 }
 
 int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs) {
-  int32_t ret = tfsRename(pTfs, srcPath, dstPath);
+  int32_t ret = 0;
-  if (ret != 0) return ret;
 
   char oldRname[TSDB_FILENAME_LEN] = {0};
   char newRname[TSDB_FILENAME_LEN] = {0};
   char tsdbPath[TSDB_FILENAME_LEN] = {0};
   char tsdbFilePrefix[TSDB_FILENAME_LEN] = {0};
-  snprintf(tsdbPath, TSDB_FILENAME_LEN, "%s%stsdb", dstPath, TD_DIRSEP);
+  snprintf(tsdbPath, TSDB_FILENAME_LEN, "%s%stsdb", srcPath, TD_DIRSEP);
   snprintf(tsdbFilePrefix, TSDB_FILENAME_LEN, "tsdb%sv", TD_DIRSEP);
 
   STfsDir *tsdbDir = tfsOpendir(pTfs, tsdbPath);
@@ -168,7 +167,7 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr
 
       ret = tfsRename(pTfs, tsdbFile->rname, newRname);
       if (ret != 0) {
-        vInfo("vgId:%d, failed to rename file from %s to %s since %s", dstVgId, tsdbFile->rname, newRname, terrstr());
+        vError("vgId:%d, failed to rename file from %s to %s since %s", dstVgId, tsdbFile->rname, newRname, terrstr());
         tfsClosedir(tsdbDir);
         return ret;
       }
@@ -176,6 +175,21 @@ int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t sr
   }
 
   tfsClosedir(tsdbDir);
+
+  vInfo("vgId:%d, rename dir from %s to %s", dstVgId, srcPath, dstPath);
+  ret = tfsRename(pTfs, srcPath, dstPath);
+  if (ret != 0) {
+    vError("vgId:%d, failed to rename dir from %s to %s since %s", dstVgId, srcPath, dstPath, terrstr());
+  }
+  return ret;
+}
+
+int32_t vnodeGetAbsDir(const char *relPath, STfs *pTfs, char *buf, size_t bufLen) {
+  if (pTfs) {
+    snprintf(buf, bufLen, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, relPath);
+  } else {
+    snprintf(buf, bufLen, "%s", relPath);
+  }
   return 0;
 }
 
@@ -184,13 +198,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod
   char       dir[TSDB_FILENAME_LEN] = {0};
   int32_t    ret = 0;
 
-  if (pTfs) {
+  vnodeGetAbsDir(srcPath, pTfs, dir, TSDB_FILENAME_LEN);
-    snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, srcPath);
-  } else {
-    snprintf(dir, TSDB_FILENAME_LEN, "%s", srcPath);
-  }
-
-  // todo add stat file to handle exception while vnode open
 
   ret = vnodeLoadInfo(dir, &info);
   if (ret < 0) {
@@ -245,6 +253,42 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod
   return 0;
 }
 
+int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs) {
+  SVnodeInfo info = {0};
+  char       dir[TSDB_FILENAME_LEN] = {0};
+
+  vnodeGetAbsDir(dstPath, pTfs, dir, TSDB_FILENAME_LEN);
+  if (vnodeLoadInfo(dir, &info) == 0) {
+    if (info.config.vgId != dstVgId) {
+      vError("vgId:%d, unexpected vnode config.vgId:%d", dstVgId, info.config.vgId);
+      return -1;
+    }
+    return dstVgId;
+  }
+
+  vnodeGetAbsDir(srcPath, pTfs, dir, TSDB_FILENAME_LEN);
+  if (vnodeLoadInfo(dir, &info) < 0) {
+    vError("vgId:%d, failed to read vnode config from %s since %s", srcVgId, srcPath, tstrerror(terrno));
+    return -1;
+  }
+
+  if (info.config.vgId == srcVgId) {
+    vInfo("vgId:%d, rollback alter hashrange", srcVgId);
+    return srcVgId;
+  } else if (info.config.vgId != dstVgId) {
+    vError("vgId:%d, unexpected vnode config.vgId:%d", dstVgId, info.config.vgId);
+    return -1;
+  }
+
+  vInfo("vgId:%d, rename %s to %s", dstVgId, srcPath, dstPath);
+  if (vnodeRenameVgroupId(srcPath, dstPath, srcVgId, dstVgId, pTfs) < 0) {
+    vError("vgId:%d, failed to rename vnode from %s to %s since %s", dstVgId, srcPath, dstPath, tstrerror(terrno));
+    return -1;
+  }
+
+  return dstVgId;
+}
+
 void vnodeDestroy(const char *path, STfs *pTfs) {
   vInfo("path:%s is removed while destroy vnode", path);
   tfsRmdir(pTfs, path);

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -37,7 +37,7 @@ static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pRe
 static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 
-static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t ctime, int64_t *pUid) {
+static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t btime, int64_t *pUid) {
   int32_t code = 0;
   int32_t lino = 0;
 
@@ -66,8 +66,8 @@ static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, i
   }
   *(int64_t *)(pCoder->data + pCoder->pos) = uid;
 
-  // ctime
+  // btime
-  *(int64_t *)(pCoder->data + pCoder->pos + 8) = ctime;
+  *(int64_t *)(pCoder->data + pCoder->pos + 8) = btime;
 
   tEndDecode(pCoder);
 
@@ -84,7 +84,7 @@ static int32_t vnodePreProcessCreateTableMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   int32_t code = 0;
   int32_t lino = 0;
 
-  int64_t  ctime = taosGetTimestampMs();
+  int64_t  btime = taosGetTimestampMs();
   SDecoder dc = {0};
   int32_t  nReqs;
 
@@ -99,7 +99,7 @@ static int32_t vnodePreProcessCreateTableMsg(SVnode *pVnode, SRpcMsg *pMsg) {
     TSDB_CHECK_CODE(code, lino, _exit);
   }
   for (int32_t iReq = 0; iReq < nReqs; iReq++) {
-    code = vnodePreprocessCreateTableReq(pVnode, &dc, ctime, NULL);
+    code = vnodePreprocessCreateTableReq(pVnode, &dc, btime, NULL);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
@@ -109,8 +109,35 @@ _exit:
   tDecoderClear(&dc);
   return code;
 }
+
+static int32_t vnodePreProcessAlterTableMsg(SVnode *pVnode, SRpcMsg *pMsg) {
+  int32_t code = TSDB_CODE_INVALID_MSG;
+  int32_t lino = 0;
+
+  SDecoder dc = {0};
+  tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead));
+
+  SVAlterTbReq vAlterTbReq = {0};
+  int64_t      ctimeMs = taosGetTimestampMs();
+  if (tDecodeSVAlterTbReqSetCtime(&dc, &vAlterTbReq, ctimeMs) < 0) {
+    goto _exit;
+  }
+
+  code = 0;
+
+_exit:
+  tDecoderClear(&dc);
+  if (code) {
+    vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
+  } else {
+    vTrace("vgId:%d %s done, table:%s ctimeMs generated:%" PRId64, TD_VID(pVnode), __func__, vAlterTbReq.tbName,
+           ctimeMs);
+  }
+  return code;
+}
+
 extern int64_t tsMaxKeyByPrecision[];
-static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int64_t ctime) {
+static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int64_t btimeMs, int64_t ctimeMs) {
   int32_t code = 0;
   int32_t lino = 0;
 
@@ -127,7 +154,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int
 
   int64_t uid;
   if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) {
-    code = vnodePreprocessCreateTableReq(pVnode, pCoder, ctime, &uid);
+    code = vnodePreprocessCreateTableReq(pVnode, pCoder, btimeMs, &uid);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
@@ -153,7 +180,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int
   }
 
   // scan and check
-  TSKEY now = ctime;
+  TSKEY now = btimeMs;
   if (pVnode->config.tsdbCfg.precision == TSDB_TIME_PRECISION_MICRO) {
     now *= 1000;
   } else if (pVnode->config.tsdbCfg.precision == TSDB_TIME_PRECISION_NANO) {
@@ -170,7 +197,6 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int
 
     SColData colData = {0};
     pCoder->pos += tGetColData(pCoder->data + pCoder->pos, &colData);
-
     if (colData.flag != HAS_VALUE) {
       code = TSDB_CODE_INVALID_MSG;
       goto _exit;
@@ -182,6 +208,10 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int
         goto _exit;
       }
     }
+
+    for (uint64_t i = 1; i < nColData; i++) {
+      pCoder->pos += tGetColData(pCoder->data + pCoder->pos, &colData);
+    }
   } else {
     uint64_t nRow;
     if (tDecodeU64v(pCoder, &nRow) < 0) {
@@ -200,6 +230,9 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int
     }
   }
 
+  *(int64_t *)(pCoder->data + pCoder->pos) = ctimeMs;
+  pCoder->pos += sizeof(int64_t);
+
   tEndDecode(pCoder);
 
 _exit:
@@ -229,9 +262,10 @@ static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) {
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  int64_t ctime = taosGetTimestampMs();
+  int64_t btimeMs = taosGetTimestampMs();
+  int64_t ctimeMs = btimeMs;
   for (int32_t i = 0; i < nSubmitTbData; i++) {
-    code = vnodePreProcessSubmitTbData(pVnode, pCoder, ctime);
+    code = vnodePreProcessSubmitTbData(pVnode, pCoder, btimeMs, ctimeMs);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
@@ -261,6 +295,7 @@ static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res);
   if (code) goto _exit;
 
+  res.ctimeMs = taosGetTimestampMs();
   // malloc and encode
   tEncodeSize(tEncodeDeleteRes, &res, size, ret);
   pCont = rpcMallocCont(size + sizeof(SMsgHead));
@@ -282,6 +317,31 @@ _exit:
   return code;
 }
 
+static int32_t vnodePreProcessBatchDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  int64_t         ctimeMs = taosGetTimestampMs();
+  SBatchDeleteReq pReq = {0};
+  SDecoder       *pCoder = &(SDecoder){0};
+
+  tDecoderInit(pCoder, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead));
+
+  if (tDecodeSBatchDeleteReqSetCtime(pCoder, &pReq, ctimeMs) < 0) {
+    code = TSDB_CODE_INVALID_MSG;
+  }
+
+  tDecoderClear(pCoder);
+  taosArrayDestroy(pReq.deleteReqs);
+
+  if (code) {
+    vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
+  } else {
+    vTrace("vgId:%d %s done, ctimeMs generated:%" PRId64, TD_VID(pVnode), __func__, ctimeMs);
+  }
+  return code;
+}
+
 int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   int32_t code = 0;
 
@@ -289,12 +349,18 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
     case TDMT_VND_CREATE_TABLE: {
       code = vnodePreProcessCreateTableMsg(pVnode, pMsg);
     } break;
+    case TDMT_VND_ALTER_TABLE: {
+      code = vnodePreProcessAlterTableMsg(pVnode, pMsg);
+    } break;
     case TDMT_VND_SUBMIT: {
       code = vnodePreProcessSubmitMsg(pVnode, pMsg);
     } break;
     case TDMT_VND_DELETE: {
       code = vnodePreProcessDeleteMsg(pVnode, pMsg);
     } break;
+    case TDMT_VND_BATCH_DEL: {
+      code = vnodePreProcessBatchDeleteMsg(pVnode, pMsg);
+    } break;
     default:
       break;
   }
@@ -642,8 +708,8 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq,
     goto end;
   }
 
-  vDebug("vgId:%d, drop ttl table req will be processed, time:%d", pVnode->config.vgId, ttlReq.timestamp);
+  vDebug("vgId:%d, drop ttl table req will be processed, time:%" PRId32, pVnode->config.vgId, ttlReq.timestampSec);
-  int32_t ret = metaTtlDropTable(pVnode->pMeta, ttlReq.timestamp, tbUids);
+  int32_t ret = metaTtlDropTable(pVnode->pMeta, (int64_t)ttlReq.timestampSec * 1000, tbUids);
   if (ret != 0) {
     goto end;
   }
@@ -651,7 +717,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq,
     tqUpdateTbUidList(pVnode->pTq, tbUids, false);
   }
 
-  vnodeSyncRetention(pVnode, ttlReq.timestamp);
+  vnodeAsyncRentention(pVnode, ttlReq.timestampSec);
 
 end:
   taosArrayDestroy(tbUids);
@@ -1387,6 +1453,9 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in
     code = tsdbInsertTableData(pVnode->pTsdb, ver, pSubmitTbData, &affectedRows);
     if (code) goto _exit;
 
+    code = metaUpdateChangeTime(pVnode->pMeta, pSubmitTbData->uid, pSubmitTbData->ctimeMs);
+    if (code) goto _exit;
+
     pSubmitRsp->affectedRows += affectedRows;
   }
 
@@ -1641,6 +1710,14 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pRe
              TD_VID(pVnode), terrstr(), deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs);
     }
 
+    code = metaUpdateChangeTime(pVnode->pMeta, uid, deleteReq.ctimeMs);
+    if (code < 0) {
+      terrno = code;
+      vError("vgId:%d, update change time error since %s, suid:%" PRId64 ", uid:%" PRId64 ", start ts:%" PRId64
+             ", end ts:%" PRId64,
+             TD_VID(pVnode), terrstr(), deleteReq.suid, uid, pOneReq->startTs, pOneReq->endTs);
+    }
+
     tDecoderClear(&mr.coder);
   }
   metaReaderClear(&mr);
@@ -1669,8 +1746,10 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, in
   ASSERT(taosArrayGetSize(pRes->uidList) == 0 || (pRes->skey != 0 && pRes->ekey != 0));
 
   for (int32_t iUid = 0; iUid < taosArrayGetSize(pRes->uidList); iUid++) {
-    code = tsdbDeleteTableData(pVnode->pTsdb, ver, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid),
+    uint64_t uid = *(uint64_t *)taosArrayGet(pRes->uidList, iUid);
-                               pRes->skey, pRes->ekey);
+    code = tsdbDeleteTableData(pVnode->pTsdb, ver, pRes->suid, uid, pRes->skey, pRes->ekey);
+    if (code) goto _err;
+    code = metaUpdateChangeTime(pVnode->pMeta, uid, pRes->ctimeMs);
     if (code) goto _err;
   }
 

source/libs/executor/src/sysscanoperator.c

@@ -1154,7 +1154,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
     int32_t tableType = mr.me.type;
     if (tableType == TSDB_CHILD_TABLE) {
       // create time
-      int64_t ts = mr.me.ctbEntry.ctime;
+      int64_t ts = mr.me.ctbEntry.btime;
       pColInfoData = taosArrayGet(p->pDataBlock, 2);
       colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false);
 
@@ -1206,7 +1206,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
     } else if (tableType == TSDB_NORMAL_TABLE) {
       // create time
       pColInfoData = taosArrayGet(p->pDataBlock, 2);
-      colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false);
+      colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.btime, false);
 
       // number of columns
       pColInfoData = taosArrayGet(p->pDataBlock, 3);
@@ -1338,7 +1338,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
     int32_t tableType = pInfo->pCur->mr.me.type;
     if (tableType == TSDB_CHILD_TABLE) {
       // create time
-      int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime;
+      int64_t ts = pInfo->pCur->mr.me.ctbEntry.btime;
       pColInfoData = taosArrayGet(p->pDataBlock, 2);
       colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false);
 
@@ -1392,7 +1392,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
     } else if (tableType == TSDB_NORMAL_TABLE) {
       // create time
       pColInfoData = taosArrayGet(p->pDataBlock, 2);
-      colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false);
+      colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.btime, false);
 
       // number of columns
       pColInfoData = taosArrayGet(p->pDataBlock, 3);

source/libs/parser/src/parTranslater.c

@@ -4965,6 +4965,14 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt
   if (TSDB_CODE_SUCCESS == code) {
     code = checkTableSchema(pCxt, pStmt);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    if (createStable && pStmt->pOptions->ttl != 0) {
+      code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TABLE_OPTION,
+                                     "Only supported for create non-super table in databases "
+                                     "configured with the 'TTL' option");
+    }
+  }
+
   return code;
 }
 

source/libs/parser/test/parInitialCTest.cpp

@@ -765,7 +765,7 @@ TEST_F(ParserInitialCTest, createStable) {
       "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, "
       "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, "
       "a12 TINYINT UNSIGNED, a13 BOOL, a14 NCHAR(30), a15 VARCHAR(50)) "
-      "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m "
+      "COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m "
       "DELETE_MARK 1000s,200m");
   clearCreateStbReq();
 }
@@ -1005,7 +1005,7 @@ TEST_F(ParserInitialCTest, createTable) {
         ASSERT_EQ(pReq->flags, pExpect->flags);
         ASSERT_EQ(std::string(pReq->name), std::string(pExpect->name));
         ASSERT_EQ(pReq->uid, pExpect->uid);
-        ASSERT_EQ(pReq->ctime, pExpect->ctime);
+        ASSERT_EQ(pReq->btime, pExpect->btime);
         ASSERT_EQ(pReq->ttl, pExpect->ttl);
         ASSERT_EQ(pReq->commentLen, pExpect->commentLen);
         ASSERT_EQ(std::string(pReq->comment), std::string(pExpect->comment));
@@ -1038,7 +1038,7 @@ TEST_F(ParserInitialCTest, createTable) {
       "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, a8 BINARY(20), "
       "a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, "
       "a14 NCHAR(30), a15 VARCHAR(50)) "
-      "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN)");
+      "COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN)");
 
   run("CREATE TABLE IF NOT EXISTS t1 USING st1 TAGS(1, 'wxy', NOW)");
 

source/libs/planner/src/planOptimizer.c

@@ -167,25 +167,25 @@ static bool scanPathOptMayBeOptimized(SLogicNode* pNode) {
   if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) {
     return false;
   }
-  if (NULL == pNode->pParent || (QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) &&
+  return true;
-                                 QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) &&
+}
-                                 QUERY_NODE_LOGIC_PLAN_PARTITION != nodeType(pNode->pParent))) {
+
-    return false;
+static bool scanPathOptShouldGetFuncs(SLogicNode* pNode) {
-  }
+  if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode) &&
-  if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) &&
+       WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode)->winType) ||
-       WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType) ||
+      (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) && pNode->pParent &&
-      (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode->pParent) && pNode->pParent->pParent &&
+       QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent) &&
-       QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent->pParent) &&
        WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType)) {
     return true;
   }
-  if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode->pParent)) {
+  if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode)) {
-    return !scanPathOptHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys);
+    return !scanPathOptHaveNormalCol(((SAggLogicNode*)pNode)->pGroupKeys);
   }
   return false;
 }
 
 static SNodeList* scanPathOptGetAllFuncs(SLogicNode* pNode) {
+  if (!scanPathOptShouldGetFuncs(pNode)) return NULL;
   switch (nodeType(pNode)) {
     case QUERY_NODE_LOGIC_PLAN_WINDOW:
       return ((SWindowLogicNode*)pNode)->pFuncs;

source/libs/qcom/src/queryUtil.c

@@ -532,7 +532,7 @@ int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) {
     (*pDst)->name = taosStrdup(pSrc->name);
   }
   (*pDst)->uid = pSrc->uid;
-  (*pDst)->ctime = pSrc->ctime;
+  (*pDst)->btime = pSrc->btime;
   (*pDst)->ttl = pSrc->ttl;
   (*pDst)->commentLen = pSrc->commentLen;
   if (pSrc->comment) {

source/libs/qworker/src/qworker.c

@@ -427,6 +427,7 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes
 
   pRes->suid = pDelRes->suid;
   pRes->uidList = pDelRes->uidList;
+  pRes->ctimeMs = taosGetTimestampMs();
   pRes->skey = pDelRes->skey;
   pRes->ekey = pDelRes->ekey;
   pRes->affectedRows = pDelRes->affectedRows;

source/libs/stream/CMakeLists.txt

@@ -9,6 +9,14 @@ target_include_directories(
 
 if(${BUILD_WITH_ROCKSDB})
   IF (TD_LINUX)
+  target_include_directories(
+    stream
+    PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static"
+  )
+  target_link_directories(
+    stream 
+    PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static"
+  )
   target_link_libraries(
     stream
     PUBLIC rocksdb tdb

source/libs/tdb/inc/tdb.h

@@ -46,12 +46,16 @@ int32_t tdbAlter(TDB *pDb, int pages);
 int32_t tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb,
                   int8_t rollback);
 int32_t tdbTbClose(TTB *pTb);
+bool    tdbTbExist(const char *tbname, TDB *pEnv);
+int     tdbTbDropByName(const char *tbname, TDB *pEnv, TXN* pTxn);
 int32_t tdbTbDrop(TTB *pTb);
 int32_t tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn);
 int32_t tdbTbDelete(TTB *pTb, const void *pKey, int kLen, TXN *pTxn);
 int32_t tdbTbUpsert(TTB *pTb, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
 int32_t tdbTbGet(TTB *pTb, const void *pKey, int kLen, void **ppVal, int *vLen);
 int32_t tdbTbPGet(TTB *pTb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
+int32_t tdbTbTraversal(TTB *pTb, void *data,
+                       int32_t (*func)(const void *pKey, int keyLen, const void *pVal, int valLen, void *data));
 
 // TBC
 int32_t tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn);

source/libs/tdb/src/db/tdbTable.c

@@ -134,11 +134,67 @@ int tdbTbClose(TTB *pTb) {
   return 0;
 }
 
+bool tdbTbExist(const char *tbname, TDB *pEnv) {
+  bool    exist = false;
+  SPager *pPager;
+  char    fFullName[TDB_FILENAME_LEN];
+
+#ifdef USE_MAINDB
+
+  snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, TDB_MAINDB_NAME);
+
+  if (strcmp(TDB_MAINDB_NAME, tbname)) {
+    pPager = tdbEnvGetPager(pEnv, fFullName);
+
+    exist = tdbTbGet(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, NULL, NULL) == 0;
+  } else {
+    exist = taosCheckExistFile(fFullName);
+  }
+
+#else
+
+  snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, tbname);
+
+  exist = taosCheckExistFile(fFullName);
+
+#endif
+
+  return exist;
+}
+
 int tdbTbDrop(TTB *pTb) {
   // TODO
   return 0;
 }
 
+int tdbTbDropByName(const char *tbname, TDB *pEnv, TXN *pTxn) {
+  int     ret;
+  SPager *pPager;
+  char    fFullName[TDB_FILENAME_LEN];
+
+#ifdef USE_MAINDB
+
+  snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, TDB_MAINDB_NAME);
+
+  if (strcmp(TDB_MAINDB_NAME, tbname)) {
+    pPager = tdbEnvGetPager(pEnv, fFullName);
+
+    ret = tdbTbDelete(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, pTxn);
+  } else {
+    ret = taosRemoveFile(fFullName);
+  }
+
+#else
+
+  snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, tbname);
+
+  ret = taosRemoveFile(fFullName);
+
+#endif
+
+  return ret;
+}
+
 int tdbTbInsert(TTB *pTb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) {
   return tdbBtreeInsert(pTb->pBt, pKey, keyLen, pVal, valLen, pTxn);
 }
@@ -173,6 +229,38 @@ int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn) {
   return 0;
 }
 
+int32_t tdbTbTraversal(TTB *pTb, void *data,
+                       int32_t (*func)(const void *pKey, int keyLen, const void *pVal, int valLen, void *data)) {
+  TBC *pCur;
+  int  ret = tdbTbcOpen(pTb, &pCur, NULL);
+  if (ret < 0) {
+    return ret;
+  }
+
+  tdbTbcMoveToFirst(pCur);
+
+  void *pKey = NULL;
+  int   kLen = 0;
+  void *pValue = NULL;
+  int   vLen = 0;
+
+  while (1) {
+    ret = tdbTbcNext(pCur, &pKey, &kLen, &pValue, &vLen);
+    if (ret < 0) {
+      ret = 0;
+      break;
+    }
+
+    ret = func(pKey, kLen, pValue, vLen, data);
+    if (ret < 0) break;
+  }
+  tdbFree(pKey);
+  tdbFree(pValue);
+  tdbTbcClose(pCur);
+
+  return 0;
+}
+
 int tdbTbcMoveTo(TBC *pTbc, const void *pKey, int kLen, int *c) { return tdbBtcMoveTo(&pTbc->btc, pKey, kLen, c); }
 
 int tdbTbcMoveToFirst(TBC *pTbc) { return tdbBtcMoveToFirst(&pTbc->btc); }

tests/parallel_test/cases.task

@@ -152,6 +152,7 @@
 ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py
 ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py
 ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py
+,,n,system-test,python3 ./test.py -f 0-others/splitVGroup.py -N 5
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py

tests/script/tsim/query/r/explain_tsorder.result

@@ -2000,7 +2000,7 @@ QUERY_PLAN:    -> Data Exchange 1:1 (width=12)
 *************************** 6.row ***************************
 QUERY_PLAN:          Output: columns=2 width=12
 *************************** 7.row ***************************
-QUERY_PLAN:       -> Sort input_order=unknown output_order=unknown  (columns=2 width=12)
+QUERY_PLAN:       -> Sort input_order=asc output_order=unknown  (columns=2 width=12)
 *************************** 8.row ***************************
 QUERY_PLAN:             Output: columns=2 width=12
 *************************** 9.row ***************************
@@ -2014,7 +2014,7 @@ QUERY_PLAN:    -> Data Exchange 1:1 (width=12)
 *************************** 13.row ***************************
 QUERY_PLAN:          Output: columns=2 width=12
 *************************** 14.row ***************************
-QUERY_PLAN:       -> Sort input_order=unknown output_order=unknown  (columns=2 width=12)
+QUERY_PLAN:       -> Sort input_order=asc output_order=unknown  (columns=2 width=12)
 *************************** 15.row ***************************
 QUERY_PLAN:             Output: columns=2 width=12
 *************************** 16.row ***************************
@@ -2074,7 +2074,7 @@ QUERY_PLAN:          -> Data Exchange 1:1 (width=12)
 *************************** 12.row ***************************
 QUERY_PLAN:                Output: columns=2 width=12
 *************************** 13.row ***************************
-QUERY_PLAN:             -> Sort input_order=unknown output_order=unknown  (columns=2 width=12)
+QUERY_PLAN:             -> Sort input_order=asc output_order=unknown  (columns=2 width=12)
 *************************** 14.row ***************************
 QUERY_PLAN:                   Output: columns=2 width=12
 *************************** 15.row ***************************
@@ -2088,7 +2088,7 @@ QUERY_PLAN:          -> Data Exchange 1:1 (width=12)
 *************************** 19.row ***************************
 QUERY_PLAN:                Output: columns=2 width=12
 *************************** 20.row ***************************
-QUERY_PLAN:             -> Sort input_order=unknown output_order=unknown  (columns=2 width=12)
+QUERY_PLAN:             -> Sort input_order=asc output_order=unknown  (columns=2 width=12)
 *************************** 21.row ***************************
 QUERY_PLAN:                   Output: columns=2 width=12
 *************************** 22.row ***************************

tests/system-test/0-others/splitVGroup.py

@@ -0,0 +1,377 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import random
+import time
+import copy
+import string
+
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+class TDTestCase:
+
+    # random string
+    def random_string(self, count):
+        letters = string.ascii_letters
+        return ''.join(random.choice(letters) for i in range(count))
+
+    # get col value and total max min ...
+    def getColsValue(self, i, j):
+        # c1 value
+        if random.randint(1, 10) == 5:
+            c1 = None
+        else:
+            c1 = 1
+
+        # c2 value
+        if j % 3200 == 0:
+            c2 = 8764231
+        elif random.randint(1, 10) == 5:
+            c2 = None
+        else:
+            c2 = random.randint(-87654297, 98765321)    
+
+
+        value = f"({self.ts}, "
+
+        # c1
+        if c1 is None:
+            value += "null,"
+        else:
+            self.c1Cnt += 1
+            value += f"{c1},"
+        # c2
+        if c2 is None:
+            value += "null,"
+        else:
+            value += f"{c2},"
+            # total count
+            self.c2Cnt += 1
+            # max
+            if self.c2Max is None:
+                self.c2Max = c2
+            else:
+                if c2 > self.c2Max:
+                    self.c2Max = c2
+            # min
+            if self.c2Min is None:
+                self.c2Min = c2
+            else:
+                if c2 < self.c2Min:
+                    self.c2Min = c2
+            # sum
+            if self.c2Sum is None:
+                self.c2Sum = c2
+            else:
+                self.c2Sum += c2
+
+        # c3 same with ts
+        value += f"{self.ts})"
+        
+        # move next
+        self.ts += 1
+
+        return value
+
+    # insert data
+    def insertData(self):
+        tdLog.info("insert data ....")
+        sqls = ""
+        for i in range(self.childCnt):
+            # insert child table
+            values = ""
+            pre_insert = f"insert into @db_name.t{i} values "
+            for j in range(self.childRow):
+                if values == "":
+                    values = self.getColsValue(i, j)
+                else:
+                    values += "," + self.getColsValue(i, j)
+
+                # batch insert    
+                if j % self.batchSize == 0  and values != "":
+                    sql = pre_insert + values
+                    self.exeDouble(sql)
+                    values = ""
+            # append last
+            if values != "":
+                sql = pre_insert + values
+                self.exeDouble(sql)
+                values = ""
+
+        # insert nomal talbe
+        for i in range(20):
+            self.ts += 1000
+            name = self.random_string(20)
+            sql = f"insert into @db_name.ta values({self.ts}, {i}, {self.ts%100000}, '{name}', false)"
+            self.exeDouble(sql)
+
+        # insert finished
+        tdLog.info(f"insert data successfully.\n"
+        f"                            inserted child table = {self.childCnt}\n"
+        f"                            inserted child rows  = {self.childRow}\n"
+        f"                            total inserted rows  = {self.childCnt*self.childRow}\n")
+        return
+    
+    def exeDouble(self, sql):
+        # dbname replace
+        sql1 = sql.replace("@db_name", self.db1)
+
+        if len(sql1) > 100:
+            tdLog.info(sql1[:100])
+        else:
+            tdLog.info(sql1)
+        tdSql.execute(sql1)
+
+        sql2 = sql.replace("@db_name", self.db2)
+        if len(sql1) > 100:
+            tdLog.info(sql1[:100])
+        else:
+            tdLog.info(sql1)
+        tdSql.execute(sql2)
+        
+
+    # prepareEnv
+    def prepareEnv(self):
+        # init                
+        self.ts = 1680000000000
+        self.childCnt = 10
+        self.childRow = 10000
+        self.batchSize = 5000
+        self.vgroups1  = 4
+        self.vgroups2  = 4
+        self.db1 = "db1"
+        self.db2 = "db2"
+        
+        # total
+        self.c1Cnt = 0
+        self.c2Cnt = 0
+        self.c2Max = None
+        self.c2Min = None
+        self.c2Sum = None
+
+        # create database  db
+        sql = f"create database @db_name vgroups {self.vgroups1} replica 3"
+        self.exeDouble(sql)
+
+        # create super talbe st
+        sql = f"create table @db_name.st(ts timestamp, c1 int, c2 bigint, ts1 timestamp) tags(area int)"
+        self.exeDouble(sql)
+
+        # create child table
+        for i in range(self.childCnt):
+            sql = f"create table @db_name.t{i} using @db_name.st tags({i}) "
+            self.exeDouble(sql)
+
+        # create normal table
+        sql = f"create table @db_name.ta(ts timestamp, c1 int, c2 bigint, c3 binary(32), c4 bool)"
+        self.exeDouble(sql)
+
+        # insert data
+        self.insertData()
+
+    # check data correct
+    def checkExpect(self, sql, expectVal):
+        tdSql.query(sql)
+        rowCnt = tdSql.getRows()
+        for i in range(rowCnt):
+            val = tdSql.getData(i,0)
+            if val != expectVal:
+                tdLog.exit(f"Not expect . query={val} expect={expectVal} i={i} sql={sql}")
+                return False
+
+        tdLog.info(f"check expect ok. sql={sql} expect ={expectVal} rowCnt={rowCnt}")
+        return True
+
+    # init
+    def init(self, conn, logSql, replicaVar=1):
+        seed = time.clock_gettime(time.CLOCK_REALTIME)
+        random.seed(seed)
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), True)
+
+    # check query result same
+    def queryDouble(self, sql):
+        # sql
+        sql1 = sql.replace('@db_name', self.db1)
+        tdLog.info(sql1)
+        start1 = time.time()
+        rows1 = tdSql.query(sql1)
+        spend1 = time.time() - start1
+        res1 = copy.copy(tdSql.queryResult)
+
+        sql2 = sql.replace('@db_name', self.db2)
+        tdLog.info(sql2)
+        start2 = time.time()
+        tdSql.query(sql2)
+        spend2 = time.time() - start2
+        res2 = tdSql.queryResult
+
+        rowlen1 = len(res1)
+        rowlen2 = len(res2)
+
+        if rowlen1 != rowlen2:
+            tdLog.exit(f"rowlen1={rowlen1} rowlen2={rowlen2} both not equal.")
+            return False
+        
+        for i in range(rowlen1):
+            row1 = res1[i]
+            row2 = res2[i]
+            collen1 = len(row1)
+            collen2 = len(row2)
+            if collen1 != collen2:
+                tdLog.exit(f"collen1={collen1} collen2={collen2} both not equal.")
+                return False
+            for j in range(collen1):
+                if row1[j] != row2[j]:
+                    tdLog.exit(f"col={j} col1={row1[j]} col2={row2[j]} both col not equal.")
+                    return False
+
+        # warning performance
+        diff = (spend2 - spend1)*100/spend1
+        tdLog.info("spend1=%.6fs spend2=%.6fs diff=%.1f%%"%(spend1, spend2, diff))
+        if spend2 > spend1 and diff > 20:
+            tdLog.info("warning: the diff for performance after spliting is over 20%")
+
+        return True
+
+
+    # check result
+    def checkResult(self):
+        # check vgroupid
+        sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{self.db2}'"
+        tdSql.query(sql)
+        tdSql.checkRows(self.vgroups2)
+
+        # check child table count same
+        sql = "select table_name from information_schema.ins_tables where db_name='@db_name' order by table_name"
+        self.queryDouble(sql)
+
+        # check row value is ok
+        sql = "select * from @db_name.st order by ts"
+        self.queryDouble(sql)
+        
+        # where
+        sql = "select *,tbname from @db_name.st where c1 < 1000 order by ts"
+        self.queryDouble(sql)
+
+        # max
+        sql = "select max(c1) from @db_name.st"
+        self.queryDouble(sql)
+
+        # min
+        sql = "select min(c2) from @db_name.st"
+        self.queryDouble(sql)
+
+        # sum
+        sql = "select sum(c1) from @db_name.st"
+        self.queryDouble(sql)
+
+        # normal table
+
+        # all rows
+        sql = "select * from @db_name.ta"
+        self.queryDouble(sql)
+
+        # count
+        sql = "select count(*) from @db_name.ta"
+        self.queryDouble(sql)
+
+        # sum
+        sql = "select sum(c1) from @db_name.ta"
+        self.queryDouble(sql)
+
+
+    # get vgroup list
+    def getVGroup(self, db_name):
+        vgidList = []
+        sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'"
+        res = tdSql.getResult(sql)
+        rows = len(res)
+        for i in range(rows):
+            vgidList.append(res[i][0])
+
+        return vgidList;        
+
+    # split vgroup on db2
+    def splitVGroup(self, db_name):
+        vgids = self.getVGroup(db_name)
+        selid = random.choice(vgids)
+        sql = f"split vgroup {selid}"
+        tdLog.info(sql)
+        tdSql.execute(sql)
+
+        # wait end
+        for i in range(100):
+            sql ="show transactions;"
+            rows = tdSql.query(sql)
+            if rows == 0:
+                tdLog.info("split vgroup finished.")
+                return True
+            #tdLog.info(f"i={i} wait split vgroup ...")
+            time.sleep(1)
+
+        tdLog.exit("split vgroup transaction is not finished after executing 50s")
+        return False
+
+    # split empty database
+    def splitEmptyDB(self):
+        
+        dbName = "emptydb"
+        vgNum = 2
+        # create database
+        sql = f"create database {dbName} vgroups {vgNum}"
+        tdLog.info(sql)
+        tdSql.execute(sql)
+
+        # split vgroup
+        self.splitVGroup(dbName)
+        vgList = self.getVGroup(dbName)
+        vgNum1 = len(vgList)
+        vgNum2 = vgNum + 1
+        if vgNum1 != vgNum2:
+            tdLog.exit(f" vglist len={vgNum1} is not same for expect {vgNum2}")
+            return
+
+    # run
+    def run(self):
+        
+        # prepare env
+        self.prepareEnv()
+
+        for i in range(5):
+            # split vgroup on db2
+            self.splitVGroup(self.db2)
+            self.vgroups2 += 1
+
+            # check two db query result same
+            self.checkResult()
+
+            tdLog.info(f"split vgroup i={i} passed.")
+
+        # split empty db
+        self.splitEmptyDB()    
+
+
+    # stop
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/1-insert/table_param_ttl.py

@@ -35,14 +35,14 @@ class TDTestCase:
             tdSql.execute(f'create table db.{self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.ttl_param}')
         tdSql.query(f'show db.tables')
         tdSql.checkRows(self.tbnum)
-        sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'])
+        sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'] + 1)
         tdSql.query(f'show db.tables')
         tdSql.checkRows(0)
         for i in range(self.tbnum):
             tdSql.execute(f'create table db.{self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.default_ttl}')
         for i in range(int(self.tbnum/2)):
             tdSql.execute(f'alter table db.{self.ntbname}_{i} ttl {self.modify_ttl}')
-        sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval'])
+        sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval'] + 1)
         tdSql.query(f'show db.tables')
         tdSql.checkRows(self.tbnum - int(self.tbnum/2))
         tdSql.execute('drop database db')
@@ -54,7 +54,7 @@ class TDTestCase:
             tdSql.execute(f'create table db.{self.stbname}_{i} using db.{self.stbname} tags({i}) ttl {self.ttl_param}')
         tdSql.query(f'show db.tables')
         tdSql.checkRows(self.tbnum)
-        sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'])
+        sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'] + 1)
         tdSql.query(f'show db.tables')
         tdSql.checkRows(0)
         for i in range(self.tbnum):
@@ -63,7 +63,7 @@ class TDTestCase:
         tdSql.checkRows(self.tbnum)
         for i in range(int(self.tbnum/2)):
             tdSql.execute(f'alter table db.{self.stbname}_{i} ttl {self.modify_ttl}')
-        sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval'])
+        sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval'] + 1)
         tdSql.query(f'show db.tables')
         tdSql.checkRows(self.tbnum - int(self.tbnum/2))
         tdSql.execute('drop database db')
@@ -75,7 +75,7 @@ class TDTestCase:
             tdSql.execute(f'insert into db.{self.stbname}_{i} using db.{self.stbname} tags({i}) ttl {self.ttl_param} values(now,1)')
         tdSql.query(f'show db.tables')
         tdSql.checkRows(self.tbnum)
-        sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'])
+        sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval'] + 1)
         tdSql.query(f'show db.tables')
         tdSql.checkRows(0)
         tdSql.execute('drop database db')

utils/test/c/write_raw_block_test.c

@@ -52,7 +52,7 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes) {
 
   pRes = taos_query(pConn, "create table d2 using meters tags(3, 'San Francisco')");
   if (taos_errno(pRes) != 0) {
-    printf("failed to create child table d1, reason:%s\n", taos_errstr(pRes));
+    printf("failed to create child table d2, reason:%s\n", taos_errstr(pRes));
     return -1;
   }
   taos_free_result(pRes);