Merge branch '3.0' into feature/3_liaohj

docs/en/12-taos-sql/10-function.md

@@ -532,7 +532,12 @@ TIMEDIFF(expr1, expr2 [, time_unit])
 #### TIMETRUNCATE
 
 ```sql
-TIMETRUNCATE(expr, time_unit)
+TIMETRUNCATE(expr, time_unit [, ignore_timezone])
+
+ignore_timezone: {
+    0
+  | 1
+}
 ```
 
 **Description**: Truncate the input timestamp with unit specified by `time_unit`
@@ -548,7 +553,10 @@ TIMETRUNCATE(expr, time_unit)
           1b (nanoseconds), 1u (microseconds), 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), or 1w (weeks)
 - The precision of the returned timestamp is same as the precision set for the current data base in use
 - If the input data is not formatted as a timestamp, the returned value is null.
-
+- If `1d` is used as `time_unit` to truncate the timestamp, `ignore_timezone` option can be set to indicate if the returned result is affected by client timezone or not.
+  For example, if client timezone is set to UTC+0800, TIMETRUNCATE('2020-01-01 23:00:00', 1d, 0) will return '2020-01-01 08:00:00'.
+  Otherwise, TIMETRUNCATE('2020-01-01 23:00:00', 1d, 1) will return '2020-01-01 00:00:00'.
+  If `ignore_timezone` option is omitted, the default value is set to 1.
 
 #### TIMEZONE
 

docs/en/14-reference/03-connector/index.mdx

@@ -59,10 +59,10 @@ The different database framework specifications for various programming language
 | -------------------------------------- | ------------- | --------------- | ------------- | ------------- | ------------- | ------------- |
 | **Connection Management**              | Support       | Support         | Support       | Support       | Support       | Support       |
 | **Regular Query**                      | Support       | Support         | Support       | Support       | Support       | Support       |
-| **Parameter Binding**                  | Not supported | Not supported   | Not supported | Support       | Not supported | Support       |
+| **Parameter Binding**                  | Not supported | Not supported   | support       | Support       | Not supported | Support       |
-| **Subscription (TMQ) **                | Not supported | Not supported   | Not supported | Not supported | Not supported | Support       |
+| **Subscription (TMQ) **                | Not supported | Not supported   | support       | Not supported | Not supported | Support       |
 | **Schemaless**                         | Not supported | Not supported   | Not supported | Not supported | Not supported | Not supported |
-| **Bulk Pulling (based on WebSocket) ** | Support       | Support         | Not Supported | support       | Not Supported | Supported     |
+| **Bulk Pulling (based on WebSocket) ** | Support       | Support         | Support       | support       | Support       | Support       |
 | **DataFrame**                          | Not supported | Support         | Not supported | Not supported | Not supported | Not supported |
 
 :::warning

docs/examples/csharp/wsConnect/Program.cs

@@ -5,22 +5,24 @@ namespace Examples
 {
     public class WSConnExample
     {
-        static void Main(string[] args)
+        static int Main(string[] args)
         {
             string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
             IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
   
             if (wsConn == IntPtr.Zero)
             {
-                throw new Exception("get WS connection failed");
+                Console.WriteLine("get WS connection failed");
+                return -1;
             }
             else
             {
                 Console.WriteLine("Establish connect success.");
+                // close connection.
+                LibTaosWS.WSClose(wsConn);
             }
 
-            // close connection.
+            return 0;
-            LibTaosWS.WSClose(wsConn);
         }
     }
-}
+}

docs/examples/csharp/wsInsert/Program.cs

@@ -5,7 +5,7 @@ namespace Examples
 {
     public class WSInsertExample
     {
-        static void Main(string[] args)
+        static int Main(string[] args)
         {
             string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
             IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
@@ -13,7 +13,8 @@ namespace Examples
             // Assert if connection is validate
             if (wsConn == IntPtr.Zero)
             {
-                throw new Exception("get WS connection failed");
+                Console.WriteLine("get WS connection failed");
+                return -1;
             }
             else
             {
@@ -36,6 +37,8 @@ namespace Examples
 
             // close connection.
             LibTaosWS.WSClose(wsConn);
+
+            return 0;
         }
 
         static void ValidInsert(string desc, IntPtr wsRes)
@@ -43,7 +46,7 @@ namespace Examples
             int code = LibTaosWS.WSErrorNo(wsRes);
             if (code != 0)
             {
-                throw new Exception($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
+                Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
             }
             else
             {
@@ -55,4 +58,4 @@ namespace Examples
 }
 // Establish connect success.
 // create table success affect 0 rows, cost 3717542 nanoseconds
-// insert data success affect 8 rows, cost 2613637 nanoseconds
+// insert data success affect 8 rows, cost 2613637 nanoseconds

docs/examples/csharp/wsQuery/Program.cs

@@ -7,13 +7,14 @@ namespace Examples
 {
     public class WSQueryExample
     {
-        static void Main(string[] args)
+        static int Main(string[] args)
         {
             string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
             IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
             if (wsConn == IntPtr.Zero)
             {
-                throw new Exception("get WS connection failed");
+                Console.WriteLine("get WS connection failed");
+                return -1;
             }
             else
             {
@@ -28,7 +29,9 @@ namespace Examples
             int code = LibTaosWS.WSErrorNo(wsRes);
             if (code != 0)
             {
-                throw new Exception($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
+                Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
+                LibTaosWS.WSFreeResult(wsRes);
+                return -1;
             }
 
             // get meta data
@@ -58,6 +61,8 @@ namespace Examples
 
             // close connection.
             LibTaosWS.WSClose(wsConn);
+
+            return 0;
         }
     }
 }
@@ -71,4 +76,4 @@ namespace Examples
 // 1538548685000   |       10.3    |       219     |       0.31    |       California.SanFrancisco |       2       |
 // 1538548695000   |       12.6    |       218     |       0.33    |       California.SanFrancisco |       2       |
 // 1538548696800   |       12.3    |       221     |       0.31    |       California.SanFrancisco |       2       |
-// 1538548696650   |       10.3    |       218     |       0.25    |       California.SanFrancisco |       3       |
+// 1538548696650   |       10.3    |       218     |       0.25    |       California.SanFrancisco |       3       |

docs/examples/csharp/wsStmt/Program.cs

@@ -7,7 +7,7 @@ namespace Examples
 {
     public class WSStmtExample
     {
-        static void Main(string[] args)
+        static int Main(string[] args)
         {
             const string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
             const string table = "meters";
@@ -21,7 +21,8 @@ namespace Examples
             IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
             if (wsConn == IntPtr.Zero)
             {
-                throw new Exception($"get WS connection failed");
+                Console.WriteLine($"get WS connection failed");
+                return -1;
             }
             else
             {
@@ -66,18 +67,20 @@ namespace Examples
             }
             else
             {
-                throw new Exception("Init STMT failed...");
+                Console.WriteLine("Init STMT failed...");
             }
 
             // close connection.
             LibTaosWS.WSClose(wsConn);
+
+            return 0;
         }
 
         static void ValidStmtStep(int code, IntPtr wsStmt, string desc)
         {
             if (code != 0)
             {
-                throw new Exception($"{desc} failed,reason: {LibTaosWS.WSErrorStr(wsStmt)}, code: {code}");
+                Console.WriteLine($"{desc} failed,reason: {LibTaosWS.WSErrorStr(wsStmt)}, code: {code}");
             }
             else
             {
@@ -92,4 +95,4 @@ namespace Examples
 // WSStmtBindParamBatch success...
 // WSStmtAddBatch success...
 // WSStmtExecute success...
-// WS STMT insert 5 rows...
+// WS STMT insert 5 rows...

docs/zh/08-connector/index.md

@@ -59,10 +59,10 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器
 | ------------------------------ | -------- | ---------- | -------- | -------- | ----------- | -------- |
 | **连接管理**                   | 支持     | 支持       | 支持     | 支持     | 支持        | 支持     |
 | **普通查询**                   | 支持     | 支持       | 支持     | 支持     | 支持        | 支持     |
-| **参数绑定**                   | 暂不支持 | 暂不支持   | 暂不支持 | 支持     | 暂不支持    | 支持     |
+| **参数绑定**                   | 暂不支持 | 暂不支持   | 支持     | 支持     | 暂不支持    | 支持     |
-| **数据订阅（TMQ）**            | 暂不支持 | 暂不支持   | 暂不支持 | 暂不支持 | 暂不支持    | 支持     |
+| **数据订阅（TMQ）**            | 暂不支持 | 暂不支持   | 支持     | 暂不支持 | 暂不支持    | 支持     |
 | **Schemaless**                 | 暂不支持 | 暂不支持   | 暂不支持 | 暂不支持 | 暂不支持    | 暂不支持 |
-| **批量拉取（基于 WebSocket）** | 支持     | 支持       | 暂不支持 | 支持     | 暂不支持    | 支持     |
+| **批量拉取（基于 WebSocket）** | 支持     | 支持       | 支持     | 支持     | 支持        | 支持     |
 | **DataFrame**                  | 不支持   | 支持       | 不支持   | 不支持   | 不支持      | 不支持   |
 
 :::warning

docs/zh/12-taos-sql/10-function.md

@@ -533,7 +533,12 @@ TIMEDIFF(expr1, expr2 [, time_unit])
 #### TIMETRUNCATE
 
 ```sql
-TIMETRUNCATE(expr, time_unit)
+TIMETRUNCATE(expr, time_unit [, ignore_timezone])
+
+ignore_timezone: {
+    0
+  | 1
+}
 ```
 
 **功能说明**：将时间戳按照指定时间单位 time_unit 进行截断。
@@ -549,6 +554,11 @@ TIMETRUNCATE(expr, time_unit)
           1b(纳秒), 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天), 1w(周)。
 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。
 - 输入包含不符合时间日期格式的字符串则返回 NULL。
+- 当使用 1d 作为时间单位对时间戳进行截断时， 可通过设置 ignore_timezone 参数指定返回结果的显示是否忽略客户端时区的影响。
+  例如客户端所配置时区为 UTC+0800, 则 TIMETRUNCATE('2020-01-01 23:00:00', 1d, 0) 返回结果为 '2020-01-01 08:00:00'。
+  而使用 TIMETRUNCATE('2020-01-01 23:00:00', 1d, 1) 设置忽略时区时，返回结果为 '2020-01-01 00:00:00'
+  ignore_timezone 如果忽略的话，则默认值为 1 。
+
 
 
 #### TIMEZONE
@@ -1085,7 +1095,7 @@ ignore_negative: {
 
 ```sql
 DIFF(expr [, ignore_negative])
- 
+
 ignore_negative: {
     0
   | 1

include/libs/function/taosudf.h

@@ -228,7 +228,7 @@ static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentR
           newSize = 8;
         }
 
-        while (newSize < data->varLenCol.payloadLen + dataLen) {
+        while (newSize < (uint32_t)(data->varLenCol.payloadLen + dataLen)) {
           newSize = newSize * UDF_MEMORY_EXP_GROWTH;
         }
 
@@ -248,7 +248,7 @@ static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentR
       data->varLenCol.payloadLen += dataLen;
     }
   }
-  data->numOfRows = (currentRow + 1 > data->numOfRows) ? (currentRow + 1) : data->numOfRows;
+  data->numOfRows = ((int32_t)(currentRow + 1) > data->numOfRows) ? (int32_t)(currentRow + 1) : data->numOfRows;
   return 0;
 }
 

include/libs/qcom/query.h

@@ -262,24 +262,23 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
   (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
    NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
 
-#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || (_code) == TSDB_CODE_VND_STOPPED)
+#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || (_code) == TSDB_CODE_VND_STOPPED || (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING)
 #define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR)
-#define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false)  // used later
+#define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_MNODE_NOT_FOUND) 
+
+#define NO_RET_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
 
 #define NEED_REDIRECT_ERROR(_code)                                                      \
-  ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL ||  \
+  (NO_RET_REDIRECT_ERROR(_code) || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
-   (_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
+   SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code))
-   SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \
+   
-   (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING)
 
 #define NEED_CLIENT_RM_TBLMETA_REQ(_type)                                                                  \
   ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
    (_type) == TDMT_MND_DROP_STB)
 
 #define NEED_SCHEDULER_REDIRECT_ERROR(_code)                                            \
-  ((_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
+  (SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code))
-   SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \
-   (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING)
 
 #define REQUEST_TOTAL_EXEC_TIMES 2
 

include/libs/stream/streamState.h

@@ -35,7 +35,7 @@ typedef struct STdbState {
   TTB*         pFillStateDb;  // todo refactor
   TTB*         pSessionStateDb;
   TTB*         pParNameDb;
-  TXN          txn;
+  TXN*         txn;
 } STdbState;
 
 // incremental state storage

include/libs/stream/tstream.h

@@ -620,7 +620,7 @@ typedef struct SStreamMeta {
   SHashObj*    pTasks;
   SHashObj*    pRecoverStatus;
   void*        ahandle;
-  TXN          txn;
+  TXN*         txn;
   FTaskExpand* expandFunc;
   int32_t      vgId;
   SRWLatch     lock;

include/libs/wal/wal.h

@@ -107,6 +107,8 @@ typedef struct SWal {
   TdFilePtr pIdxFile;
   int32_t   writeCur;
   SArray   *fileInfoSet;  // SArray<SWalFileInfo>
+  // gc
+  SArray *toDeleteFiles;  // SArray<SWalFileInfo>
   // status
   int64_t totSize;
   int64_t lastRollSeq;

include/os/osFile.h

@@ -88,6 +88,7 @@ int32_t taosFsyncFile(TdFilePtr pFile);
 int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count);
 int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset);
 int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
+int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset);
 void    taosFprintfFile(TdFilePtr pFile, const char *format, ...);
 
 int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf);

include/util/taoserror.h

@@ -72,7 +72,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_OUT_OF_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0102)
 // #define TSDB_CODE_COM_INVALID_CFG_MSG        TAOS_DEF_ERROR_CODE(0, 0x0103) // 2.x
 #define TSDB_CODE_FILE_CORRUPTED                TAOS_DEF_ERROR_CODE(0, 0x0104) //
-#define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0105) // internal
+// #define TSDB_CODE_REF_NO_MEMORY              TAOS_DEF_ERROR_CODE(0, 0x0105) // 2.x
 #define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0106) // internal
 #define TSDB_CODE_REF_ID_REMOVED                TAOS_DEF_ERROR_CODE(0, 0x0107) // internal
 #define TSDB_CODE_REF_INVALID_ID                TAOS_DEF_ERROR_CODE(0, 0x0108) // internal
@@ -163,7 +163,7 @@ int32_t* taosGetErrno();
 // #define TSDB_CODE_MND_ACTION_NEED_REPROCESSEDTAOS_DEF_ERROR_CODE(0, 0x0302) // 2.x
 #define TSDB_CODE_MND_NO_RIGHTS                 TAOS_DEF_ERROR_CODE(0, 0x0303)
 // #define TSDB_CODE_MND_APP_ERROR              TAOS_DEF_ERROR_CODE(0, 0x0304) // 2.x
-#define TSDB_CODE_MND_INVALID_CONNECTION        TAOS_DEF_ERROR_CODE(0, 0x0305)
+// #define TSDB_CODE_MND_INVALID_CONNECTION     TAOS_DEF_ERROR_CODE(0, 0x0305) // 2.x
 // #define TSDB_CODE_MND_INVALID_MSG_VERSION    TAOS_DEF_ERROR_CODE(0, 0x0306) // 2.x
 // #define TSDB_CODE_MND_INVALID_MSG_LEN        TAOS_DEF_ERROR_CODE(0, 0x0307) // 2.x
 // #define TSDB_CODE_MND_INVALID_MSG_TYPE       TAOS_DEF_ERROR_CODE(0, 0x0308) // 2.x
@@ -171,27 +171,28 @@ int32_t* taosGetErrno();
 // #define TSDB_CODE_MND_OUT_OF_MEMORY          TAOS_DEF_ERROR_CODE(0, 0x030A) // 2.x
 #define TSDB_CODE_MND_INVALID_SHOWOBJ           TAOS_DEF_ERROR_CODE(0, 0x030B)
 #define TSDB_CODE_MND_INVALID_QUERY_ID          TAOS_DEF_ERROR_CODE(0, 0x030C)
-#define TSDB_CODE_MND_INVALID_STREAM_ID         TAOS_DEF_ERROR_CODE(0, 0x030D)
+// #define TSDB_CODE_MND_INVALID_STREAM_ID      TAOS_DEF_ERROR_CODE(0, 0x030D) // 2.x
 #define TSDB_CODE_MND_INVALID_CONN_ID           TAOS_DEF_ERROR_CODE(0, 0x030E)
-#define TSDB_CODE_MND_MNODE_IS_RUNNING          TAOS_DEF_ERROR_CODE(0, 0x0310)
+// #define TSDB_CODE_MND_MNODE_IS_RUNNING       TAOS_DEF_ERROR_CODE(0, 0x0310) // 2.x
-#define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC     TAOS_DEF_ERROR_CODE(0, 0x0311)
+// #define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC  TAOS_DEF_ERROR_CODE(0, 0x0311) // 2.x
-#define TSDB_CODE_MND_FAILED_TO_START_SYNC      TAOS_DEF_ERROR_CODE(0, 0x0312)
+// #define TSDB_CODE_MND_FAILED_TO_START_SYNC   TAOS_DEF_ERROR_CODE(0, 0x0312) // 2.x
-#define TSDB_CODE_MND_FAILED_TO_CREATE_DIR      TAOS_DEF_ERROR_CODE(0, 0x0313)
+// #define TSDB_CODE_MND_FAILED_TO_CREATE_DIR   TAOS_DEF_ERROR_CODE(0, 0x0313) // 2.x
-#define TSDB_CODE_MND_FAILED_TO_INIT_STEP       TAOS_DEF_ERROR_CODE(0, 0x0314)
+// #define TSDB_CODE_MND_FAILED_TO_INIT_STEP    TAOS_DEF_ERROR_CODE(0, 0x0314) // 2.x
 #define TSDB_CODE_MND_USER_DISABLED             TAOS_DEF_ERROR_CODE(0, 0x0315)
 
 // mnode-sdb
-#define TSDB_CODE_SDB_OBJ_ALREADY_THERE         TAOS_DEF_ERROR_CODE(0, 0x0320)
+#define TSDB_CODE_SDB_OBJ_ALREADY_THERE         TAOS_DEF_ERROR_CODE(0, 0x0320) // internal
-// #define TSDB_CODE_APP_ERROR                  TAOS_DEF_ERROR_CODE(0, 0x0321) // 2.x
+// #define TSDB_CODE_MND_SDB_ERROR              TAOS_DEF_ERROR_CODE(0, 0x0321) // 2.x
-#define TSDB_CODE_SDB_INVALID_TABLE_TYPE        TAOS_DEF_ERROR_CODE(0, 0x0322)
+#define TSDB_CODE_SDB_INVALID_TABLE_TYPE        TAOS_DEF_ERROR_CODE(0, 0x0322) // internal
 #define TSDB_CODE_SDB_OBJ_NOT_THERE             TAOS_DEF_ERROR_CODE(0, 0x0323)
-#define TSDB_CODE_SDB_INVALID_KEY_TYPE          TAOS_DEF_ERROR_CODE(0, 0x0325)
+// #define TSDB_CODE_MND_SDB_INVAID_META_ROW    TAOS_DEF_ERROR_CODE(0, 0x0324) // 2.x
-#define TSDB_CODE_SDB_INVALID_ACTION_TYPE       TAOS_DEF_ERROR_CODE(0, 0x0326)
+// #define TSDB_CODE_SDB_INVALID_KEY_TYPE       TAOS_DEF_ERROR_CODE(0, 0x0325) // 2.x
-#define TSDB_CODE_SDB_INVALID_STATUS_TYPE       TAOS_DEF_ERROR_CODE(0, 0x0327)
+#define TSDB_CODE_SDB_INVALID_ACTION_TYPE       TAOS_DEF_ERROR_CODE(0, 0x0326) // internal
-#define TSDB_CODE_SDB_INVALID_DATA_VER          TAOS_DEF_ERROR_CODE(0, 0x0328)
+// #define TSDB_CODE_SDB_INVALID_STATUS_TYPE    TAOS_DEF_ERROR_CODE(0, 0x0327) // unused
-#define TSDB_CODE_SDB_INVALID_DATA_LEN          TAOS_DEF_ERROR_CODE(0, 0x0329)
+#define TSDB_CODE_SDB_INVALID_DATA_VER          TAOS_DEF_ERROR_CODE(0, 0x0328) // internal
+#define TSDB_CODE_SDB_INVALID_DATA_LEN          TAOS_DEF_ERROR_CODE(0, 0x0329) // internal
 #define TSDB_CODE_SDB_INVALID_DATA_CONTENT      TAOS_DEF_ERROR_CODE(0, 0x032A)
-#define TSDB_CODE_SDB_INVALID_WAl_VER           TAOS_DEF_ERROR_CODE(0, 0x032B)
+// #define TSDB_CODE_SDB_INVALID_WAl_VER        TAOS_DEF_ERROR_CODE(0, 0x032B) // unused
 #define TSDB_CODE_SDB_OBJ_CREATING              TAOS_DEF_ERROR_CODE(0, 0x032C)
 #define TSDB_CODE_SDB_OBJ_DROPPING              TAOS_DEF_ERROR_CODE(0, 0x032D)
 
@@ -512,7 +513,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TQ_INVALID_CONFIG             TAOS_DEF_ERROR_CODE(0, 0x0A00)
 #define TSDB_CODE_TQ_INIT_FAILED                TAOS_DEF_ERROR_CODE(0, 0x0A01)
 #define TSDB_CODE_TQ_NO_DISK_PERMISSIONS        TAOS_DEF_ERROR_CODE(0, 0x0A03)
-#define TSDB_CODE_TQ_FILE_CORRUPTED             TAOS_DEF_ERROR_CODE(0, 0x0A04)
+// #define TSDB_CODE_TQ_FILE_CORRUPTED          TAOS_DEF_ERROR_CODE(0, 0x0A04)
 // #define TSDB_CODE_TQ_OUT_OF_MEMORY           TAOS_DEF_ERROR_CODE(0, 0x0A05)
 #define TSDB_CODE_TQ_FILE_ALREADY_EXISTS        TAOS_DEF_ERROR_CODE(0, 0x0A06)
 #define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR       TAOS_DEF_ERROR_CODE(0, 0x0A07)
@@ -698,7 +699,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_RSMA_INVALID_ENV               TAOS_DEF_ERROR_CODE(0, 0x3150)
 #define TSDB_CODE_RSMA_INVALID_STAT              TAOS_DEF_ERROR_CODE(0, 0x3151)
 #define TSDB_CODE_RSMA_QTASKINFO_CREATE          TAOS_DEF_ERROR_CODE(0, 0x3152)
-#define TSDB_CODE_RSMA_FILE_CORRUPTED            TAOS_DEF_ERROR_CODE(0, 0x3153)
+// #define TSDB_CODE_RSMA_FILE_CORRUPTED         TAOS_DEF_ERROR_CODE(0, 0x3153)
 #define TSDB_CODE_RSMA_REMOVE_EXISTS             TAOS_DEF_ERROR_CODE(0, 0x3154)
 #define TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP        TAOS_DEF_ERROR_CODE(0, 0x3155)
 #define TSDB_CODE_RSMA_EMPTY_INFO                TAOS_DEF_ERROR_CODE(0, 0x3156)

source/client/src/clientEnv.c

@@ -397,8 +397,8 @@ void taos_init_imp(void) {
   deltaToUtcInitOnce();
 
   if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
-    tscInitRes = -1;
+    // ignore create log failed, only print
-    return;
+    printf(" WARING: Create taoslog failed. configDir=%s\n", configDir);
   }
 
   if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) {

source/client/src/clientJniConnector.c

@@ -136,7 +136,7 @@ int32_t check_for_params(jobject jobj, jlong conn, jlong res) {
   }
 
   if ((TAOS_RES *)res == NULL) {
-    jniError("jobj:%p, conn:%p, res is null", jobj, (TAOS *)conn);
+    jniError("jobj:%p, conn:%p, param res is null", jobj, (TAOS *)conn);
     return JNI_RESULT_SET_NULL;
   }
 
@@ -393,9 +393,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp(
     return code;
   }
 
-  taos_free_result((void *)res);
   jniDebug("jobj:%p, conn:%p, free resultset:%p", jobj, (TAOS *)con, (void *)res);
-
+  taos_free_result((void *)res);
   return JNI_SUCCESS;
 }
 
@@ -489,7 +488,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
                numOfFields);
       return JNI_FETCH_END;
     } else {
-      jniDebug("jobj:%p, conn:%p, interrupted query", jobj, tscon);
+      jniDebug("jobj:%p, conn:%p, interrupted query. fetch row error code: %d, msg:%s", jobj, tscon, code, taos_errstr(result));
       return JNI_RESULT_SET_NULL;
     }
   }
@@ -584,7 +583,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI
       jniDebug("jobj:%p, conn:%p, resultset:%p, no data to retrieve", jobj, tscon, (void *)res);
       return JNI_FETCH_END;
     } else {
-      jniError("jobj:%p, conn:%p, query interrupted", jobj, tscon);
+      jniError("jobj:%p, conn:%p, query interrupted. fetch block error code:%d, msg:%s", jobj, tscon, error_code, taos_errstr(tres));
       return JNI_RESULT_SET_NULL;
     }
   }

source/client/src/clientTmqConnector.c

@@ -303,7 +303,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(
       jniDebug("jobj:%p, conn:%p, resultset:%p, no data to retrieve", jobj, tscon, (void *)res);
       return JNI_FETCH_END;
     } else {
-      jniError("jobj:%p, conn:%p, query interrupted", jobj, tscon);
+      jniError("jobj:%p, conn:%p, query interrupted, tmq fetch block error code:%d, msg:%s", jobj, tscon, error_code, taos_errstr(tres));
       return JNI_RESULT_SET_NULL;
     }
   }

source/dnode/mnode/impl/src/mndShow.c

@@ -198,9 +198,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
   }
 
   if (retrieveReq.showId == 0) {
-    STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb));
+    STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb));
     if (pMeta == NULL) {
-      pMeta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb));
+      pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb));
       if (pMeta == NULL) {
         terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
         mError("failed to process show-retrieve req:%p since %s", pShow, terrstr());

source/dnode/vnode/src/inc/meta.h

@@ -70,7 +70,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid);
 int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo);
 int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid);
 int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo);
-void    metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t delta);
+void    metaUpdateStbStats(SMeta* pMeta, int64_t uid, int64_t delta);
 int32_t metaUidFilterCacheGet(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, LRUHandle** pHandle);
 
 struct SMeta {
@@ -79,7 +79,7 @@ struct SMeta {
   char*   path;
   SVnode* pVnode;
   TDB*    pEnv;
-  TXN     txn;
+  TXN*    txn;
   TTB*    pTbDb;
   TTB*    pSkmDb;
   TTB*    pUidIdx;

source/dnode/vnode/src/inc/vnd.h

@@ -77,7 +77,7 @@ void    vnodeBufPoolReset(SVBufPool* pPool);
 
 // vnodeQuery.c
 int32_t vnodeQueryOpen(SVnode* pVnode);
-void    vnodeQueryPreClose(SVnode *pVnode);
+void    vnodeQueryPreClose(SVnode* pVnode);
 void    vnodeQueryClose(SVnode* pVnode);
 int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg, bool direct);
 int     vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg, bool direct);
@@ -86,7 +86,6 @@ int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg);
 // vnodeCommit.c
 int32_t vnodeBegin(SVnode* pVnode);
 int32_t vnodeShouldCommit(SVnode* pVnode);
-int32_t vnodeCommit(SVnode* pVnode);
 void    vnodeRollback(SVnode* pVnode);
 int32_t vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg);
 int32_t vnodeCommitInfo(const char* dir, const SVnodeInfo* pInfo);

source/dnode/vnode/src/inc/vnodeInt.h

@@ -75,6 +75,7 @@ typedef struct SStreamStateWriter SStreamStateWriter;
 typedef struct SRSmaSnapReader    SRSmaSnapReader;
 typedef struct SRSmaSnapWriter    SRSmaSnapWriter;
 typedef struct SSnapDataHdr       SSnapDataHdr;
+typedef struct SCommitInfo        SCommitInfo;
 
 #define VNODE_META_DIR  "meta"
 #define VNODE_TSDB_DIR  "tsdb"
@@ -100,8 +101,9 @@ typedef struct STbUidStore STbUidStore;
 int             metaOpen(SVnode* pVnode, SMeta** ppMeta, int8_t rollback);
 int             metaClose(SMeta* pMeta);
 int             metaBegin(SMeta* pMeta, int8_t fromSys);
-int             metaCommit(SMeta* pMeta);
+TXN*            metaGetTxn(SMeta* pMeta);
-int             metaFinishCommit(SMeta* pMeta);
+int             metaCommit(SMeta* pMeta, TXN* txn);
+int             metaFinishCommit(SMeta* pMeta, TXN* txn);
 int             metaPrepareAsyncCommit(SMeta* pMeta);
 int             metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
 int             metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
@@ -146,7 +148,8 @@ int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pR
 int     tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback);
 int     tsdbClose(STsdb** pTsdb);
 int32_t tsdbBegin(STsdb* pTsdb);
-int32_t tsdbCommit(STsdb* pTsdb);
+int32_t tsdbPrepareCommit(STsdb* pTsdb);
+int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo);
 int32_t tsdbFinishCommit(STsdb* pTsdb);
 int32_t tsdbRollbackCommit(STsdb* pTsdb);
 int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now);
@@ -203,8 +206,8 @@ int32_t smaBegin(SSma* pSma);
 int32_t smaSyncPreCommit(SSma* pSma);
 int32_t smaSyncCommit(SSma* pSma);
 int32_t smaSyncPostCommit(SSma* pSma);
-int32_t smaPreCommit(SSma* pSma);
+int32_t smaPrepareAsyncCommit(SSma* pSma);
-int32_t smaCommit(SSma* pSma);
+int32_t smaCommit(SSma* pSma, SCommitInfo* pInfo);
 int32_t smaFinishCommit(SSma* pSma);
 int32_t smaPostCommit(SSma* pSma);
 int32_t smaDoRetention(SSma* pSma, int64_t now);
@@ -406,6 +409,12 @@ struct SSnapDataHdr {
   uint8_t data[];
 };
 
+struct SCommitInfo {
+  SVnodeInfo info;
+  SVnode*    pVnode;
+  TXN*       txn;
+};
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/meta/metaCommit.c

@@ -20,12 +20,19 @@ static FORCE_INLINE void  metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVB
 
 // begin a meta txn
 int metaBegin(SMeta *pMeta, int8_t fromSys) {
+  void *(*xMalloc)(void *, size_t);
+  void (*xFree)(void *, void *);
+  void *xArg = NULL;
+
   if (fromSys) {
-    tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+    xMalloc = tdbDefaultMalloc;
+    xFree = tdbDefaultFree;
   } else {
-    tdbTxnOpen(&pMeta->txn, 0, metaMalloc, metaFree, pMeta->pVnode->inUse, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+    xMalloc = metaMalloc;
+    xFree = metaFree;
+    xArg = pMeta->pVnode->inUse;
   }
-  if (tdbBegin(pMeta->pEnv, &pMeta->txn) < 0) {
+  if (tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
     return -1;
   }
 
@@ -33,9 +40,16 @@ int metaBegin(SMeta *pMeta, int8_t fromSys) {
 }
 
 // commit the meta txn
-int metaCommit(SMeta *pMeta) { return tdbCommit(pMeta->pEnv, &pMeta->txn); }
+TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; }
-int metaFinishCommit(SMeta *pMeta) { return tdbPostCommit(pMeta->pEnv, &pMeta->txn); }
+int  metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); }
-int metaPrepareAsyncCommit(SMeta *pMeta) { return tdbPrepareAsyncCommit(pMeta->pEnv, &pMeta->txn); }
+int  metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); }
+int  metaPrepareAsyncCommit(SMeta *pMeta) {
+   // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn);
+  int code = 0;
+  code = tdbCommit(pMeta->pEnv, pMeta->txn);
+
+  return code;
+}
 
 // abort the meta txn
-int metaAbort(SMeta *pMeta) { return tdbAbort(pMeta->pEnv, &pMeta->txn); }
+int metaAbort(SMeta *pMeta) { return tdbAbort(pMeta->pEnv, pMeta->txn); }

source/dnode/vnode/src/meta/metaSma.c

@@ -117,7 +117,7 @@ static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) {
   tEncoderClear(&coder);
 
   // write to table.db
-  if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
+  if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, pMeta->txn) < 0) {
     goto _err;
   }
 
@@ -131,17 +131,17 @@ _err:
 
 static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SUidIdxVal uidIdxVal = {.suid = pME->smaEntry.tsma->indexUid, .version = pME->version, .skmVer = 0};
-  return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), &pMeta->txn);
+  return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), pMeta->txn);
 }
 
 static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
+  return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn);
 }
 
 static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid};
 
-  return tdbTbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn);
+  return tdbTbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, pMeta->txn);
 }
 
 static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) {

source/dnode/vnode/src/meta/metaSnapshot.c

@@ -163,9 +163,9 @@ int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
   if (rollback) {
     ASSERT(0);
   } else {
-    code = metaCommit(pWriter->pMeta);
+    code = metaCommit(pWriter->pMeta, pWriter->pMeta->txn);
     if (code) goto _err;
-    code = metaFinishCommit(pWriter->pMeta);
+    code = metaFinishCommit(pWriter->pMeta, pWriter->pMeta->txn);
     if (code) goto _err;
   }
   taosMemoryFree(pWriter);

source/dnode/vnode/src/meta/metaTable.c

@@ -261,7 +261,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb
   // drop all child tables
   TBC *pCtbIdxc = NULL;
 
-  tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
+  tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL);
   rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c);
   if (rc < 0) {
     tdbTbcClose(pCtbIdxc);
@@ -295,10 +295,10 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb
 _drop_super_table:
   tdbTbGet(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pData, &nData);
   tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = ((SUidIdxVal *)pData)[0].version, .uid = pReq->suid},
-              sizeof(STbDbKey), &pMeta->txn);
+              sizeof(STbDbKey), pMeta->txn);
-  tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pMeta->txn);
+  tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, pMeta->txn);
-  tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn);
+  tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn);
-  tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn);
+  tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn);
 
   metaULock(pMeta);
 
@@ -321,7 +321,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   int32_t     ret;
   int32_t     c = -2;
 
-  tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL);
   ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c);
   if (ret < 0 || c) {
     tdbTbcClose(pUidIdxc);
@@ -340,7 +340,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
 
   oversion = ((SUidIdxVal *)pData)[0].version;
 
-  tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
+  tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL);
   ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c);
   ASSERT(ret == 0 && c == 0);
 
@@ -595,7 +595,7 @@ static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
   STtlIdxKey ttlKey = {0};
   metaBuildTtlIdxKey(&ttlKey, pME);
   if (ttlKey.dtime == 0) return 0;
-  return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), &pMeta->txn);
+  return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), pMeta->txn);
 }
 
 static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
@@ -657,7 +657,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
 
           if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid,
                                   &pTagIdxKey, &nTagIdxKey) == 0) {
-            tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, &pMeta->txn);
+            tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn);
           }
           metaDestroyTagIdxKey(pTagIdxKey);
         }
@@ -667,9 +667,9 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
     }
   }
 
-  tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pMeta->txn);
+  tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), pMeta->txn);
-  tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, &pMeta->txn);
+  tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, pMeta->txn);
-  tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn);
+  tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), pMeta->txn);
 
   if (e.type == TSDB_CHILD_TABLE || e.type == TSDB_NORMAL_TABLE) metaDeleteCtimeIdx(pMeta, &e);
   if (e.type == TSDB_NORMAL_TABLE) metaDeleteNcolIdx(pMeta, &e);
@@ -677,7 +677,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
   if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e);
 
   if (e.type == TSDB_CHILD_TABLE) {
-    tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), &pMeta->txn);
+    tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), pMeta->txn);
 
     --pMeta->pVnode->config.vndStats.numOfCTables;
 
@@ -689,7 +689,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
     --pMeta->pVnode->config.vndStats.numOfNTables;
     pMeta->pVnode->config.vndStats.numOfNTimeSeries -= e.ntbEntry.schemaRow.nCols - 1;
   } else if (e.type == TSDB_SUPER_TABLE) {
-    tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), &pMeta->txn);
+    tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), pMeta->txn);
     // drop schema.db (todo)
 
     metaStatsCacheDrop(pMeta, uid);
@@ -710,7 +710,7 @@ int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
   if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) {
     return 0;
   }
-  return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, &pMeta->txn);
+  return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn);
 }
 
 int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -718,14 +718,14 @@ int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
   if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) {
     return 0;
   }
-  return tdbTbDelete(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), &pMeta->txn);
+  return tdbTbDelete(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), pMeta->txn);
 }
 int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SNcolIdxKey ncolKey = {0};
   if (metaBuildNColIdxKey(&ncolKey, pME) < 0) {
     return 0;
   }
-  return tdbTbInsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, &pMeta->txn);
+  return tdbTbInsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn);
 }
 
 int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
@@ -733,7 +733,7 @@ int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
   if (metaBuildNColIdxKey(&ncolKey, pME) < 0) {
     return 0;
   }
-  return tdbTbDelete(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), &pMeta->txn);
+  return tdbTbDelete(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), pMeta->txn);
 }
 
 static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) {
@@ -768,7 +768,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
   // search uid index
   TBC *pUidIdxc = NULL;
 
-  tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL);
   tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
   ASSERT(c == 0);
 
@@ -778,7 +778,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
   // search table.db
   TBC *pTbDbc = NULL;
 
-  tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
+  tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL);
   tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c);
   ASSERT(c == 0);
   tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
@@ -959,7 +959,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   // search uid index
   TBC *pUidIdxc = NULL;
 
-  tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL);
   tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
   ASSERT(c == 0);
 
@@ -972,7 +972,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   SDecoder dc2 = {0};
 
   /* get ctbEntry */
-  tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
+  tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL);
   tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c);
   ASSERT(c == 0);
   tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
@@ -1075,7 +1075,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   ASSERT(ctbEntry.ctbEntry.pTags);
   SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid};
   tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags,
-              ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn);
+              ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn);
 
   metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid);
 
@@ -1125,7 +1125,7 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
   // search uid index
   TBC *pUidIdxc = NULL;
 
-  tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn);
+  tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL);
   tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c);
   ASSERT(c == 0);
 
@@ -1135,7 +1135,7 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
   // search table.db
   TBC *pTbDbc = NULL;
 
-  tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn);
+  tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL);
   tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c);
   ASSERT(c == 0);
   tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData);
@@ -1242,7 +1242,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
   tEncoderClear(&coder);
 
   // write to table.db
-  if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) {
+  if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, pMeta->txn) < 0) {
     goto _err;
   }
 
@@ -1265,29 +1265,29 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) {
 
   SUidIdxVal uidIdxVal = {.suid = info.suid, .version = info.version, .skmVer = info.skmVer};
 
-  return tdbTbUpsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), &pMeta->txn);
+  return tdbTbUpsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), pMeta->txn);
 }
 
 static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, &pMeta->txn);
+  return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn);
 }
 
 static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
-  return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn);
+  return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn);
 }
 
 static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
   STtlIdxKey ttlKey = {0};
   metaBuildTtlIdxKey(&ttlKey, pME);
   if (ttlKey.dtime == 0) return 0;
-  return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
+  return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn);
 }
 
 static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
   SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
 
   return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags,
-                     ((STag *)(pME->ctbEntry.pTags))->len, &pMeta->txn);
+                     ((STag *)(pME->ctbEntry.pTags))->len, pMeta->txn);
 }
 
 int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid,
@@ -1379,7 +1379,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
       ret = -1;
       goto end;
     }
-    tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn);
+    tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn);
   }
 end:
   metaDestroyTagIdxKey(pTagIdxKey);
@@ -1426,7 +1426,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
   tEncoderInit(&coder, pVal, vLen);
   tEncodeSSchemaWrapper(&coder, pSW);
 
-  if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) {
+  if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, pMeta->txn) < 0) {
     rcode = -1;
     goto _exit;
   }

source/dnode/vnode/src/sma/smaCommit.c

@@ -23,7 +23,7 @@ static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma);
 static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma);
 #endif
 static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma);
-static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma);
+static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo);
 static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma);
 static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat);
 
@@ -59,7 +59,7 @@ int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaSyncPostCommitImpl(p
  * @param pSma
  * @return int32_t
  */
-int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); }
+int32_t smaPrepareAsyncCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); }
 
 /**
  * @brief async commit, only applicable to Rollup SMA
@@ -67,7 +67,7 @@ int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma);
  * @param pSma
  * @return int32_t
  */
-int32_t smaCommit(SSma *pSma) { return tdProcessRSmaAsyncCommitImpl(pSma); }
+int32_t smaCommit(SSma *pSma, SCommitInfo *pInfo) { return tdProcessRSmaAsyncCommitImpl(pSma, pInfo); }
 
 /**
  * @brief async commit, only applicable to Rollup SMA
@@ -127,8 +127,8 @@ _exit:
 }
 
 int32_t smaFinishCommit(SSma *pSma) {
-  int32_t  code = 0;
+  int32_t code = 0;
-  SVnode  *pVnode = pSma->pVnode;
+  SVnode *pVnode = pSma->pVnode;
 
   if (VND_RSMA1(pVnode) && (code = tsdbFinishCommit(VND_RSMA1(pVnode))) < 0) {
     smaError("vgId:%d, failed to finish commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(code));
@@ -378,6 +378,11 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
   taosWUnLockLatch(SMA_ENV_LOCK(pEnv));
 #endif
 
+  // all rsma results are written completely
+  STsdb *pTsdb = NULL;
+  if ((pTsdb = VND_RSMA1(pSma->pVnode))) tsdbPrepareCommit(pTsdb);
+  if ((pTsdb = VND_RSMA2(pSma->pVnode))) tsdbPrepareCommit(pTsdb);
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -387,9 +392,9 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
  * @param pSma
  * @return int32_t
  */
-static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) {
+static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo) {
-  int32_t  code = 0;
+  int32_t code = 0;
-  SVnode  *pVnode = pSma->pVnode;
+  SVnode *pVnode = pSma->pVnode;
 #if 0
   SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv);
 
@@ -399,11 +404,11 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) {
   }
 #endif
 
-  if ((code = tsdbCommit(VND_RSMA1(pVnode))) < 0) {
+  if ((code = tsdbCommit(VND_RSMA1(pVnode), pInfo)) < 0) {
     smaError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(code));
     goto _exit;
   }
-  if ((code = tsdbCommit(VND_RSMA2(pVnode))) < 0) {
+  if ((code = tsdbCommit(VND_RSMA2(pVnode), pInfo)) < 0) {
     smaError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(code));
     goto _exit;
   }

source/dnode/vnode/src/tq/tqMeta.c

@@ -108,24 +108,22 @@ int32_t tqMetaClose(STQ* pTq) {
 }
 
 int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen) {
-  TXN txn;
+  TXN* txn;
-  if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
+
+  if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
+      0) {
     return -1;
   }
 
-  if (tdbBegin(pTq->pMetaDB, &txn) < 0) {
+  if (tdbTbUpsert(pTq->pCheckStore, key, strlen(key), value, vLen, txn) < 0) {
     return -1;
   }
 
-  if (tdbTbUpsert(pTq->pCheckStore, key, strlen(key), value, vLen, &txn) < 0) {
+  if (tdbCommit(pTq->pMetaDB, txn) < 0) {
     return -1;
   }
 
-  if (tdbCommit(pTq->pMetaDB, &txn) < 0) {
+  if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
-    return -1;
-  }
-
-  if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) {
     return -1;
   }
 
@@ -133,25 +131,22 @@ int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_
 }
 
 int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) {
-  TXN txn;
+  TXN* txn;
 
-  if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
+  if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
+      0) {
     ASSERT(0);
   }
 
-  if (tdbBegin(pTq->pMetaDB, &txn) < 0) {
+  if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), txn) < 0) {
-    ASSERT(0);
-  }
-
-  if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), &txn) < 0) {
     /*ASSERT(0);*/
   }
 
-  if (tdbCommit(pTq->pMetaDB, &txn) < 0) {
+  if (tdbCommit(pTq->pMetaDB, txn) < 0) {
     ASSERT(0);
   }
 
-  if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) {
+  if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
     ASSERT(0);
   }
 
@@ -219,25 +214,22 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
     ASSERT(0);
   }
 
-  TXN txn;
+  TXN* txn;
 
-  if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
+  if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
+      0) {
     ASSERT(0);
   }
 
-  if (tdbBegin(pTq->pMetaDB, &txn) < 0) {
+  if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, txn) < 0) {
     ASSERT(0);
   }
 
-  if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, &txn) < 0) {
+  if (tdbCommit(pTq->pMetaDB, txn) < 0) {
     ASSERT(0);
   }
 
-  if (tdbCommit(pTq->pMetaDB, &txn) < 0) {
+  if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
-    ASSERT(0);
-  }
-
-  if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) {
     ASSERT(0);
   }
 
@@ -247,25 +239,22 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
 }
 
 int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) {
-  TXN txn;
+  TXN* txn;
 
-  if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
+  if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
+      0) {
     ASSERT(0);
   }
 
-  if (tdbBegin(pTq->pMetaDB, &txn) < 0) {
+  if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), txn) < 0) {
-    ASSERT(0);
-  }
-
-  if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), &txn) < 0) {
     /*ASSERT(0);*/
   }
 
-  if (tdbCommit(pTq->pMetaDB, &txn) < 0) {
+  if (tdbCommit(pTq->pMetaDB, txn) < 0) {
     ASSERT(0);
   }
 
-  if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) {
+  if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
     ASSERT(0);
   }
 

source/dnode/vnode/src/tq/tqSnapshot.c

@@ -129,7 +129,7 @@ struct STqSnapWriter {
   STQ*    pTq;
   int64_t sver;
   int64_t ever;
-  TXN     txn;
+  TXN*    txn;
 };
 
 int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) {
@@ -146,8 +146,10 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p
   pWriter->sver = sver;
   pWriter->ever = ever;
 
-  if (tdbTxnOpen(&pWriter->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
+  if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
-    ASSERT(0);
+    code = -1;
+    taosMemoryFree(pWriter);
+    goto _err;
   }
 
   *ppWriter = pWriter;
@@ -165,11 +167,11 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) {
   STQ*           pTq = pWriter->pTq;
 
   if (rollback) {
-    tdbAbort(pWriter->pTq->pMetaDB, &pWriter->txn);
+    tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn);
   } else {
-    code = tdbCommit(pWriter->pTq->pMetaDB, &pWriter->txn);
+    code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn);
     if (code) goto _err;
-    code = tdbPostCommit(pWriter->pTq->pMetaDB, &pWriter->txn);
+    code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn);
     if (code) goto _err;
   }
 

source/dnode/vnode/src/tq/tqStreamStateSnap.c

@@ -129,7 +129,7 @@ struct STqSnapWriter {
   STQ*    pTq;
   int64_t sver;
   int64_t ever;
-  TXN     txn;
+  TXN*    txn;
 };
 
 int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) {
@@ -146,8 +146,10 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p
   pWriter->sver = sver;
   pWriter->ever = ever;
 
-  if (tdbTxnOpen(&pWriter->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
+  if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
-    ASSERT(0);
+    code = -1;
+    taosMemoryFree(pWriter);
+    goto _err;
   }
 
   *ppWriter = pWriter;
@@ -165,11 +167,12 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) {
   STQ*           pTq = pWriter->pTq;
 
   if (rollback) {
+    tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn);
     ASSERT(0);
   } else {
-    code = tdbCommit(pWriter->pTq->pMetaDB, &pWriter->txn);
+    code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn);
     if (code) goto _err;
-    code = tdbPostCommit(pWriter->pTq->pMetaDB, &pWriter->txn);
+    code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn);
     if (code) goto _err;
   }
 

source/dnode/vnode/src/tq/tqStreamTaskSnap.c

@@ -129,7 +129,7 @@ struct STqSnapWriter {
   STQ*    pTq;
   int64_t sver;
   int64_t ever;
-  TXN     txn;
+  TXN*    txn;
 };
 
 int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) {
@@ -146,8 +146,10 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p
   pWriter->sver = sver;
   pWriter->ever = ever;
 
-  if (tdbTxnOpen(&pWriter->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
+  if (tdbBegin(pTq->pMetaStore, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
-    ASSERT(0);
+    code = -1;
+    taosMemoryFree(pWriter);
+    goto _err;
   }
 
   *ppWriter = pWriter;
@@ -165,11 +167,12 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) {
   STQ*           pTq = pWriter->pTq;
 
   if (rollback) {
+    tdbAbort(pWriter->pTq->pMetaStore, pWriter->txn);
     ASSERT(0);
   } else {
-    code = tdbCommit(pWriter->pTq->pMetaStore, &pWriter->txn);
+    code = tdbCommit(pWriter->pTq->pMetaStore, pWriter->txn);
     if (code) goto _err;
-    code = tdbPostCommit(pWriter->pTq->pMetaStore, &pWriter->txn);
+    code = tdbPostCommit(pWriter->pTq->pMetaStore, pWriter->txn);
     if (code) goto _err;
   }
 

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -93,7 +93,7 @@ typedef struct {
   SArray      *aDelData;  // SArray<SDelData>
 } SCommitter;
 
-static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter);
+static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter, SCommitInfo *pInfo);
 static int32_t tsdbCommitData(SCommitter *pCommitter);
 static int32_t tsdbCommitDel(SCommitter *pCommitter);
 static int32_t tsdbCommitCache(SCommitter *pCommitter);
@@ -150,18 +150,28 @@ _exit:
   return code;
 }
 
-int32_t tsdbCommit(STsdb *pTsdb) {
+int32_t tsdbPrepareCommit(STsdb *pTsdb) {
+  taosThreadRwlockWrlock(&pTsdb->rwLock);
+  ASSERT(pTsdb->imem == NULL);
+  pTsdb->imem = pTsdb->mem;
+  pTsdb->mem = NULL;
+  taosThreadRwlockUnlock(&pTsdb->rwLock);
+
+  return 0;
+}
+
+int32_t tsdbCommit(STsdb *pTsdb, SCommitInfo *pInfo) {
   if (!pTsdb) return 0;
 
   int32_t    code = 0;
   int32_t    lino = 0;
   SCommitter commith;
-  SMemTable *pMemTable = pTsdb->mem;
+  SMemTable *pMemTable = pTsdb->imem;
 
   // check
   if (pMemTable->nRow == 0 && pMemTable->nDel == 0) {
     taosThreadRwlockWrlock(&pTsdb->rwLock);
-    pTsdb->mem = NULL;
+    pTsdb->imem = NULL;
     taosThreadRwlockUnlock(&pTsdb->rwLock);
 
     tsdbUnrefMemTable(pMemTable);
@@ -169,7 +179,7 @@ int32_t tsdbCommit(STsdb *pTsdb) {
   }
 
   // start commit
-  code = tsdbStartCommit(pTsdb, &commith);
+  code = tsdbStartCommit(pTsdb, &commith, pInfo);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // commit impl
@@ -806,26 +816,21 @@ _exit:
 }
 
 // ----------------------------------------------------------------------------
-static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
+static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter, SCommitInfo *pInfo) {
   int32_t code = 0;
   int32_t lino = 0;
 
   memset(pCommitter, 0, sizeof(*pCommitter));
-  ASSERT(pTsdb->mem && pTsdb->imem == NULL && "last tsdb commit incomplete");
+  ASSERT(pTsdb->imem && "last tsdb commit incomplete");
-
-  taosThreadRwlockWrlock(&pTsdb->rwLock);
-  pTsdb->imem = pTsdb->mem;
-  pTsdb->mem = NULL;
-  taosThreadRwlockUnlock(&pTsdb->rwLock);
 
   pCommitter->pTsdb = pTsdb;
-  pCommitter->commitID = pTsdb->pVnode->state.commitID;
+  pCommitter->commitID = pInfo->info.state.commitID;
   pCommitter->minutes = pTsdb->keepCfg.days;
   pCommitter->precision = pTsdb->keepCfg.precision;
-  pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
+  pCommitter->minRow = pInfo->info.config.tsdbCfg.minRows;
-  pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
+  pCommitter->maxRow = pInfo->info.config.tsdbCfg.maxRows;
-  pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
+  pCommitter->cmprAlg = pInfo->info.config.tsdbCfg.compression;
-  pCommitter->sttTrigger = pTsdb->pVnode->config.sttTrigger;
+  pCommitter->sttTrigger = pInfo->info.config.sttTrigger;
   pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem);
   if (pCommitter->aTbDataP == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -750,7 +750,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
   // head
   tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom);
   tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo);
-  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+  pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
   if (pOutFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -771,7 +771,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
   // data
   tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom);
   tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo);
-  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+  pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
   if (pOutFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -781,7 +781,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
-  n = taosFSendFile(pOutFD, PInFD, 0, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage));
+  n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage));
   if (n < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -792,7 +792,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
   // sma
   tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom);
   tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo);
-  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+  pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
   if (pOutFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -814,7 +814,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
   for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) {
     tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom);
     tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo);
-    pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+    pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
     if (pOutFD == NULL) {
       code = TAOS_SYSTEM_ERROR(errno);
       goto _err;

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -14,14 +14,14 @@
  */
 
 #include "vnd.h"
+#include "vnodeInt.h"
 
-#define VND_INFO_FNAME "vnode.json"
+#define VND_INFO_FNAME     "vnode.json"
 #define VND_INFO_FNAME_TMP "vnode_tmp.json"
 
-static int  vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData);
+static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData);
-static int  vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo);
+static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo);
-static int  vnodeCommitImpl(void *arg);
+static int vnodeCommitImpl(SCommitInfo *pInfo);
-static void vnodeWaitCommit(SVnode *pVnode);
 
 int vnodeBegin(SVnode *pVnode) {
   // alloc buffer pool
@@ -107,7 +107,8 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
   // free info binary
   taosMemoryFree(data);
 
-  vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d", pInfo->config.vgId, fname, pInfo->config.syncCfg.replicaNum);
+  vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d", pInfo->config.vgId, fname,
+        pInfo->config.syncCfg.replicaNum);
 
   return 0;
 
@@ -185,29 +186,77 @@ _err:
   return -1;
 }
 
+static void vnodePrepareCommit(SVnode *pVnode) {
+  tsem_wait(&pVnode->canCommit);
+
+  tsdbPrepareCommit(pVnode->pTsdb);
+  metaPrepareAsyncCommit(pVnode->pMeta);
+  smaPrepareAsyncCommit(pVnode->pSma);
+
+  vnodeBufPoolUnRef(pVnode->inUse);
+  pVnode->inUse = NULL;
+}
+static int32_t vnodeCommitTask(void *arg) {
+  int32_t code = 0;
+
+  SCommitInfo *pInfo = (SCommitInfo *)arg;
+
+  // commit
+  code = vnodeCommitImpl(pInfo);
+  if (code) goto _exit;
+
+  // end commit
+  tsem_post(&pInfo->pVnode->canCommit);
+
+_exit:
+  taosMemoryFree(pInfo);
+  return code;
+}
 int vnodeAsyncCommit(SVnode *pVnode) {
-  vnodeWaitCommit(pVnode);
+  int32_t code = 0;
 
-  // vnodeBufPoolSwitch(pVnode);
+  // prepare to commit
-  // tsdbPrepareCommit(pVnode->pTsdb);
+  vnodePrepareCommit(pVnode);
 
-  vnodeScheduleTask(vnodeCommitImpl, pVnode);
+  // schedule the task
+  pVnode->state.commitTerm = pVnode->state.applyTerm;
 
-  return 0;
+  SCommitInfo *pInfo = (SCommitInfo *)taosMemoryCalloc(1, sizeof(*pInfo));
+  if (NULL == pInfo) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+  pInfo->info.config = pVnode->config;
+  pInfo->info.state.committed = pVnode->state.applied;
+  pInfo->info.state.commitTerm = pVnode->state.applyTerm;
+  pInfo->info.state.commitID = pVnode->state.commitID;
+  pInfo->pVnode = pVnode;
+  pInfo->txn = metaGetTxn(pVnode->pMeta);
+  vnodeScheduleTask(vnodeCommitTask, pInfo);
+
+_exit:
+  if (code) {
+    vError("vgId:%d %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code),
+           pVnode->state.commitID);
+  } else {
+    vDebug("vgId:%d %s done", TD_VID(pVnode), __func__);
+  }
+  return code;
 }
 
 int vnodeSyncCommit(SVnode *pVnode) {
   vnodeAsyncCommit(pVnode);
-  vnodeWaitCommit(pVnode);
+  tsem_wait(&pVnode->canCommit);
-  tsem_post(&(pVnode->canCommit));
+  tsem_post(&pVnode->canCommit);
   return 0;
 }
 
-int vnodeCommit(SVnode *pVnode) {
+static int vnodeCommitImpl(SCommitInfo *pInfo) {
-  int32_t    code = 0;
+  int32_t code = 0;
-  int32_t    lino = 0;
+  int32_t lino = 0;
-  SVnodeInfo info = {0};
+
-  char       dir[TSDB_FILENAME_LEN];
+  char    dir[TSDB_FILENAME_LEN] = {0};
+  SVnode *pVnode = pInfo->pVnode;
 
   vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode),
         pVnode->state.commitID, pVnode->state.applied, pVnode->state.applyTerm);
@@ -218,19 +267,13 @@ int vnodeCommit(SVnode *pVnode) {
     return -1;
   }
 
-  pVnode->state.commitTerm = pVnode->state.applyTerm;
-
   // save info
-  info.config = pVnode->config;
-  info.state.committed = pVnode->state.applied;
-  info.state.commitTerm = pVnode->state.applyTerm;
-  info.state.commitID = pVnode->state.commitID;
   if (pVnode->pTfs) {
     snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
   } else {
     snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path);
   }
-  if (vnodeSaveInfo(dir, &info) < 0) {
+  if (vnodeSaveInfo(dir, &pInfo->info) < 0) {
     code = terrno;
     TSDB_CHECK_CODE(code, lino, _exit);
   }
@@ -238,23 +281,12 @@ int vnodeCommit(SVnode *pVnode) {
   // walBeginSnapshot(pVnode->pWal, pVnode->state.applied);
   syncBeginSnapshot(pVnode->sync, pVnode->state.applied);
 
-  code = smaPreCommit(pVnode->pSma);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  vnodeBufPoolUnRef(pVnode->inUse);
-  pVnode->inUse = NULL;
-
   // commit each sub-system
-  if (metaCommit(pVnode->pMeta) < 0) {
+  code = tsdbCommit(pVnode->pTsdb, pInfo);
-    code = TSDB_CODE_FAILED;
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-  code = tsdbCommit(pVnode->pTsdb);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   if (VND_IS_RSMA(pVnode)) {
-    code = smaCommit(pVnode->pSma);
+    code = smaCommit(pVnode->pSma, pInfo);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
@@ -264,7 +296,7 @@ int vnodeCommit(SVnode *pVnode) {
   }
 
   // commit info
-  if (vnodeCommitInfo(dir, &info) < 0) {
+  if (vnodeCommitInfo(dir, &pInfo->info) < 0) {
     code = terrno;
     TSDB_CHECK_CODE(code, lino, _exit);
   }
@@ -277,19 +309,18 @@ int vnodeCommit(SVnode *pVnode) {
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  if (metaFinishCommit(pVnode->pMeta) < 0) {
+  if (metaFinishCommit(pVnode->pMeta, pInfo->txn) < 0) {
     code = terrno;
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  pVnode->state.committed = info.state.committed;
+  pVnode->state.committed = pInfo->info.state.committed;
 
   if (smaPostCommit(pVnode->pSma) < 0) {
     vError("vgId:%d, failed to post-commit sma since %s", TD_VID(pVnode), tstrerror(terrno));
     return -1;
   }
 
-  // apply the commit (TODO)
   // walEndSnapshot(pVnode->pWal);
   syncEndSnapshot(pVnode->sync);
 
@@ -318,20 +349,6 @@ void vnodeRollback(SVnode *pVnode) {
   (void)taosRemoveFile(tFName);
 }
 
-static int vnodeCommitImpl(void *arg) {
-  SVnode *pVnode = (SVnode *)arg;
-
-  // metaCommit(pVnode->pMeta);
-  tqCommit(pVnode->pTq);
-  // tsdbCommit(pVnode->pTsdb, );
-
-  // vnodeBufPoolRecycle(pVnode);
-  tsem_post(&(pVnode->canCommit));
-  return 0;
-}
-
-static FORCE_INLINE void vnodeWaitCommit(SVnode *pVnode) { tsem_wait(&pVnode->canCommit); }
-
 static int vnodeEncodeState(const void *pObj, SJson *pJson) {
   const SVState *pState = (SVState *)pObj;
 

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -242,14 +242,14 @@ _err:
   return NULL;
 }
 
-void vnodePreClose(SVnode *pVnode) { 
+void vnodePreClose(SVnode *pVnode) {
   vnodeQueryPreClose(pVnode);
-  vnodeSyncPreClose(pVnode); 
+  vnodeSyncPreClose(pVnode);
 }
 
 void vnodeClose(SVnode *pVnode) {
   if (pVnode) {
-    vnodeCommit(pVnode);
+    vnodeSyncCommit(pVnode);
     vnodeSyncClose(pVnode);
     vnodeQueryClose(pVnode);
     walClose(pVnode->pWal);

source/dnode/vnode/src/vnd/vnodeSnapshot.c

@@ -259,7 +259,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr
   pWriter->ever = ever;
 
   // commit it
-  code = vnodeCommit(pVnode);
+  code = vnodeSyncCommit(pVnode);
   if (code) {
     taosMemoryFree(pWriter);
     goto _err;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -292,7 +292,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
       vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp);
       break;
     case TDMT_VND_COMMIT:
-      goto _do_commit;
+      vnodeSyncCommit(pVnode);
+      vnodeBegin(pVnode);
+      goto _exit;
     default:
       vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType);
       return -1;
@@ -310,13 +312,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
 
   // commit if need
   if (vnodeShouldCommit(pVnode)) {
-  _do_commit:
     vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version);
-    // commit current change
+#if 0
-    if (vnodeCommit(pVnode) < 0) {
+    vnodeSyncCommit(pVnode);
-      vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno));
+#else
-      goto _err;
+    vnodeAsyncCommit(pVnode);
-    }
+#endif
 
     // start a new one
     if (vnodeBegin(pVnode) < 0) {
@@ -325,6 +326,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
     }
   }
 
+_exit:
   return 0;
 
 _err:

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -606,16 +606,7 @@ void vnodeSyncPreClose(SVnode *pVnode) {
   vInfo("vgId:%d, pre close sync", pVnode->config.vgId);
   syncLeaderTransfer(pVnode->sync);
   syncPreStop(pVnode->sync);
-#if 0
+
-  while (syncSnapshotRecving(pVnode->sync)) {
-    vInfo("vgId:%d, snapshot is recving", pVnode->config.vgId);
-    taosMsleep(300);
-  }
-  while (syncSnapshotSending(pVnode->sync)) {
-    vInfo("vgId:%d, snapshot is sending", pVnode->config.vgId);
-    taosMsleep(300);
-  }
-#endif
   taosThreadMutexLock(&pVnode->lock);
   if (pVnode->blocked) {
     vInfo("vgId:%d, post block after close sync", pVnode->config.vgId);

source/libs/function/src/builtins.c

@@ -1924,7 +1924,8 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int
 }
 
 static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
-  if (2 != LIST_LENGTH(pFunc->pParameterList)) {
+  int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
+  if (2 != numOfParams && 3 != numOfParams) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
@@ -1935,9 +1936,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // add database precision as param
   uint8_t dbPrec = pFunc->node.resType.precision;
-
   int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1));
   if (ret == TIME_UNIT_TOO_SMALL) {
     return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
@@ -1948,11 +1947,30 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
         "TIMETRUNCATE function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]");
   }
 
+  if (3 == numOfParams) {
+    uint8_t para3Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type;
+    if (!IS_INTEGER_TYPE(para3Type)) {
+      return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+    SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2);
+    if (pValue->datum.i != 0 && pValue->datum.i != 1) {
+      return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName);
+    }
+  }
+
+  // add database precision as param
+
   int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
 
+  // add client timezone as param
+  code = addTimezoneParam(pFunc->pParameterList);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
   pFunc->node.resType =
       (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP};
   return TSDB_CODE_SUCCESS;

source/libs/function/src/builtinsimpl.c

@@ -423,6 +423,8 @@ typedef struct SGroupKeyInfo {
     (_p).val = (_v);                \
   } while (0)
 
+static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst);
+
 bool functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
   if (pResultInfo->initialized) {
     return false;
@@ -457,11 +459,12 @@ int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
   SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
   SFirstLastRes*       pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
 
-  if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || pDBuf->ts > pSBuf->ts)) {
+  if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pSBuf, pDBuf, true)) {
-    memcpy(pDBuf->buf, pSBuf->buf, bytes);
+    pDBuf->hasResult = true;
-    pDBuf->ts = pSBuf->ts;
-    pDResInfo->numOfRes = 1;
   }
+
+  pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes);
+  pDResInfo->isNullRes &= pSResInfo->isNullRes;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1274,17 +1277,8 @@ int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
   SStddevRes*          pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
   int16_t              type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type;
 
-  if (IS_SIGNED_NUMERIC_TYPE(type)) {
+  stddevTransferInfo(pSBuf, pDBuf);
-    pDBuf->isum += pSBuf->isum;
+
-    pDBuf->quadraticISum += pSBuf->quadraticISum;
-  } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
-    pDBuf->usum += pSBuf->usum;
-    pDBuf->quadraticUSum += pSBuf->quadraticUSum;
-  } else {
-    pDBuf->dsum += pSBuf->dsum;
-    pDBuf->quadraticDSum += pSBuf->quadraticDSum;
-  }
-  pDBuf->count += pSBuf->count;
   pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes);
   pDResInfo->isNullRes &= pSResInfo->isNullRes;
   return TSDB_CODE_SUCCESS;
@@ -2289,16 +2283,15 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
   return TSDB_CODE_SUCCESS;
 }
 
-static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst,
+static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) {
-                                  int32_t rowIndex) {
   if (pOutput->hasResult) {
     if (isFirst) {
       if (pInput->ts > pOutput->ts) {
-        return;
+        return TSDB_CODE_FAILED;
       }
     } else {
       if (pInput->ts < pOutput->ts) {
-        return;
+        return TSDB_CODE_FAILED;
       }
     }
   }
@@ -2308,9 +2301,15 @@ static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, S
   pOutput->bytes = pInput->bytes;
 
   memcpy(pOutput->buf, pInput->buf, pOutput->bytes);
-  firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput);
+  return TSDB_CODE_SUCCESS;
+}
 
-  pOutput->hasResult = true;
+static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst,
+                                  int32_t rowIndex) {
+  if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pInput, pOutput, isFirst)) {
+    firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput);
+    pOutput->hasResult = true;
+  }
 }
 
 static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuery) {
@@ -2378,7 +2377,6 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   return 1;
 }
 
-// todo rewrite:
 int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
   SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx);
   SFirstLastRes*       pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
@@ -2387,11 +2385,12 @@ int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
   SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
   SFirstLastRes*       pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
 
-  if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || pDBuf->ts < pSBuf->ts)) {
+  if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pSBuf, pDBuf, false)) {
-    memcpy(pDBuf->buf, pSBuf->buf, bytes);
+    pDBuf->hasResult = true;
-    pDBuf->ts = pSBuf->ts;
-    pDResInfo->numOfRes = 1;
   }
+
+  pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes);
+  pDResInfo->isNullRes &= pSResInfo->isNullRes;
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/function/src/udfd.c

@@ -1019,8 +1019,8 @@ int main(int argc, char *argv[]) {
   }
 
   if (udfdInitLog() != 0) {
+    // ignore create log failed, because this error no matter
     printf("failed to start since init log error\n");
-    return -1;
   }
 
   if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {

source/libs/scalar/src/sclfunc.c

@@ -1174,12 +1174,35 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
 }
 
 /** Time functions **/
+static int64_t offsetFromTz(char *timezone, int64_t factor) {
+  char *minStr = &timezone[3];
+  int64_t minutes = taosStr2Int64(minStr, NULL, 10);
+  memset(minStr, 0, strlen(minStr));
+  int64_t hours = taosStr2Int64(timezone, NULL, 10);
+  int64_t seconds = hours * 3600 + minutes * 60;
+
+  return seconds * factor;
+
+}
+
 int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   int32_t type = GET_PARAM_TYPE(&pInput[0]);
 
   int64_t timeUnit, timePrec, timeVal = 0;
+  bool    ignoreTz = true;
+  char    timezone[20] = {0};
+
   GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData);
-  GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData);
+
+  int32_t timePrecIdx = 2, timeZoneIdx = 3;
+  if (inputNum == 5) {
+    timePrecIdx += 1;
+    timeZoneIdx += 1;
+    GET_TYPED_DATA(ignoreTz, bool, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData);
+  }
+
+  GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[timePrecIdx]), pInput[timePrecIdx].columnData->pData);
+  memcpy(timezone, varDataVal(pInput[timeZoneIdx].columnData->pData), varDataLen(pInput[timeZoneIdx].columnData->pData));
 
   int64_t factor = TSDB_TICK_PER_SECOND(timePrec);
   int64_t unit = timeUnit * 1000 / factor;
@@ -1294,13 +1317,29 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
       }
       case 86400000: { /* 1d */
         if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) {
-          timeVal = timeVal / 1000 / 86400 * 86400 * 1000;
+          if (ignoreTz) {
+            timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000)) % (86400L * 1000);
+          } else {
+            timeVal = timeVal / 1000 / 86400 * 86400 * 1000;
+          }
         } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) {
-          timeVal = timeVal / 1000000 / 86400 * 86400 * 1000000;
+          if (ignoreTz) {
+            timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000)) % (86400L * 1000000);
+          } else {
+            timeVal = timeVal / 1000000 / 86400 * 86400 * 1000000;
+          }
         } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
-          timeVal = timeVal / 1000000000 / 86400 * 86400 * 1000000000;
+          if (ignoreTz) {
+            timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000000)) % (86400L * 1000000000);
+          } else {
+            timeVal = timeVal / 1000000000 / 86400 * 86400 * 1000000000;
+          }
         } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) {
-          timeVal = timeVal * factor / factor / 86400 * 86400 * factor;
+          if (ignoreTz) {
+            timeVal = (timeVal - (timeVal + offsetFromTz(timezone, 1)) % (86400L)) * factor;
+          } else {
+            timeVal = timeVal * factor / factor / 86400 * 86400 * factor;
+          }
         } else {
           colDataAppendNULL(pOutput->columnData, i);
           continue;

source/libs/scheduler/inc/schInt.h

@@ -293,6 +293,7 @@ typedef struct SSchJob {
   void                *chkKillParam;
   SSchTask            *fetchTask;
   int32_t              errCode;
+  int32_t              redirectCode;
   SRWLatch             resLock;
   SExecResult          execRes;
   void                *fetchRes;  // TODO free it or not
@@ -331,6 +332,9 @@ extern SSchedulerMgmt schMgmt;
   ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \
    (!SCH_IS_DATA_BIND_QRY_TASK(_task)))
 
+#define SCH_UPDATE_REDICT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code)
+#define SCH_GET_REDICT_CODE(job, _code) (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode)
+
 #define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st)
 #define SCH_GET_TASK_STATUS(task)     atomic_load_8(&(task)->status)
 #define SCH_GET_TASK_STATUS_STR(task) jobTaskStatusStr(SCH_GET_TASK_STATUS(task))

source/libs/scheduler/src/schJob.c

@@ -481,6 +481,10 @@ _return:
 }
 
 int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) {
+  if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) {
+    return TSDB_CODE_SCH_IGNORE_ERROR;
+  }
+  
   schUpdateJobErrCode(pJob, errCode);
 
   int32_t code = atomic_load_32(&pJob->errCode);

source/libs/scheduler/src/schTask.c

@@ -340,7 +340,7 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) {
+int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, int32_t rspCode) {
   SSchRedirectCtx *pCtx = &pTask->redirectCtx;
   if (!pCtx->inRedirect) {
     pCtx->inRedirect = true;
@@ -380,7 +380,7 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet)
     if (lastTime > tsMaxRetryWaitTime) {
       SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d",
                     nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes);
-      SCH_ERR_RET(TSDB_CODE_TIMEOUT_ERROR);
+      SCH_ERR_RET(SCH_GET_REDICT_CODE(pJob, rspCode));
     }
 
     pCtx->periodMs *= tsRedirectFactor;
@@ -415,7 +415,11 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
     pTask->retryTimes = 0;
   }
 
-  SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL));
+  if (!NO_RET_REDIRECT_ERROR(rspCode)) {
+    SCH_UPDATE_REDICT_CODE(pJob, rspCode);
+  }
+
+  SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL, rspCode));
 
   pTask->waitRetry = true;
 

source/libs/stream/src/streamMeta.c

@@ -69,8 +69,8 @@ _err:
 }
 
 void streamMetaClose(SStreamMeta* pMeta) {
-  tdbCommit(pMeta->db, &pMeta->txn);
+  tdbCommit(pMeta->db, pMeta->txn);
-  tdbPostCommit(pMeta->db, &pMeta->txn);
+  tdbPostCommit(pMeta->db, pMeta->txn);
   tdbTbClose(pMeta->pTaskDb);
   tdbTbClose(pMeta->pCheckpointDb);
   tdbClose(pMeta->db);
@@ -115,7 +115,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg,
     goto FAIL;
   }
 
-  if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, &pMeta->txn) < 0) {
+  if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) {
     taosHashRemove(pMeta->pTasks, &pTask->taskId, sizeof(int32_t));
     ASSERT(0);
     goto FAIL;
@@ -152,7 +152,7 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) {
   tEncodeSStreamTask(&encoder, pTask);
   tEncoderClear(&encoder);
 
-  if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), buf, len, &pMeta->txn) < 0) {
+  if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) {
     ASSERT(0);
     return -1;
   }
@@ -220,42 +220,35 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
 }
 
 int32_t streamMetaBegin(SStreamMeta* pMeta) {
-  if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
+  if (tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL,
-      0) {
+               TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
-    return -1;
-  }
-
-  if (tdbBegin(pMeta->db, &pMeta->txn) < 0) {
     return -1;
   }
   return 0;
 }
 
 int32_t streamMetaCommit(SStreamMeta* pMeta) {
-  if (tdbCommit(pMeta->db, &pMeta->txn) < 0) {
+  if (tdbCommit(pMeta->db, pMeta->txn) < 0) {
     return -1;
   }
-  memset(&pMeta->txn, 0, sizeof(TXN));
+  if (tdbPostCommit(pMeta->db, pMeta->txn) < 0) {
-  if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
-      0) {
     return -1;
   }
-  if (tdbBegin(pMeta->db, &pMeta->txn) < 0) {
+
+  if (tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL,
+               TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
     return -1;
   }
   return 0;
 }
 
 int32_t streamMetaAbort(SStreamMeta* pMeta) {
-  if (tdbAbort(pMeta->db, &pMeta->txn) < 0) {
+  if (tdbAbort(pMeta->db, pMeta->txn) < 0) {
     return -1;
   }
-  memset(&pMeta->txn, 0, sizeof(TXN));
+
-  if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
+  if (tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL,
-      0) {
+               TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
-    return -1;
-  }
-  if (tdbBegin(pMeta->db, &pMeta->txn) < 0) {
     return -1;
   }
   return 0;

source/libs/stream/src/streamState.c

@@ -179,8 +179,8 @@ _err:
 }
 
 void streamStateClose(SStreamState* pState) {
-  tdbCommit(pState->pTdbState->db, &pState->pTdbState->txn);
+  tdbCommit(pState->pTdbState->db, pState->pTdbState->txn);
-  tdbPostCommit(pState->pTdbState->db, &pState->pTdbState->txn);
+  tdbPostCommit(pState->pTdbState->db, pState->pTdbState->txn);
   tdbTbClose(pState->pTdbState->pStateDb);
   tdbTbClose(pState->pTdbState->pFuncStateDb);
   tdbTbClose(pState->pTdbState->pFillStateDb);
@@ -192,71 +192,61 @@ void streamStateClose(SStreamState* pState) {
 }
 
 int32_t streamStateBegin(SStreamState* pState) {
-  if (tdbTxnOpen(&pState->pTdbState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL,
+  if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL,
-                 TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
+               TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
-    return -1;
+    tdbAbort(pState->pTdbState->db, pState->pTdbState->txn);
-  }
-
-  if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn) < 0) {
-    tdbTxnClose(&pState->pTdbState->txn);
     return -1;
   }
   return 0;
 }
 
 int32_t streamStateCommit(SStreamState* pState) {
-  if (tdbCommit(pState->pTdbState->db, &pState->pTdbState->txn) < 0) {
+  if (tdbCommit(pState->pTdbState->db, pState->pTdbState->txn) < 0) {
     return -1;
   }
-  if (tdbPostCommit(pState->pTdbState->db, &pState->pTdbState->txn) < 0) {
+  if (tdbPostCommit(pState->pTdbState->db, pState->pTdbState->txn) < 0) {
     return -1;
   }
-  memset(&pState->pTdbState->txn, 0, sizeof(TXN));
+
-  if (tdbTxnOpen(&pState->pTdbState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL,
+  if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL,
-                 TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
+               TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
-    return -1;
-  }
-  if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn) < 0) {
     return -1;
   }
   return 0;
 }
 
 int32_t streamStateAbort(SStreamState* pState) {
-  if (tdbAbort(pState->pTdbState->db, &pState->pTdbState->txn) < 0) {
+  if (tdbAbort(pState->pTdbState->db, pState->pTdbState->txn) < 0) {
     return -1;
   }
-  memset(&pState->pTdbState->txn, 0, sizeof(TXN));
+
-  if (tdbTxnOpen(&pState->pTdbState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL,
+  if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL,
-                 TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
+               TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
-    return -1;
-  }
-  if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn) < 0) {
     return -1;
   }
   return 0;
 }
 
 int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen) {
-  return tdbTbUpsert(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, &pState->pTdbState->txn);
+  return tdbTbUpsert(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, pState->pTdbState->txn);
 }
 int32_t streamStateFuncGet(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen) {
   return tdbTbGet(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), pVal, pVLen);
 }
 
 int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) {
-  return tdbTbDelete(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), &pState->pTdbState->txn);
+  return tdbTbDelete(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), pState->pTdbState->txn);
 }
 
 // todo refactor
 int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) {
   SStateKey sKey = {.key = *key, .opNum = pState->number};
-  return tdbTbUpsert(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, &pState->pTdbState->txn);
+  return tdbTbUpsert(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, pState->pTdbState->txn);
 }
 
 // todo refactor
 int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) {
-  return tdbTbUpsert(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), value, vLen, &pState->pTdbState->txn);
+  return tdbTbUpsert(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), value, vLen, pState->pTdbState->txn);
 }
 
 // todo refactor
@@ -273,7 +263,7 @@ int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal
 // todo refactor
 int32_t streamStateDel(SStreamState* pState, const SWinKey* key) {
   SStateKey sKey = {.key = *key, .opNum = pState->number};
-  return tdbTbDelete(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), &pState->pTdbState->txn);
+  return tdbTbDelete(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), pState->pTdbState->txn);
 }
 
 int32_t streamStateClear(SStreamState* pState) {
@@ -297,7 +287,7 @@ void streamStateSetNumber(SStreamState* pState, int32_t number) { pState->number
 
 // todo refactor
 int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) {
-  return tdbTbDelete(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), &pState->pTdbState->txn);
+  return tdbTbDelete(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), pState->pTdbState->txn);
 }
 
 int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) {
@@ -531,7 +521,7 @@ void streamFreeVal(void* val) { tdbFree(val); }
 int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) {
   SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
   return tdbTbUpsert(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen,
-                     &pState->pTdbState->txn);
+                     pState->pTdbState->txn);
 }
 
 int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) {
@@ -554,7 +544,7 @@ int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVa
 
 int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) {
   SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
-  return tdbTbDelete(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), &pState->pTdbState->txn);
+  return tdbTbDelete(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), pState->pTdbState->txn);
 }
 
 SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key) {
@@ -833,7 +823,7 @@ _end:
 
 int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) {
   tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN,
-              &pState->pTdbState->txn);
+              pState->pTdbState->txn);
   return 0;
 }
 

source/libs/sync/inc/syncEnv.h

@@ -22,14 +22,12 @@ extern "C" {
 
 #include "syncInt.h"
 
-#define TIMER_MAX_MS         0x7FFFFFFF
+#define TIMER_MAX_MS       0x7FFFFFFF
-#define ENV_TICK_TIMER_MS    1000
+#define ENV_TICK_TIMER_MS  1000
-#define PING_TIMER_MS        5000
+#define PING_TIMER_MS      5000
-#define ELECT_TIMER_MS_MIN   2500
+#define ELECT_TIMER_MS_MIN 2500
-#define ELECT_TIMER_MS_MAX   (ELECT_TIMER_MS_MIN * 2)
+#define HEARTBEAT_TIMER_MS 1000
-#define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN)
+#define HEARTBEAT_TICK_NUM 20
-#define HEARTBEAT_TIMER_MS   1000
-#define HEARTBEAT_TICK_NUM   20
 
 typedef struct SSyncEnv {
   uint8_t isStart;

source/libs/sync/src/syncMain.c

@@ -1025,8 +1025,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
   }
   // timer ms init
   pSyncNode->pingBaseLine = PING_TIMER_MS;
-  pSyncNode->electBaseLine = ELECT_TIMER_MS_MIN;
+  pSyncNode->electBaseLine = tsElectInterval;
-  pSyncNode->hbBaseLine = HEARTBEAT_TIMER_MS;
+  pSyncNode->hbBaseLine = tsHeartbeatInterval;
 
   // init ping timer
   pSyncNode->pPingTimer = NULL;
@@ -1221,6 +1221,26 @@ int32_t syncNodeStartStandBy(SSyncNode* pSyncNode) {
 }
 
 void syncNodePreClose(SSyncNode* pSyncNode) {
+  if (pSyncNode != NULL && pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpApplyQueueItems != NULL) {
+    while (1) {
+      int32_t aqItems = pSyncNode->pFsm->FpApplyQueueItems(pSyncNode->pFsm);
+      sTrace("vgId:%d, pre close, %d items in apply queue", pSyncNode->vgId, aqItems);
+      if (aqItems == 0 || aqItems == -1) {
+        break;
+      }
+      taosMsleep(20);
+    }
+  }
+
+  if (pSyncNode->pNewNodeReceiver != NULL) {
+    if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) {
+      snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver);
+    }
+
+    snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver);
+    pSyncNode->pNewNodeReceiver = NULL;
+  }
+
   // stop elect timer
   syncNodeStopElectTimer(pSyncNode);
 
@@ -2384,7 +2404,7 @@ bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) {
       continue;
     }
 
-    if (tsNow - recvTime > SYNC_HEART_TIMEOUT_MS) {
+    if (tsNow - recvTime > tsHeartbeatTimeout) {
       toCount++;
     }
   }

source/libs/sync/src/syncSnapshot.c

@@ -548,6 +548,7 @@ _START_RECEIVER:
       sNTrace(pSyncNode, "snapshot receiver pre waitting for true time, now:%" PRId64 ", stime:%" PRId64, timeNow,
               pMsg->startTime);
       taosMsleep(10);
+      timeNow = taosGetTimestampMs();
     }
 
     if (snapshotReceiverIsStart(pReceiver)) {
@@ -883,9 +884,9 @@ int32_t syncNodeOnSnapshotReply(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) {
       // receive ack is finish, close sender
       if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) {
         snapshotSenderStop(pSender, true);
-        SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId);
+        SSyncLogReplMgr *pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId);
         if (pMgr) {
-            syncLogReplMgrReset(pMgr);
+          syncLogReplMgrReset(pMgr);
         }
         return 0;
       }

source/libs/tdb/inc/tdb.h

@@ -17,6 +17,7 @@
 #define _TD_TDB_H_
 
 #include "os.h"
+#include "tdbOs.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -33,7 +34,8 @@ typedef struct STxn TXN;
 // TDB
 int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback);
 int32_t tdbClose(TDB *pDb);
-int32_t tdbBegin(TDB *pDb, TXN *pTxn);
+int32_t tdbBegin(TDB *pDb, TXN **pTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg,
+                 int flags);
 int32_t tdbCommit(TDB *pDb, TXN *pTxn);
 int32_t tdbPostCommit(TDB *pDb, TXN *pTxn);
 int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn);
@@ -77,12 +79,16 @@ int32_t tdbTxnClose(TXN *pTxn);
 // other
 void tdbFree(void *);
 
+typedef struct hashset_st *hashset_t;
+
 struct STxn {
   int     flags;
   int64_t txnId;
   void *(*xMalloc)(void *, size_t);
   void (*xFree)(void *, void *);
-  void *xArg;
+  void     *xArg;
+  tdb_fd_t  jfd;
+  hashset_t jPageSet;
 };
 
 // error code

source/libs/tdb/src/db/tdbBtree.c

@@ -69,7 +69,7 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
 static int tdbBtcMoveDownward(SBTC *pBtc);
 static int tdbBtcMoveUpward(SBTC *pBtc);
 
-int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr,
+int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, TDB *pEnv,
                  SBTree **ppBt) {
   SBTree *pBt;
   int     ret;
@@ -106,22 +106,26 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
   if (pgno == 0) {
     // fetch page & insert into main db
     SPage *pPage;
-    TXN    txn;
+    TXN   *txn;
-    tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
 
-    pPager->inTran = 1;
+    ret = tdbBegin(pEnv, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+    if (ret < 0) {
+      return -1;
+    }
 
     SBtreeInitPageArg zArg;
     zArg.flags = 0x1 | 0x2;  // root leaf node;
     zArg.pBt = pBt;
-    ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, &txn);
+    ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, txn);
     if (ret < 0) {
+      tdbAbort(pEnv, txn);
       return -1;
     }
 
     ret = tdbPagerWrite(pPager, pPage);
     if (ret < 0) {
       tdbError("failed to write page since %s", terrstr());
+      tdbAbort(pEnv, txn);
       return -1;
     }
 
@@ -130,18 +134,18 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg
       pBt->info.nLevel = 1;
       pBt->info.nData = 0;
       pBt->tbname = (char *)tbname;
-      // ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pgno, sizeof(SPgno), &txn);
+
-      ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pBt->info, sizeof(pBt->info), &txn);
+      ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pBt->info, sizeof(pBt->info), txn);
       if (ret < 0) {
+        tdbAbort(pEnv, txn);
         return -1;
       }
     }
 
-    // tdbUnrefPage(pPage);
+    tdbPCacheRelease(pPager->pCache, pPage, txn);
-    tdbPCacheRelease(pPager->pCache, pPage, &txn);
+
-    tdbCommit(pPager->pEnv, &txn);
+    tdbCommit(pPager->pEnv, txn);
-    tdbPostCommit(pPager->pEnv, &txn);
+    tdbPostCommit(pPager->pEnv, txn);
-    tdbTxnClose(&txn);
   }
 
   ASSERT(pgno != 0);
@@ -1534,10 +1538,21 @@ int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn) {
   memset(&pBtc->coder, 0, sizeof(SCellDecoder));
 
   if (pTxn == NULL) {
-    pBtc->pTxn = &pBtc->txn;
+    TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn));
-    tdbTxnOpen(pBtc->pTxn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0);
+    if (!pTxn) {
+      return -1;
+    }
+
+    if (tdbTxnOpen(pTxn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
+      tdbOsFree(pTxn);
+      return -1;
+    }
+
+    pBtc->pTxn = pTxn;
+    pBtc->freeTxn = 1;
   } else {
     pBtc->pTxn = pTxn;
+    pBtc->freeTxn = 0;
   }
 
   return 0;
@@ -2231,6 +2246,10 @@ int tdbBtcClose(SBTC *pBtc) {
     tdbFree(pBtc->coder.pVal);
   }
 
+  if (pBtc->freeTxn) {
+    tdbTxnClose(pBtc->pTxn);
+  }
+
   return 0;
 }
 

source/libs/tdb/src/db/tdbDb.c

@@ -99,19 +99,37 @@ int tdbClose(TDB *pDb) {
 
 int32_t tdbAlter(TDB *pDb, int pages) { return tdbPCacheAlter(pDb->pCache, pages); }
 
-int32_t tdbBegin(TDB *pDb, TXN *pTxn) {
+int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg,
+                 int flags) {
   SPager *pPager;
   int     ret;
+  int64_t txnId = ++pDb->txnId;
+  if (txnId == INT64_MAX) {
+    pDb->txnId = 0;
+  }
+
+  TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn));
+  if (!pTxn) {
+    return -1;
+  }
+
+  if (tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags) < 0) {
+    tdbOsFree(pTxn);
+    return -1;
+  }
 
   for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
     ret = tdbPagerBegin(pPager, pTxn);
     if (ret < 0) {
       tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
                pTxn->txnId);
+      tdbTxnClose(pTxn);
       return -1;
     }
   }
 
+  *ppTxn = pTxn;
+
   return 0;
 }
 
@@ -144,6 +162,8 @@ int32_t tdbPostCommit(TDB *pDb, TXN *pTxn) {
     }
   }
 
+  tdbTxnClose(pTxn);
+
   return 0;
 }
 
@@ -176,6 +196,8 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) {
     }
   }
 
+  tdbTxnClose(pTxn);
+
   return 0;
 }
 

source/libs/tdb/src/db/tdbPCache.c

@@ -184,9 +184,9 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
   //        TDB_PAGE_PGNO(pPage), pPage, nRef);
 
   if (pPage) {
-    tdbDebug("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef);
+    tdbTrace("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef);
   } else {
-    tdbDebug("pcache/fetch page %p", pPage);
+    tdbTrace("pcache/fetch page %p", pPage);
   }
 
   return pPage;
@@ -202,7 +202,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
 
   tdbPCacheLock(pCache);
   nRef = tdbUnrefPage(pPage);
-  tdbDebug("pcache/release page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef);
+  tdbTrace("pcache/release page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef);
   if (nRef == 0) {
     // test the nRef again to make sure
     // it is safe th handle the page
@@ -221,8 +221,6 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
     // }
   }
   tdbPCacheUnlock(pCache);
-  // printf("thread %" PRId64 " relas page %d pgno %d pPage %p nRef %d\n", taosGetSelfPthreadId(), pPage->id,
-  //        TDB_PAGE_PGNO(pPage), pPage, nRef);
 }
 
 int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->szPage; }
@@ -335,8 +333,7 @@ static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) {
 
     pCache->nRecyclable--;
 
-    // printf("pin page %d pgno %d pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage);
+    tdbTrace("pcache/pin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
-    tdbDebug("pcache/pin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
   }
 }
 
@@ -349,7 +346,7 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
 
   ASSERT(pPage->pLruNext == NULL);
 
-  tdbDebug("pCache:%p unpin page %p/%d/%d, nPages:%d", pCache, pPage, TDB_PAGE_PGNO(pPage), pPage->id, pCache->nPages);
+  tdbTrace("pCache:%p unpin page %p/%d/%d, nPages:%d", pCache, pPage, TDB_PAGE_PGNO(pPage), pPage->id, pCache->nPages);
   if (pPage->id < pCache->nPages) {
     pPage->pLruPrev = &(pCache->lru);
     pPage->pLruNext = pCache->lru.pLruNext;
@@ -359,9 +356,9 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
     pCache->nRecyclable++;
 
     // printf("unpin page %d pgno %d pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage);
-    tdbDebug("pcache/unpin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
+    tdbTrace("pcache/unpin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
   } else {
-    tdbDebug("pcache destroy page: %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
+    tdbTrace("pcache destroy page: %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
 
     tdbPCacheRemovePageFromHash(pCache, pPage);
     tdbPageDestroy(pPage, tdbDefaultFree, NULL);
@@ -381,7 +378,7 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) {
     // printf("rmv page %d to hash, pgno %d, pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage);
   }
 
-  tdbDebug("pcache/remove page %p/%d/%d from hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h);
+  tdbTrace("pcache/remove page %p/%d/%d from hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h);
 }
 
 static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) {
@@ -392,8 +389,7 @@ static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) {
 
   pCache->nPage++;
 
-  // printf("add page %d to hash, pgno %d, pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage);
+  tdbTrace("pcache/add page %p/%d/%d to hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h);
-  tdbDebug("pcache/add page %p/%d/%d to hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h);
 }
 
 static int tdbPCacheOpenImpl(SPCache *pCache) {

source/libs/tdb/src/db/tdbPage.c

@@ -69,18 +69,19 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t)
 
   *ppPage = pPage;
 
-  tdbDebug("page/create: %p %p", pPage, xMalloc);
+  tdbTrace("page/create: %p/%d %p", pPage, pPage->id, xMalloc);
   return 0;
 }
 
 int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) {
   u8 *ptr;
 
-  tdbDebug("page/destroy: %p %p", pPage, xFree);
+  tdbTrace("page/destroy: %p/%d %p", pPage, pPage->id, xFree);
+  ASSERT(!pPage->isDirty);
   ASSERT(xFree);
 
   for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) {
-    tdbDebug("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage);
+    tdbTrace("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage);
     tdbOsFree(pPage->apOvfl[iOvfl]);
   }
 
@@ -91,7 +92,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg)
 }
 
 void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) {
-  tdbDebug("page/zero: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize);
+  tdbTrace("page/zero: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize);
   pPage->pPageHdr = pPage->pData + szAmHdr;
   TDB_PAGE_NCELLS_SET(pPage, 0);
   TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr));
@@ -108,7 +109,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell
 }
 
 void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) {
-  tdbDebug("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize);
+  tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize);
   pPage->pPageHdr = pPage->pData + szAmHdr;
   pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage);
   pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage);
@@ -153,7 +154,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl
     pNewCell = (SCell *)tdbOsMalloc(szCell);
     memcpy(pNewCell, pCell, szCell);
 
-    tdbDebug("tdbPage/insert/new ovfl cell: %p/%p", pNewCell, pPage);
+    tdbTrace("tdbPage/insert/new ovfl cell: %p/%p", pNewCell, pPage);
 
     pPage->apOvfl[iOvfl] = pNewCell;
     pPage->aiOvfl[iOvfl] = idx;
@@ -203,7 +204,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) {
     if (pPage->aiOvfl[iOvfl] == idx) {
       // remove the over flow cell
       tdbOsFree(pPage->apOvfl[iOvfl]);
-      tdbDebug("tdbPage/drop/free ovfl cell: %p", pPage->apOvfl[iOvfl]);
+      tdbTrace("tdbPage/drop/free ovfl cell: %p", pPage->apOvfl[iOvfl]);
       for (; (++iOvfl) < pPage->nOverflow;) {
         pPage->aiOvfl[iOvfl - 1] = pPage->aiOvfl[iOvfl] - 1;
         pPage->apOvfl[iOvfl - 1] = pPage->apOvfl[iOvfl];
@@ -256,7 +257,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
       int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL);
       pNewCell = (SCell *)tdbOsMalloc(szCell);
       memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell);
-      tdbDebug("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage);
+      tdbTrace("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage);
     }
 
     pToPage->apOvfl[iOvfl] = pNewCell;

source/libs/tdb/src/db/tdbPager.c

@@ -142,7 +142,7 @@ int hashset_contains(hashset_t set, void *item) {
 static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *, int), void *arg,
                             u8 loadPage);
 static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage);
-static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage);
+static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage);
 
 static FORCE_INLINE int32_t pageCmpFn(const SRBTreeNode *lhs, const SRBTreeNode *rhs) {
   SPage *pPageL = (SPage *)(((uint8_t *)lhs) - offsetof(SPage, node));
@@ -219,85 +219,27 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
 
 int tdbPagerClose(SPager *pPager) {
   if (pPager) {
+    /*
     if (pPager->inTran) {
       tdbOsClose(pPager->jfd);
     }
+    */
     tdbOsClose(pPager->fd);
     tdbOsFree(pPager);
   }
   return 0;
 }
-/*
-int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) {
-  SPgno  pgno;
-  SPage *pPage;
-  int    ret;
 
-  if (pPager->dbOrigSize > 0) {
-    pgno = 1;
-  } else {
-    pgno = 0;
-  }
-
-  {
-    // TODO: try to search the main DB to get the page number
-    // pgno = 0;
-  }
-
-  if (pgno == 0 && toCreate) {
-    // allocate a new child page
-    TXN txn;
-    tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0);
-
-    pPager->inTran = 1;
-
-    SBtreeInitPageArg zArg;
-    zArg.flags = 0x1 | 0x2;  // root leaf node;
-    zArg.pBt = pBt;
-    ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, &txn);
-    if (ret < 0) {
-      return -1;
-    }
-
-    //    ret = tdbPagerAllocPage(pPager, &pPage, &pgno);
-    // if (ret < 0) {
-    //  return -1;
-    //}
-
-    // TODO: Need to zero the page
-
-    ret = tdbPagerWrite(pPager, pPage);
-    if (ret < 0) {
-      tdbError("failed to write page since %s", terrstr());
-      return -1;
-    }
-
-    tdbTxnClose(&txn);
-  }
-
-  *ppgno = pgno;
-  return 0;
-}
-*/
 int tdbPagerWrite(SPager *pPager, SPage *pPage) {
   int     ret;
   SPage **ppPage;
 
-  ASSERT(pPager->inTran);
+  // ASSERT(pPager->inTran);
-#if 0
-  if (pPager->inTran == 0) {
-    ret = tdbPagerBegin(pPager);
-    if (ret < 0) {
-      return -1;
-    }
-  }
-#endif
-
   if (pPage->isDirty) return 0;
 
   // ref page one more time so the page will not be release
   tdbRefPage(pPage);
-  tdbDebug("pager/mdirty page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
+  tdbTrace("pager/mdirty page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
 
   // Set page as dirty
   pPage->isDirty = 1;
@@ -322,15 +264,16 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
 
   // Write page to journal if neccessary
   if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize &&
-      (pPager->jPageSet == NULL || !hashset_contains(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) {
+      (pPager->pActiveTxn->jPageSet == NULL ||
+       !hashset_contains(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) {
     ret = tdbPagerWritePageToJournal(pPager, pPage);
     if (ret < 0) {
       tdbError("failed to write page to journal since %s", tstrerror(terrno));
       return -1;
     }
 
-    if (pPager->jPageSet) {
+    if (pPager->pActiveTxn->jPageSet) {
-      hashset_add(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage)));
+      hashset_add(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage)));
     }
   }
 
@@ -338,23 +281,28 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
 }
 
 int tdbPagerBegin(SPager *pPager, TXN *pTxn) {
+  /*
   if (pPager->inTran) {
     return 0;
   }
-
+  */
   // Open the journal
-  pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
+  char jTxnFileName[TDB_FILENAME_LEN];
-  if (TDB_FD_INVALID(pPager->jfd)) {
+  sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId);
+  pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755);
+  if (TDB_FD_INVALID(pTxn->jfd)) {
     tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
-  pPager->jPageSet = hashset_create();
+  pTxn->jPageSet = hashset_create();
+
+  pPager->pActiveTxn = pTxn;
   // TODO: write the size of the file
-
+  /*
   pPager->inTran = 1;
-
+  */
   return 0;
 }
 
@@ -363,9 +311,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
   int    ret;
 
   // sync the journal file
-  ret = tdbOsFSync(pPager->jfd);
+  ret = tdbOsFSync(pTxn->jfd);
   if (ret < 0) {
-    tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+    tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -377,7 +325,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
     pPage = (SPage *)pNode;
 
     ASSERT(pPage->nOverflow == 0);
-    ret = tdbPagerWritePageToDB(pPager, pPage);
+    ret = tdbPagerPWritePageToDB(pPager, pPage);
     if (ret < 0) {
       tdbError("failed to write page to db since %s", tstrerror(terrno));
       return -1;
@@ -395,8 +343,8 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
     pPage->isDirty = 0;
 
     tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
-    if (pPager->jPageSet) {
+    if (pTxn->jPageSet) {
-      hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage)));
+      hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage)));
     }
     tdbPCacheRelease(pPager->pCache, pPage, pTxn);
   }
@@ -415,35 +363,39 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
 }
 
 int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) {
+  char jTxnFileName[TDB_FILENAME_LEN];
+  sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId);
+
   // remove the journal file
-  if (tdbOsClose(pPager->jfd) < 0) {
+  if (tdbOsClose(pTxn->jfd) < 0) {
-    tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName);
+    tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
-  if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) {
+  if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) {
-    tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName);
+    tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
-  if (pPager->jPageSet) {
+  if (pTxn->jPageSet) {
-    hashset_destroy(pPager->jPageSet);
+    hashset_destroy(pTxn->jPageSet);
   }
-  pPager->inTran = 0;
+  // pPager->inTran = 0;
 
   return 0;
 }
 
 int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
   SPage *pPage;
+  SPgno  maxPgno = pPager->dbOrigSize;
   int    ret;
 
   // sync the journal file
-  ret = tdbOsFSync(pPager->jfd);
+  ret = tdbOsFSync(pTxn->jfd);
   if (ret < 0) {
-    tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+    tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -454,7 +406,12 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
   while ((pNode = tRBTreeIterNext(&iter)) != NULL) {
     pPage = (SPage *)pNode;
     if (pPage->isLocal) continue;
-    ret = tdbPagerWritePageToDB(pPager, pPage);
+
+    SPgno pgno = TDB_PAGE_PGNO(pPage);
+    if (pgno > maxPgno) {
+      maxPgno = pgno;
+    }
+    ret = tdbPagerPWritePageToDB(pPager, pPage);
     if (ret < 0) {
       tdbError("failed to write page to db since %s", tstrerror(terrno));
       return -1;
@@ -462,7 +419,8 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
   }
 
   tdbTrace("tdbttl commit:%p, %d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize);
-  pPager->dbOrigSize = pPager->dbFileSize;
+  pPager->dbOrigSize = maxPgno;
+  //  pPager->dbOrigSize = pPager->dbFileSize;
 
   // release the page
   iter = tRBTreeIterCreate(&pPager->rbt, 1);
@@ -474,6 +432,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
     tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
     tdbPCacheRelease(pPager->pCache, pPage, pTxn);
   }
+
   /*
   tdbTrace("reset dirty tree: %p", &pPager->rbt);
   tRBTreeCreate(&pPager->rbt, pageCmpFn);
@@ -495,15 +454,15 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
   SPgno  journalSize = 0;
   int    ret;
 
-  // 0, sync the journal file
+  // sync the journal file
-  ret = tdbOsFSync(pPager->jfd);
+  ret = tdbOsFSync(pTxn->jfd);
   if (ret < 0) {
-    tdbError("failed to fsync jfd due to %s. file:%s", strerror(errno), pPager->jFileName);
+    tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
-  tdb_fd_t jfd = pPager->jfd;
+  tdb_fd_t jfd = pTxn->jfd;
 
   ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize);
   if (ret < 0) {
@@ -567,7 +526,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
     pPage->isDirty = 0;
 
     tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
-    hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage)));
+    hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage)));
     tdbPCacheRelease(pPager->pCache, pPage, pTxn);
   }
 
@@ -575,11 +534,24 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
   tRBTreeCreate(&pPager->rbt, pageCmpFn);
 
   // 4, remove the journal file
-  tdbOsClose(pPager->jfd);
+  if (tdbOsClose(pTxn->jfd) < 0) {
-  (void)tdbOsRemove(pPager->jFileName);
+    tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId);
-  hashset_destroy(pPager->jPageSet);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
 
-  pPager->inTran = 0;
+  char jTxnFileName[TDB_FILENAME_LEN];
+  sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId);
+
+  if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) {
+    tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  hashset_destroy(pTxn->jPageSet);
+
+  // pPager->inTran = 0;
 
   return 0;
 }
@@ -604,7 +576,7 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) {
     if (pgno > maxPgno) {
       maxPgno = pgno;
     }
-    ret = tdbPagerWritePageToDB(pPager, pPage);
+    ret = tdbPagerPWritePageToDB(pPager, pPage);
     if (ret < 0) {
       tdbError("failed to write page to db since %s", tstrerror(terrno));
       return -1;
@@ -802,17 +774,18 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) {
 
   pgno = TDB_PAGE_PGNO(pPage);
 
-  ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno));
+  ret = tdbOsWrite(pPager->pActiveTxn->jfd, &pgno, sizeof(pgno));
   if (ret < 0) {
-    tdbError("failed to write pgno due to %s. file:%s, pgno:%u", strerror(errno), pPager->jFileName, pgno);
+    tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(errno), pPager->jFileName,
+             pgno, pPager->pActiveTxn->txnId);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
-  ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize);
+  ret = tdbOsWrite(pPager->pActiveTxn->jfd, pPage->pData, pPage->pageSize);
   if (ret < 0) {
-    tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName,
+    tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(errno),
-             (long)pPage->pageSize);
+             pPager->jFileName, pPage->pageSize, pPager->pActiveTxn->txnId);
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -820,13 +793,6 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) {
   return 0;
 }
 /*
-struct TdFile {
-  TdThreadRwlock rwlock;
-  int            refId;
-  int            fd;
-  FILE          *fp;
-} TdFile;
-*/
 static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
   i64 offset;
   int ret;
@@ -846,16 +812,32 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
     return -1;
   }
 
-  // pwrite(pPager->fd->fd, pPage->pData, pPage->pageSize, offset);
+  return 0;
+}
+*/
+static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) {
+  i64 offset;
+  int ret;
+
+  offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
+
+  ret = tdbOsPWrite(pPager->fd, pPage->pData, pPage->pageSize, offset);
+  if (ret < 0) {
+    tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName,
+             pPage->pageSize);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
   return 0;
 }
 
-int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
+static int tdbPagerRestore(SPager *pPager, SBTree *pBt, const char *jFileName) {
   int   ret = 0;
   SPgno journalSize = 0;
   u8   *pageBuf = NULL;
 
-  tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755);
+  tdb_fd_t jfd = tdbOsOpen(jFileName, TDB_O_RDWR, 0755);
   if (jfd == NULL) {
     return 0;
   }
@@ -928,12 +910,50 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) {
   return 0;
 }
 
-int tdbPagerRollback(SPager *pPager) {
+int tdbPagerRestoreJournals(SPager *pPager, SBTree *pBt) {
-  if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) {
+  tdbDirEntryPtr pDirEntry;
-    tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName);
+  tdbDirPtr      pDir = taosOpenDir(pPager->pEnv->dbName);
-    terrno = TAOS_SYSTEM_ERROR(errno);
+  if (pDir == NULL) {
+    tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno));
     return -1;
   }
 
+  while ((pDirEntry = tdbReadDir(pDir)) != NULL) {
+    char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry));
+    if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) {
+      if (tdbPagerRestore(pPager, pBt, name) < 0) {
+        tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), name);
+        return -1;
+      }
+    }
+  }
+
+  tdbCloseDir(&pDir);
+
+  return 0;
+}
+
+int tdbPagerRollback(SPager *pPager) {
+  tdbDirEntryPtr pDirEntry;
+  tdbDirPtr      pDir = taosOpenDir(pPager->pEnv->dbName);
+  if (pDir == NULL) {
+    tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno));
+    return -1;
+  }
+
+  while ((pDirEntry = tdbReadDir(pDir)) != NULL) {
+    char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry));
+
+    if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) {
+      if (tdbOsRemove(name) < 0 && errno != ENOENT) {
+        tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name);
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        return -1;
+      }
+    }
+  }
+
+  tdbCloseDir(&pDir);
+
   return 0;
 }

source/libs/tdb/src/db/tdbTable.c

@@ -108,7 +108,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
   ASSERT(pPager != NULL);
 
   // pTb->pBt
-  ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, &(pTb->pBt));
+  ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt));
   if (ret < 0) {
     tdbOsFree(pTb);
     return -1;
@@ -117,7 +117,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
   if (rollback) {
     tdbPagerRollback(pPager);
   } else {
-    ret = tdbPagerRestore(pPager, pTb->pBt);
+    ret = tdbPagerRestoreJournals(pPager, pTb->pBt);
     if (ret < 0) {
       tdbOsFree(pTb);
       return -1;

source/libs/tdb/src/db/tdbTxn.c

@@ -28,4 +28,10 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void
   return 0;
 }
 
-int tdbTxnClose(TXN *pTxn) { return 0; }
+int tdbTxnClose(TXN *pTxn) {
+  if (pTxn) {
+    tdbOsFree(pTxn);
+  }
+
+  return 0;
+}

source/libs/tdb/src/inc/tdbInt.h

@@ -147,11 +147,11 @@ struct SBTC {
   SPage       *pgStack[BTREE_MAX_DEPTH + 1];
   SCellDecoder coder;
   TXN         *pTxn;
-  TXN          txn;
+  i8           freeTxn;
 };
 
 // SBTree
-int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr,
+int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, TDB *pEnv,
                  SBTree **ppBt);
 int tdbBtreeClose(SBTree *pBt);
 int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
@@ -197,7 +197,7 @@ int  tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initP
                        TXN *pTxn);
 void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
 int  tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
-int  tdbPagerRestore(SPager *pPager, SBTree *pBt);
+int  tdbPagerRestoreJournals(SPager *pPager, SBTree *pBt);
 int  tdbPagerRollback(SPager *pPager);
 
 // tdbPCache.c ====================================
@@ -382,26 +382,24 @@ struct STDB {
 #ifdef USE_MAINDB
   TTB *pMainDb;
 #endif
+  int64_t txnId;
 };
 
-typedef struct hashset_st *hashset_t;
-
 struct SPager {
   char    *dbFileName;
   char    *jFileName;
   int      pageSize;
   uint8_t  fid[TDB_FILE_ID_LEN];
   tdb_fd_t fd;
-  tdb_fd_t jfd;
   SPCache *pCache;
   SPgno    dbFileSize;
   SPgno    dbOrigSize;
-  //SPage   *pDirty;
+  // SPage   *pDirty;
-  hashset_t jPageSet;
+  SRBTree rbt;
-  SRBTree  rbt;
+  // u8        inTran;
-  u8       inTran;
+  TXN    *pActiveTxn;
-  SPager  *pNext;      // used by TDB
+  SPager *pNext;      // used by TDB
-  SPager  *pHashNext;  // used by TDB
+  SPager *pHashNext;  // used by TDB
 #ifdef USE_MAINDB
   TDB *pEnv;
 #endif

source/libs/tdb/src/inc/tdbOs.h

@@ -47,15 +47,21 @@ typedef TdFilePtr tdb_fd_t;
 #define TDB_O_RDWR   (TD_FILE_WRITE) | (TD_FILE_READ)
 
 #define tdbOsOpen(PATH, OPTION, MODE) taosOpenFile((PATH), (OPTION))
-
+#define tdbOsClose(FD)                taosCloseFile(&(FD))
-#define tdbOsClose(FD)           taosCloseFile(&(FD))
+#define tdbOsRead                     taosReadFile
-#define tdbOsRead                taosReadFile
+#define tdbOsPRead                    taosPReadFile
-#define tdbOsPRead               taosPReadFile
+#define tdbOsWrite                    taosWriteFile
-#define tdbOsWrite               taosWriteFile
+#define tdbOsPWrite                   taosPWriteFile
-#define tdbOsFSync               taosFsyncFile
+#define tdbOsFSync                    taosFsyncFile
-#define tdbOsLSeek               taosLSeekFile
+#define tdbOsLSeek                    taosLSeekFile
-#define tdbOsRemove              remove
+#define tdbDirPtr                     TdDirPtr
-#define tdbOsFileSize(FD, PSIZE) taosFStatFile(FD, PSIZE, NULL)
+#define tdbDirEntryPtr                TdDirEntryPtr
+#define tdbReadDir                    taosReadDir
+#define tdbGetDirEntryName            taosGetDirEntryName
+#define tdbDirEntryBaseName           taosDirEntryBaseName
+#define tdbCloseDir                   taosCloseDir
+#define tdbOsRemove                   remove
+#define tdbOsFileSize(FD, PSIZE)      taosFStatFile(FD, PSIZE, NULL)
 
 /* directory */
 #define tdbOsMkdir taosMkDir

source/libs/tdb/test/tdbExOVFLTest.cpp

@@ -170,11 +170,9 @@ static void insertOfp(void) {
   SPoolMem *pPool = openPool();
 
   // start a transaction
-  TXN     txn;
+  TXN *txn = NULL;
-  int64_t txnid = 0;
+
-  ++txnid;
+  tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-  tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-  tdbBegin(pEnv, &txn);
 
   // generate value payload
   // char val[((4083 - 4 - 3 - 2) + 1) * 100];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
@@ -185,12 +183,12 @@ static void insertOfp(void) {
   // insert the generated big data
   // char const *key = "key1";
   char const *key = "key123456789";
-  ret = tdbTbInsert(pDb, key, strlen(key), val, valLen, &txn);
+  ret = tdbTbInsert(pDb, key, strlen(key), val, valLen, txn);
   GTEST_ASSERT_EQ(ret, 0);
 
   // commit current transaction
-  tdbCommit(pEnv, &txn);
+  tdbCommit(pEnv, txn);
-  tdbTxnClose(&txn);
+  tdbPostCommit(pEnv, txn);
 }
 
 // TEST(TdbOVFLPagesTest, DISABLED_TbInsertTest) {
@@ -258,11 +256,9 @@ TEST(TdbOVFLPagesTest, TbDeleteTest) {
   SPoolMem *pPool = openPool();
 
   // start a transaction
-  TXN     txn;
+  TXN *txn;
-  int64_t txnid = 0;
+
-  ++txnid;
+  tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-  tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-  tdbBegin(pEnv, &txn);
 
   // generate value payload
   // char val[((4083 - 4 - 3 - 2) + 1) * 100];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
@@ -271,7 +267,7 @@ TEST(TdbOVFLPagesTest, TbDeleteTest) {
   generateBigVal(val, valLen);
 
   {  // insert the generated big data
-    ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, &txn);
+    ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn);
     GTEST_ASSERT_EQ(ret, 0);
   }
 
@@ -297,7 +293,7 @@ tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_REA
 tdbBegin(pEnv, &txn);
   */
   {  // upsert the data
-    ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), &txn);
+    ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), txn);
     GTEST_ASSERT_EQ(ret, 0);
   }
 
@@ -316,7 +312,7 @@ tdbBegin(pEnv, &txn);
   }
 
   {  // delete the data
-    ret = tdbTbDelete(pDb, "key1", strlen("key1"), &txn);
+    ret = tdbTbDelete(pDb, "key1", strlen("key1"), txn);
     GTEST_ASSERT_EQ(ret, 0);
   }
 
@@ -335,8 +331,8 @@ tdbBegin(pEnv, &txn);
   }
 
   // commit current transaction
-  tdbCommit(pEnv, &txn);
+  tdbCommit(pEnv, txn);
-  tdbTxnClose(&txn);
+  tdbPostCommit(pEnv, txn);
 }
 
 // TEST(tdb_test, DISABLED_simple_insert1) {
@@ -346,7 +342,7 @@ TEST(tdb_test, simple_insert1) {
   TTB          *pDb;
   tdb_cmpr_fn_t compFunc;
   int           nData = 1;
-  TXN           txn;
+  TXN          *txn;
   int const     pageSize = 4096;
 
   taosRemoveDir("tdb");
@@ -365,16 +361,13 @@ TEST(tdb_test, simple_insert1) {
     // char      val[(4083 - 4 - 3 - 2)]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
     char      val[(4083 - 4 - 3 - 2) + 1];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
     int64_t   poolLimit = 4096;             // 1M pool limit
-    int64_t   txnid = 0;
     SPoolMem *pPool;
 
     // open the pool
     pPool = openPool();
 
     // start a transaction
-    txnid++;
+    tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-    tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-    tdbBegin(pEnv, &txn);
 
     for (int iData = 1; iData <= nData; iData++) {
       sprintf(key, "key0");
@@ -393,26 +386,25 @@ TEST(tdb_test, simple_insert1) {
         val[i] = c;
       }
 
-      ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, &txn);
+      ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn);
       GTEST_ASSERT_EQ(ret, 0);
 
       // if pool is full, commit the transaction and start a new one
       if (pPool->size >= poolLimit) {
         // commit current transaction
-        tdbCommit(pEnv, &txn);
+        tdbCommit(pEnv, txn);
-        tdbTxnClose(&txn);
+        tdbPostCommit(pEnv, txn);
 
         // start a new transaction
         clearPool(pPool);
-        txnid++;
+
-        tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+        tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-        tdbBegin(pEnv, &txn);
       }
     }
 
     // commit the transaction
-    tdbCommit(pEnv, &txn);
+    tdbCommit(pEnv, txn);
-    tdbTxnClose(&txn);
+    tdbPostCommit(pEnv, txn);
 
     {  // Query the data
       void *pVal = NULL;

source/libs/tdb/test/tdbTest.cpp

@@ -125,7 +125,7 @@ TEST(tdb_test, DISABLED_simple_insert1) {
   TTB          *pDb;
   tdb_cmpr_fn_t compFunc;
   int           nData = 1000000;
-  TXN           txn;
+  TXN          *txn;
 
   taosRemoveDir("tdb");
 
@@ -142,40 +142,35 @@ TEST(tdb_test, DISABLED_simple_insert1) {
     char      key[64];
     char      val[64];
     int64_t   poolLimit = 4096;  // 1M pool limit
-    int64_t   txnid = 0;
     SPoolMem *pPool;
 
     // open the pool
     pPool = openPool();
 
     // start a transaction
-    txnid++;
+    tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-    tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-    tdbBegin(pEnv, &txn);
 
     for (int iData = 1; iData <= nData; iData++) {
       sprintf(key, "key%d", iData);
       sprintf(val, "value%d", iData);
-      ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
+      ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn);
       GTEST_ASSERT_EQ(ret, 0);
 
       // if pool is full, commit the transaction and start a new one
       if (pPool->size >= poolLimit) {
         // commit current transaction
-        tdbCommit(pEnv, &txn);
+        tdbCommit(pEnv, txn);
-        tdbTxnClose(&txn);
+        tdbPostCommit(pEnv, txn);
 
         // start a new transaction
         clearPool(pPool);
-        txnid++;
+        tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-        tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-        tdbBegin(pEnv, &txn);
       }
     }
 
     // commit the transaction
-    tdbCommit(pEnv, &txn);
+    tdbCommit(pEnv, txn);
-    tdbTxnClose(&txn);
+    tdbPostCommit(pEnv, txn);
 
     {  // Query the data
       void *pVal = NULL;
@@ -245,7 +240,7 @@ TEST(tdb_test, DISABLED_simple_insert2) {
   TTB          *pDb;
   tdb_cmpr_fn_t compFunc;
   int           nData = 1000000;
-  TXN           txn;
+  TXN          *txn;
 
   taosRemoveDir("tdb");
 
@@ -261,21 +256,18 @@ TEST(tdb_test, DISABLED_simple_insert2) {
   {
     char      key[64];
     char      val[64];
-    int64_t   txnid = 0;
     SPoolMem *pPool;
 
     // open the pool
     pPool = openPool();
 
     // start a transaction
-    txnid++;
+    tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-    tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-    tdbBegin(pEnv, &txn);
 
     for (int iData = 1; iData <= nData; iData++) {
       sprintf(key, "key%d", iData);
       sprintf(val, "value%d", iData);
-      ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
+      ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn);
       GTEST_ASSERT_EQ(ret, 0);
     }
 
@@ -312,8 +304,8 @@ TEST(tdb_test, DISABLED_simple_insert2) {
   }
 
   // commit the transaction
-  tdbCommit(pEnv, &txn);
+  tdbCommit(pEnv, txn);
-  tdbTxnClose(&txn);
+  tdbPostCommit(pEnv, txn);
 
   ret = tdbTbDrop(pDb);
   GTEST_ASSERT_EQ(ret, 0);
@@ -331,7 +323,7 @@ TEST(tdb_test, DISABLED_simple_delete1) {
   TTB      *pDb;
   char      key[128];
   char      data[128];
-  TXN       txn;
+  TXN      *txn;
   TDB      *pEnv;
   SPoolMem *pPool;
   void     *pKey = NULL;
@@ -353,14 +345,13 @@ TEST(tdb_test, DISABLED_simple_delete1) {
   ret = tdbTbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb, 0);
   GTEST_ASSERT_EQ(ret, 0);
 
-  tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+  tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-  tdbBegin(pEnv, &txn);
 
   // loop to insert batch data
   for (int iData = 0; iData < nKV; iData++) {
     sprintf(key, "key%d", iData);
     sprintf(data, "data%d", iData);
-    ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn);
+    ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), txn);
     GTEST_ASSERT_EQ(ret, 0);
   }
 
@@ -378,7 +369,7 @@ TEST(tdb_test, DISABLED_simple_delete1) {
   for (int iData = nKV - 1; iData > 30; iData--) {
     sprintf(key, "key%d", iData);
 
-    ret = tdbTbDelete(pDb, key, strlen(key), &txn);
+    ret = tdbTbDelete(pDb, key, strlen(key), txn);
     GTEST_ASSERT_EQ(ret, 0);
   }
 
@@ -413,7 +404,8 @@ TEST(tdb_test, DISABLED_simple_delete1) {
 
   tdbTbcClose(pDbc);
 
-  tdbCommit(pEnv, &txn);
+  tdbCommit(pEnv, txn);
+  tdbPostCommit(pEnv, txn);
 
   closePool(pPool);
 
@@ -430,7 +422,7 @@ TEST(tdb_test, DISABLED_simple_upsert1) {
   char      data[64];
   void     *pData = NULL;
   SPoolMem *pPool;
-  TXN       txn;
+  TXN      *txn;
 
   taosRemoveDir("tdb");
 
@@ -444,13 +436,12 @@ TEST(tdb_test, DISABLED_simple_upsert1) {
 
   pPool = openPool();
   // insert some data
-  tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+  tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-  tdbBegin(pEnv, &txn);
 
   for (int iData = 0; iData < nData; iData++) {
     sprintf(key, "key%d", iData);
     sprintf(data, "data%d", iData);
-    ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn);
+    ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), txn);
     GTEST_ASSERT_EQ(ret, 0);
   }
 
@@ -467,11 +458,12 @@ TEST(tdb_test, DISABLED_simple_upsert1) {
   for (int iData = 0; iData < nData; iData++) {
     sprintf(key, "key%d", iData);
     sprintf(data, "data%d-u", iData);
-    ret = tdbTbUpsert(pDb, key, strlen(key), data, strlen(data), &txn);
+    ret = tdbTbUpsert(pDb, key, strlen(key), data, strlen(data), txn);
     GTEST_ASSERT_EQ(ret, 0);
   }
 
-  tdbCommit(pEnv, &txn);
+  tdbCommit(pEnv, txn);
+  tdbPostCommit(pEnv, txn);
 
   // query the data
   for (int iData = 0; iData < nData; iData++) {
@@ -492,7 +484,7 @@ TEST(tdb_test, multi_thread_query) {
   TTB          *pDb;
   tdb_cmpr_fn_t compFunc;
   int           nData = 1000000;
-  TXN           txn;
+  TXN          *txn;
 
   taosRemoveDir("tdb");
 
@@ -508,26 +500,18 @@ TEST(tdb_test, multi_thread_query) {
   char      key[64];
   char      val[64];
   int64_t   poolLimit = 4096 * 20;  // 1M pool limit
-  int64_t   txnid = 0;
   SPoolMem *pPool;
 
   // open the pool
   pPool = openPool();
 
   // start a transaction
-  txnid++;
+  tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-  txn.flags = TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED;
-  txn.txnId = -1;
-  txn.xMalloc = poolMalloc;
-  txn.xFree = poolFree;
-  txn.xArg = pPool;
-  // tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, );
-  tdbBegin(pEnv, &txn);
 
   for (int iData = 1; iData <= nData; iData++) {
     sprintf(key, "key%d", iData);
     sprintf(val, "value%d", iData);
-    ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
+    ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn);
     GTEST_ASSERT_EQ(ret, 0);
   }
 
@@ -578,7 +562,7 @@ TEST(tdb_test, multi_thread_query) {
   std::vector<std::thread> threads;
   for (int i = 0; i < nThreads; i++) {
     if (i == 0) {
-      threads.push_back(std::thread(tdbCommit, pEnv, &txn));
+      threads.push_back(std::thread(tdbCommit, pEnv, txn));
     } else {
       threads.push_back(std::thread(f, pDb, nData));
     }
@@ -589,8 +573,8 @@ TEST(tdb_test, multi_thread_query) {
   }
 
   // commit the transaction
-  tdbCommit(pEnv, &txn);
+  tdbCommit(pEnv, txn);
-  tdbTxnClose(&txn);
+  tdbPostCommit(pEnv, txn);
 
   // Close a database
   tdbTbClose(pDb);
@@ -621,17 +605,12 @@ TEST(tdb_test, DISABLED_multi_thread1) {
   GTEST_ASSERT_EQ(ret, 0);
 
   auto insert = [](TDB *pDb, TTB *pTb, int nData, int *stop, std::shared_timed_mutex *mu) {
-    TXN       txn = {0};
+    TXN       *txn = NULL;
     char      key[128];
     char      val[128];
     SPoolMem *pPool = openPool();
 
-    txn.flags = TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED;
+    tdbBegin(pDb, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-    txn.txnId = -1;
-    txn.xMalloc = poolMalloc;
-    txn.xFree = poolFree;
-    txn.xArg = pPool;
-    tdbBegin(pDb, &txn);
     for (int iData = 1; iData <= nData; iData++) {
       sprintf(key, "key%d", iData);
       sprintf(val, "value%d", iData);
@@ -644,14 +623,17 @@ TEST(tdb_test, DISABLED_multi_thread1) {
       }
 
       if (pPool->size > 1024 * 1024) {
-        tdbCommit(pDb, &txn);
+        tdbCommit(pDb, txn);
+        tdbPostCommit(pDb, txn);
 
         clearPool(pPool);
-        tdbBegin(pDb, &txn);
+        tdbBegin(pDb, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
       }
     }
 
-    tdbCommit(pDb, &txn);
+    tdbCommit(pDb, txn);
+    tdbPostCommit(pDb, txn);
+
     closePool(pPool);
 
     *stop = 1;

source/libs/wal/src/walMeta.c

@@ -24,7 +24,9 @@ bool FORCE_INLINE walLogExist(SWal* pWal, int64_t ver) {
   return !walIsEmpty(pWal) && walGetFirstVer(pWal) <= ver && walGetLastVer(pWal) >= ver;
 }
 
-bool FORCE_INLINE walIsEmpty(SWal* pWal) { return pWal->vers.firstVer == -1; }
+bool FORCE_INLINE walIsEmpty(SWal* pWal) {
+  return (pWal->vers.firstVer == -1 || pWal->vers.lastVer < pWal->vers.firstVer); // [firstVer, lastVer + 1)
+}
 
 int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; }
 

source/libs/wal/src/walMgmt.c

@@ -121,7 +121,16 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
   pWal->writeCur = -1;
   pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo));
   if (pWal->fileInfoSet == NULL) {
-    wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno));
+    wError("vgId:%d, failed to init taosArray of fileInfoSet due to %s. path:%s", pWal->cfg.vgId, strerror(errno),
+           pWal->path);
+    goto _err;
+  }
+
+  // init gc
+  pWal->toDeleteFiles = taosArrayInit(8, sizeof(SWalFileInfo));
+  if (pWal->toDeleteFiles == NULL) {
+    wError("vgId:%d, failed to init taosArray of toDeleteFiles due to %s. path:%s", pWal->cfg.vgId, strerror(errno),
+           pWal->path);
     goto _err;
   }
 
@@ -203,6 +212,8 @@ void walClose(SWal *pWal) {
   pWal->pIdxFile = NULL;
   taosArrayDestroy(pWal->fileInfoSet);
   pWal->fileInfoSet = NULL;
+  taosArrayDestroy(pWal->toDeleteFiles);
+  pWal->toDeleteFiles = NULL;
 
   void *pIter = NULL;
   while (1) {

source/libs/wal/src/walRead.c

@@ -489,7 +489,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
   int32_t code;
   bool    seeked = false;
 
-  if (pReader->pWal->vers.firstVer == -1) {
+  if (walIsEmpty(pReader->pWal)) {
     terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
     return -1;
   }

source/libs/wal/src/walWrite.c

@@ -120,9 +120,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
       return -1;
     }
 
-    // delete files
+    // delete files in descending order
     int fileSetSize = taosArrayGetSize(pWal->fileInfoSet);
-    for (int i = pWal->writeCur + 1; i < fileSetSize; i++) {
+    for (int i = fileSetSize - 1; i >= pWal->writeCur + 1; i--) {
       walBuildLogName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
       wDebug("vgId:%d, wal remove file %s for rollback", pWal->cfg.vgId, fnameStr);
       taosRemoveFile(fnameStr);
@@ -217,14 +217,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
   pWal->vers.lastVer = ver - 1;
   if (pWal->vers.lastVer < pWal->vers.firstVer) {
     ASSERT(pWal->vers.lastVer == pWal->vers.firstVer - 1);
-    pWal->vers.firstVer = -1;
   }
   ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
   ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
-  if (((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer < ver - 1) {
-    ASSERT(((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize == 0);
-    ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->firstVer = -1;
-  }
   taosCloseFile(&pIdxFile);
   taosCloseFile(&pLogFile);
 
@@ -338,6 +333,7 @@ int32_t walEndSnapshot(SWal *pWal) {
     } else {
       wDebug("vgId:%d, wal no remove", pWal->cfg.vgId);
     }
+
     // iterate files, until the searched result
     for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
       wDebug("vgId:%d, wal check remove file %" PRId64 "(file size %" PRId64 " close ts %" PRId64
@@ -350,34 +346,17 @@ int32_t walEndSnapshot(SWal *pWal) {
         wDebug("vgId:%d, check pass", pWal->cfg.vgId);
         deleteCnt++;
         newTotSize -= iter->fileSize;
+        taosArrayPush(pWal->toDeleteFiles, iter);
       }
       wDebug("vgId:%d, check not pass", pWal->cfg.vgId);
     }
-    wDebug("vgId:%d, wal should delete %d files", pWal->cfg.vgId, deleteCnt);
-    int32_t actualDelete = 0;
-    char    fnameStr[WAL_FILE_LEN];
-    // remove file
-    for (int i = 0; i < deleteCnt; i++) {
-      pInfo = taosArrayGet(pWal->fileInfoSet, i);
-      walBuildLogName(pWal, pInfo->firstVer, fnameStr);
-      wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr);
-      if (taosRemoveFile(fnameStr) < 0) {
-        goto UPDATE_META;
-      }
-      walBuildIdxName(pWal, pInfo->firstVer, fnameStr);
-      wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr);
-      if (taosRemoveFile(fnameStr) < 0) {
-        ASSERT(0);
-      }
-      actualDelete++;
-    }
 
   UPDATE_META:
     // make new array, remove files
-    taosArrayPopFrontBatch(pWal->fileInfoSet, actualDelete);
+    taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt);
     if (taosArrayGetSize(pWal->fileInfoSet) == 0) {
       pWal->writeCur = -1;
-      pWal->vers.firstVer = -1;
+      pWal->vers.firstVer = pWal->vers.lastVer + 1;
     } else {
       pWal->vers.firstVer = ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
     }
@@ -392,6 +371,26 @@ int32_t walEndSnapshot(SWal *pWal) {
     goto END;
   }
 
+  // delete files
+  deleteCnt = taosArrayGetSize(pWal->toDeleteFiles);
+  wDebug("vgId:%d, wal should delete %d files", pWal->cfg.vgId, deleteCnt);
+  char fnameStr[WAL_FILE_LEN];
+  for (int i = 0; i < deleteCnt; i++) {
+    pInfo = taosArrayGet(pWal->toDeleteFiles, i);
+    walBuildLogName(pWal, pInfo->firstVer, fnameStr);
+    wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr);
+    if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) {
+      wError("vgId:%d, failed to remove log file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno));
+      goto END;
+    }
+    walBuildIdxName(pWal, pInfo->firstVer, fnameStr);
+    wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr);
+    if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) {
+      ASSERT(0);
+    }
+  }
+  taosArrayClear(pWal->toDeleteFiles);
+
 END:
   taosThreadMutexUnlock(&pWal->mutex);
   return code;
@@ -489,9 +488,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
   SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal);
   ASSERT(pFileInfo != NULL);
 
-  if (pFileInfo->firstVer == -1) {
+  ASSERT(pFileInfo->firstVer != -1);
-    pFileInfo->firstVer = index;
-  }
   pWal->writeHead.head.version = index;
   pWal->writeHead.head.bodyLen = bodyLen;
   pWal->writeHead.head.msgType = msgType;
@@ -527,7 +524,10 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
   }
 
   // set status
-  if (pWal->vers.firstVer == -1) pWal->vers.firstVer = index;
+  if (pWal->vers.firstVer == -1) {
+    ASSERT(index == 0);
+    pWal->vers.firstVer = 0;
+  }
   pWal->vers.lastVer = index;
   pWal->totSize += sizeof(SWalCkHead) + bodyLen;
   pFileInfo->lastVer = index;

source/os/src/osFile.c

@@ -491,6 +491,28 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
   return count;
 }
 
+int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) {
+  if (pFile == NULL) {
+    return 0;
+  }
+#if FILE_WITH_LOCK
+  taosThreadRwlockWrlock(&(pFile->rwlock));
+#endif
+  assert(pFile->fd >= 0);  // Please check if you have closed the file.
+#ifdef WINDOWS
+  size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR);
+  _lseeki64(pFile->fd, offset, SEEK_SET);
+  int64_t ret = _write(pFile->fd, buf, count);
+  _lseeki64(pFile->fd, pos, SEEK_SET);
+#else
+  int64_t ret = pwrite(pFile->fd, buf, count, offset);
+#endif
+#if FILE_WITH_LOCK
+  taosThreadRwlockUnlock(&(pFile->rwlock));
+#endif
+  return ret;
+}
+
 int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) {
 #if FILE_WITH_LOCK
   taosThreadRwlockRdlock(&(pFile->rwlock));

source/util/src/terror.c

@@ -57,7 +57,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED,                "Client and server's t
 TAOS_DEFINE_ERROR(TSDB_CODE_OPS_NOT_SUPPORT,              "Operation not supported")
 TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_MEMORY,                "Out of Memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED,               "Data file corrupted")
-TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY,                "Ref out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL,                     "too many Ref Objs")
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED,               "Ref ID is removed")
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID,               "Invalid Ref ID")
@@ -140,29 +139,19 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NOT_STABLE_ERROR,         "Table is not a super
 
 // mnode-common
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS,                "Insufficient privilege for operation")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION,       "Invalid message connection")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ,          "Data expired")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID,         "Invalid query id")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID,        "Invalid stream id")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID,          "Invalid connection id")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_IS_RUNNING,         "mnode is alreay running")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC,    "failed to config sync")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_START_SYNC,     "failed to start sync")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CREATE_DIR,     "failed to create mnode dir")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_INIT_STEP,      "failed to init components")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_DISABLED,            "User is disabled")
 
 // mnode-sdb
 TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_ALREADY_THERE,        "Object already there")
 TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_TABLE_TYPE,       "Invalid table type")
 TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_NOT_THERE,            "Object not there")
-TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_KEY_TYPE,         "Invalid key type")
 TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_ACTION_TYPE,      "Invalid action type")
-TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_STATUS_TYPE,      "Invalid status type")
 TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_VER,         "Invalid raw data version")
 TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_LEN,         "Invalid raw data len")
 TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_CONTENT,     "Invalid raw data content")
-TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_WAl_VER,          "Invalid wal version")
 TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_CREATING,             "Object is creating")
 TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_DROPPING,             "Object is dropping")
 
@@ -418,7 +407,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR,           "Sync internal error")
 TAOS_DEFINE_ERROR(TSDB_CODE_TQ_INVALID_CONFIG,            "TQ invalid config")
 TAOS_DEFINE_ERROR(TSDB_CODE_TQ_INIT_FAILED,               "TQ init falied")
 TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_DISK_PERMISSIONS,       "TQ no disk permissions")
-TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FILE_CORRUPTED,            "TQ file corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FILE_ALREADY_EXISTS,       "TQ file already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FAILED_TO_CREATE_DIR,      "TQ failed to create dir")
 TAOS_DEFINE_ERROR(TSDB_CODE_TQ_META_NO_SUCH_KEY,          "TQ meta no such key")
@@ -596,7 +584,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE,         "No tsma index in ca
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV,               "Invalid rsma env")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT,              "Invalid rsma state")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_QTASKINFO_CREATE,          "Rsma qtaskinfo creation error")
-TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FILE_CORRUPTED,            "Rsma file corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REMOVE_EXISTS,             "Rsma remove exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP,        "Rsma fetch msg is messed up")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_EMPTY_INFO,                "Rsma info is empty")

source/util/src/tref.c

@@ -67,14 +67,14 @@ int32_t taosOpenRef(int32_t max, RefFp fp) {
 
   nodeList = taosMemoryCalloc(sizeof(SRefNode *), (size_t)max);
   if (nodeList == NULL) {
-    terrno = TSDB_CODE_REF_NO_MEMORY;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 
   lockedBy = taosMemoryCalloc(sizeof(int64_t), (size_t)max);
   if (lockedBy == NULL) {
     taosMemoryFree(nodeList);
-    terrno = TSDB_CODE_REF_NO_MEMORY;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 
@@ -164,7 +164,7 @@ int64_t taosAddRef(int32_t rsetId, void *p) {
 
   pNode = taosMemoryCalloc(sizeof(SRefNode), 1);
   if (pNode == NULL) {
-    terrno = TSDB_CODE_REF_NO_MEMORY;
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 

tests/docs-examples-test/csharp.sh

@@ -28,10 +28,10 @@ taos -s "drop database if exists test"
 dotnet run --project optsJSON/optsJSON.csproj
 
 taos -s "create database if not exists test"
-# dotnet run --project wsConnect/wsConnect.csproj
+dotnet run --project wsConnect/wsConnect.csproj
-# dotnet run --project wsInsert/wsInsert.csproj
+dotnet run --project wsInsert/wsInsert.csproj
-# dotnet run --project wsStmt/wsStmt.csproj
+dotnet run --project wsStmt/wsStmt.csproj
-# dotnet run --project wsQuery/wsQuery.csproj
+dotnet run --project wsQuery/wsQuery.csproj
 
 taos -s "drop database if exists test"
-taos -s "drop database if exists power"
+taos -s "drop database if exists power"

tests/pytest/crash_gen/crash_gen_main.py

@@ -2022,11 +2022,10 @@ class TdSuperTable:
             conf.set("group.id", "tg2")
             conf.set("td.connect.user", "root")
             conf.set("td.connect.pass", "taosdata")
-            conf.set("enable.auto.
+#            conf.set("enable.auto.commit", "true")
-            ", "true")
+#            def tmq_commit_cb_print(tmq, resp, offset, param=None):
-            def tmq_commit_cb_print(tmq, resp, offset, param=None):
+#                print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}")
-                print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}")
+#            conf.set_auto_commit_cb(tmq_commit_cb_print, None)
-            conf.set_auto_commit_cb(tmq_commit_cb_print, None)
             consumer = conf.new_consumer()   
             topic_list = TaosTmqList()
             for topic in current_topic_list:

tests/script/tsim/stream/session1.sim

@@ -197,4 +197,98 @@ if $data01 != 1 then
   return -1
 endi
 
+sql create database test1  vgroups 1;
+sql use test1;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams3 trigger at_once  into streamt3 as select  _wstart, count(*) c1 from t1 where a > 5 session(ts, 5s);
+sql insert into t1 values(1648791213000,1,2,3,1.0);
+
+$loop_count = 0
+loop13:
+sleep 200
+
+sql select * from streamt3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+# row 0
+if $rows != 0 then
+  print =====rows=$rows
+  goto loop13
+endi
+
+sql insert into t1 values(1648791213000,6,2,3,1.0);
+
+$loop_count = 0
+loop14:
+sleep 200
+sql select * from streamt3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $data01 != 1 then
+  print =====data01=$data01
+  goto loop14
+endi
+
+sql insert into t1 values(1648791213000,2,2,3,1.0);
+
+$loop_count = 0
+loop15:
+sleep 200
+sql select * from streamt3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 0 then
+  print =====rows=$rows
+  goto loop15
+endi
+
+
+sql insert into t1 values(1648791223000,2,2,3,1.0);
+sql insert into t1 values(1648791223000,10,2,3,1.0);
+sql insert into t1 values(1648791233000,10,2,3,1.0);
+
+$loop_count = 0
+loop16:
+sleep 200
+sql select * from streamt3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 2 then
+  print =====rows=$rows
+  goto loop16
+endi
+
+sql insert into t1 values(1648791233000,2,2,3,1.0);
+
+$loop_count = 0
+loop17:
+sleep 200
+sql select * from streamt3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+if $rows != 1 then
+  print =====rows=$rows
+  goto loop17
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/system-test/2-query/timetruncate.py

@@ -30,7 +30,7 @@ class TDTestCase:
         self.stbname = f'{self.dbname}.stb'
         self.ctbname = f'{self.dbname}.ctb'
 
-    def check_ms_timestamp(self,unit,date_time):
+    def check_ms_timestamp(self,unit,date_time, ignore_tz):
         if unit.lower() == '1a':
             for i in range(len(self.ts_str)):
                 ts_result = self.get_time.get_ms_timestamp(str(tdSql.queryResult[i][0]))
@@ -50,13 +50,17 @@ class TDTestCase:
         elif unit.lower() == '1d':
             for i in range(len(self.ts_str)):
                 ts_result = self.get_time.get_ms_timestamp(str(tdSql.queryResult[i][0]))
-                tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24)*24*60*60*1000)
+                if ignore_tz == 0:
+                    tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24)*24*60*60*1000)
+                else:
+                    # assuming the client timezone is UTC+0800
+                    tdSql.checkEqual(ts_result,int(date_time[i] - (date_time[i] + 8 * 3600 * 1000) % (86400 * 1000)))
         elif unit.lower() == '1w':
             for i in range(len(self.ts_str)):
                 ts_result = self.get_time.get_ms_timestamp(str(tdSql.queryResult[i][0]))
                 tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24/7)*7*24*60*60*1000)
 
-    def check_us_timestamp(self,unit,date_time):
+    def check_us_timestamp(self,unit,date_time, ignore_tz):
         if unit.lower() == '1u':
             for i in range(len(self.ts_str)):
                 ts_result = self.get_time.get_us_timestamp(str(tdSql.queryResult[i][0]))
@@ -80,13 +84,17 @@ class TDTestCase:
         elif unit.lower() == '1d':
             for i in range(len(self.ts_str)):
                 ts_result = self.get_time.get_us_timestamp(str(tdSql.queryResult[i][0]))
-                tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24)*24*60*60*1000*1000 )
+                if ignore_tz == 0:
+                    tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24)*24*60*60*1000*1000 )
+                else:
+                    # assuming the client timezone is UTC+0800
+                    tdSql.checkEqual(ts_result,int(date_time[i] - (date_time[i] + 8 * 3600 * 1000000) % (86400 * 1000000)))
         elif unit.lower() == '1w':
             for i in range(len(self.ts_str)):
                 ts_result = self.get_time.get_us_timestamp(str(tdSql.queryResult[i][0]))
                 tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24/7)*7*24*60*60*1000*1000)
 
-    def check_ns_timestamp(self,unit,date_time):
+    def check_ns_timestamp(self,unit,date_time, ignore_tz):
         if unit.lower() == '1b':
             for i in range(len(self.ts_str)):
                 if self.rest_tag != 'rest':
@@ -114,21 +122,26 @@ class TDTestCase:
         elif unit.lower() == '1d':
             for i in range(len(self.ts_str)):
                 if self.rest_tag != 'rest':
-                    tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24)*24*60*60*1000*1000*1000 )
+                    if ignore_tz == 0:
+                        tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24)*24*60*60*1000*1000*1000 )
+                    else:
+                        # assuming the client timezone is UTC+0800
+                        tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i] - (date_time[i] + 8 * 3600 * 1000000) % (86400 * 1000000)))
         elif unit.lower() == '1w':
             for i in range(len(self.ts_str)):
                 if self.rest_tag != 'rest':
                     tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24/7)*7*24*60*60*1000*1000*1000)
 
-    def check_tb_type(self,unit,tb_type):
+    def check_tb_type(self,unit,tb_type,ignore_tz):
         if tb_type.lower() == 'ntb':
-            tdSql.query(f'select timetruncate(ts,{unit}) from {self.ntbname}')
+            tdSql.query(f'select timetruncate(ts,{unit},{ignore_tz}) from {self.ntbname}')
         elif tb_type.lower() == 'ctb':
-            tdSql.query(f'select timetruncate(ts,{unit}) from {self.ctbname}')
+            tdSql.query(f'select timetruncate(ts,{unit},{ignore_tz}) from {self.ctbname}')
         elif tb_type.lower() == 'stb':
-            tdSql.query(f'select timetruncate(ts,{unit}) from {self.stbname}')
+            tdSql.query(f'select timetruncate(ts,{unit},{ignore_tz}) from {self.stbname}')
 
     def data_check(self,date_time,precision,tb_type):
+        tz_options = [0, 1]
         for unit in self.time_unit:
             if (unit.lower() == '1u' and precision.lower() == 'ms') or (unit.lower() == '1b' and precision.lower() == 'us') or (unit.lower() == '1b' and precision.lower() == 'ms'):
                 if tb_type.lower() == 'ntb':
@@ -138,17 +151,20 @@ class TDTestCase:
                 elif tb_type.lower() == 'stb':
                     tdSql.error(f'select timetruncate(ts,{unit}) from {self.stbname}')
             elif precision.lower() == 'ms':
-                self.check_tb_type(unit,tb_type)
+                for ignore_tz in tz_options:
-                tdSql.checkRows(len(self.ts_str))
+                    self.check_tb_type(unit,tb_type,ignore_tz)
-                self.check_ms_timestamp(unit,date_time)
+                    tdSql.checkRows(len(self.ts_str))
+                    self.check_ms_timestamp(unit,date_time,ignore_tz)
             elif precision.lower() == 'us':
-                self.check_tb_type(unit,tb_type)
+                for ignore_tz in tz_options:
-                tdSql.checkRows(len(self.ts_str))
+                    self.check_tb_type(unit,tb_type,ignore_tz)
-                self.check_us_timestamp(unit,date_time)
+                    tdSql.checkRows(len(self.ts_str))
+                    self.check_us_timestamp(unit,date_time,ignore_tz)
             elif precision.lower() == 'ns':
-                self.check_tb_type(unit,tb_type)
+                for ignore_tz in tz_options:
-                tdSql.checkRows(len(self.ts_str))
+                    self.check_tb_type(unit,tb_type, ignore_tz)
-                self.check_ns_timestamp(unit,date_time)
+                    tdSql.checkRows(len(self.ts_str))
+                    self.check_ns_timestamp(unit,date_time,ignore_tz)
         for unit in self.error_unit:
             if tb_type.lower() == 'ntb':
                 tdSql.error(f'select timetruncate(ts,{unit}) from {self.ntbname}')
@@ -181,8 +197,8 @@ class TDTestCase:
             date_time = self.get_time.time_transform(self.ts_str,precision)
             self.data_check(date_time,precision,'ctb')
             self.data_check(date_time,precision,'stb')
-    
+
-   
+
     def run(self):
         self.function_check_ntb()
         self.function_check_stb()

tools/shell/src/shellEngine.c

@@ -113,6 +113,13 @@ int32_t shellRunSingleCommand(char *command) {
 }
 
 void shellRecordCommandToHistory(char *command) {
+  if (strncasecmp(command, "create user ", 12) == 0 || strncasecmp(command, "alter user ", 11) == 0) {
+    if (taosStrCaseStr(command, " pass ")) {
+      // have password command forbid record to history because security
+      return;
+    }
+  }
+
   SShellHistory *pHistory = &shell.history;
   if (pHistory->hstart == pHistory->hend ||
       pHistory->hist[(pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE] == NULL ||
@@ -135,7 +142,7 @@ int32_t shellRunCommand(char *command, bool recordHistory) {
   }
 
   // add help or help; 
-  if(strcmp(command, "help") == 0 || strcmp(command, "help;") == 0) {
+  if(strncasecmp(command, "help;", 5) == 0) {
     showHelp();
     return 0;
   }