From b2ad162a8b770032594020bd18f600f57b761610 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Tue, 20 Feb 2024 17:13:58 +0800
Subject: [PATCH 01/37] enh: remove obsolete codes

---
 include/common/tmsg.h                  | 1 -
 source/dnode/mnode/impl/src/mndDnode.c | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 1b438ba026..87d8ed76a8 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -1613,7 +1613,6 @@ typedef struct {
   SEp     ep;
   char    active[TSDB_ACTIVE_KEY_LEN];
   char    connActive[TSDB_CONN_ACTIVE_KEY_LEN];
-  char    machineId[TSDB_MACHINE_ID_LEN + 1];
 } SDnodeInfo;
 
 typedef struct {
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index cb8a24e675..6974c6b0e8 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -410,9 +410,6 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) {
     dInfo.ep.port = pDnode->port;
     dInfo.offlineReason = pDnode->offlineReason;
     tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
-    tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN);
-    tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN);
-    tstrncpy(dInfo.machineId, pDnode->machineId, TSDB_MACHINE_ID_LEN + 1);
     sdbRelease(pSdb, pDnode);
     if (mndIsMnode(pMnode, pDnode->id)) {
       dInfo.isMnode = 1;

From 87ad111b2c7177eab7751ffc32098fe41227a884 Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Tue, 20 Feb 2024 18:40:25 +0800
Subject: [PATCH 02/37] coverage: snapshot add stop taosd and start

---
 tests/army/community/cluster/snapshot.py | 18 +++++++++++++++++-
 tests/army/frame/srvCtl.py               |  4 ++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/community/cluster/snapshot.py
index b21cbb1ad8..eef650cc77 100644
--- a/tests/army/community/cluster/snapshot.py
+++ b/tests/army/community/cluster/snapshot.py
@@ -25,6 +25,7 @@ from frame.cases import *
 from frame.sql import *
 from frame.caseBase import *
 from frame import *
+from frame.srvCtl import *
 
 
 class TDTestCase(TBase):
@@ -65,6 +66,21 @@ class TDTestCase(TBase):
         sql = f"select avg(dc) from {self.db}.{self.stb}"
         tdSql.checkFirstValue(sql, 200)
 
+    def alterReplica3(self):
+        sql = f"alter database {self.db} replica 3"
+        tdSql.execute(sql, show=True)
+        time.sleep(2)
+        sc.dnodeStop(2)
+        sc.dnodeStop(3)
+        time.sleep(5)
+        sc.dnodeStart(2)
+        sc.dnodeStart(3)
+
+        if self.waitTransactionZero() is False:
+            tdLog.exit(f"{sql} transaction not finished")
+            return False
+        return True
+
     def doAction(self):
         tdLog.info(f"do action.")
         self.flushDb()
@@ -81,7 +97,7 @@ class TDTestCase(TBase):
         self.alterReplica(1)
         self.checkAggCorrect()
         self.compactDb()
-        self.alterReplica(3)
+        self.alterReplica3()
 
         vgids = self.getVGroup(self.db)
         selid = random.choice(vgids)
diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py
index 091856056b..3a9b0cdf4b 100644
--- a/tests/army/frame/srvCtl.py
+++ b/tests/army/frame/srvCtl.py
@@ -33,14 +33,14 @@ class srvCtl:
     #  control server
     #
 
-    # start
+    # start idx base is 1 
     def dnodeStart(self, idx):
         if clusterDnodes.getModel() == 'cluster':
             return clusterDnodes.starttaosd(idx)
 
         return tdDnodes.starttaosd(idx)
 
-    # stop
+    # stop idx base is 1 
     def dnodeStop(self, idx):
         if clusterDnodes.getModel() == 'cluster':
             return clusterDnodes.stoptaosd(idx)

From 73ec02ceb4cfb8a8195ab5cd04a2c024201ce7bf Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Tue, 20 Feb 2024 18:48:06 +0800
Subject: [PATCH 03/37] coverage: s3 and stop and start taosd

---
 tests/army/enterprise/s3/s3_basic.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py
index e7bc188ca5..58dbd12d7b 100644
--- a/tests/army/enterprise/s3/s3_basic.py
+++ b/tests/army/enterprise/s3/s3_basic.py
@@ -81,11 +81,15 @@ class TDTestCase(TBase):
         cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data"
         tdLog.info(cmd)
         loop = 0
-        while len(eos.runRetList(cmd)) > 0 and loop < 40:
+        while len(eos.runRetList(cmd)) > 0 and loop < 100:
             time.sleep(5)
             self.trimDb(True)
             loop += 1
             tdLog.info(f"loop={loop} wait 5s...")
+            if loop == 4:
+                sc.dnodeStop(1)
+                time.sleep(2)
+                sc.dnodeStart(1)
 
     def checkStreamCorrect(self):
         sql = f"select count(*) from {self.db}.stm1"

From db03fbd8960f98011c9ac7d2ac73a37917f2d612 Mon Sep 17 00:00:00 2001
From: yihaoDeng <yhdeng@taosdata.com>
Date: Tue, 20 Feb 2024 19:11:21 +0800
Subject: [PATCH 04/37] add test case

---
 source/util/src/tskiplist.c | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c
index eadd9a2413..f90509bec1 100644
--- a/source/util/src/tskiplist.c
+++ b/source/util/src/tskiplist.c
@@ -21,7 +21,7 @@
 #include "tutil.h"
 
 static int32_t            initForwardBackwardPtr(SSkipList *pSkipList);
-static SSkipListNode     *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur);
+static SSkipListNode *    getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur);
 static void               tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode);
 static void               tSkipListCorrectLevel(SSkipList *pSkipList);
 static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order);
@@ -131,12 +131,14 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) {
   return pNode;
 }
 
+#ifdef BUILD_NO_CALL
+
 void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate) {
   SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0};
   SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0};
   bool           hasDup = false;
-  char          *pKey = NULL;
-  char          *pDataKey = NULL;
+  char *         pKey = NULL;
+  char *         pDataKey = NULL;
   int32_t        compare = 0;
 
   tSkipListWLock(pSkipList);
@@ -260,6 +262,7 @@ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) {
   tSkipListCorrectLevel(pSkipList);
   tSkipListUnlock(pSkipList);
 }
+#endif
 
 SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList) {
   if (pSkipList == NULL) return NULL;
@@ -350,6 +353,7 @@ void *tSkipListDestroyIter(SSkipListIterator *iter) {
   return NULL;
 }
 
+#ifdef BUILD_NO_CALL
 void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
   if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) {
     return;
@@ -358,7 +362,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
   SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1);
 
   int32_t id = 1;
-  char   *prev = NULL;
+  char *  prev = NULL;
 
   while (p != pSkipList->pTail) {
     char *key = SL_GET_NODE_KEY(pSkipList, p);
@@ -392,6 +396,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
     p = SL_NODE_GET_FORWARD_POINTER(p, nlevel - 1);
   }
 }
+#endif
 
 static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward) {
   for (int32_t i = 0; i < pNode->level; ++i) {
@@ -460,7 +465,7 @@ static FORCE_INLINE int32_t tSkipListUnlock(SSkipList *pSkipList) {
 static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData) {
   int32_t compare = 0;
   bool    hasDupKey = false;
-  char   *pDataKey = pSkipList->keyFn(pData);
+  char *  pDataKey = pSkipList->keyFn(pData);
 
   if (pSkipList->size == 0) {
     for (int32_t i = 0; i < pSkipList->maxLevel; i++) {
@@ -516,6 +521,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward,
   return hasDupKey;
 }
 
+#ifdef BUILD_NO_CALL
 static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) {
   int32_t level = pNode->level;
   uint8_t dupMode = SL_DUP_MODE(pSkipList);
@@ -540,6 +546,7 @@ static void tSkipListCorrectLevel(SSkipList *pSkipList) {
     pSkipList->level -= 1;
   }
 }
+#endif
 
 UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList,
                                                          int32_t    level) {  // record link count in each level

From c7932675ce271bb98506cea5a1be7600c0583304 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Wed, 21 Feb 2024 09:07:38 +0800
Subject: [PATCH 05/37] enh: get machine id

---
 source/dnode/mgmt/mgmt_dnode/src/dmHandle.c |  7 +------
 source/dnode/mgmt/node_mgmt/src/dmMgmt.c    | 11 +++++++++++
 source/dnode/mgmt/node_util/inc/dmUtil.h    |  1 +
 source/dnode/mnode/impl/src/mndDnode.c      |  2 +-
 4 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
index a3a29f6f77..fa59f56496 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
@@ -16,7 +16,6 @@
 #define _DEFAULT_SOURCE
 #include "dmInt.h"
 #include "systable.h"
-#include "tgrant.h"
 
 extern SConfig *tsCfg;
 
@@ -118,11 +117,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   req.memTotal = tsTotalMemoryKB * 1024;
   req.memAvail = req.memTotal - tsRpcQueueMemoryAllowed - 16 * 1024 * 1024;
   tstrncpy(req.dnodeEp, tsLocalEp, TSDB_EP_LEN);
-  char *machine = tGetMachineId();
-  if (machine) {
-    tstrncpy(req.machineId, machine, TSDB_MACHINE_ID_LEN + 1);
-    taosMemoryFreeClear(machine);
-  }
+  tstrncpy(req.machineId, pMgmt->pData->machineId, TSDB_MACHINE_ID_LEN + 1);
 
   req.clusterCfg.statusInterval = tsStatusInterval;
   req.clusterCfg.checkTime = 0;
diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c
index 6cbf31b15f..049b1bdf84 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c
@@ -22,6 +22,7 @@
 #ifdef TD_TSZ
 #include "tcompression.h"
 #include "tglobal.h"
+#include "tgrant.h"
 #endif
 
 static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
@@ -137,6 +138,16 @@ int32_t dmInitVars(SDnode *pDnode) {
   pData->rebootTime = taosGetTimestampMs();
   pData->dropped = 0;
   pData->stopped = 0;
+  char *machineId = tGetMachineId();
+  if (machineId) {
+    tstrncpy(pData->machineId, machineId, TSDB_MACHINE_ID_LEN + 1);
+    taosMemoryFreeClear(machineId);
+  } else {
+#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
+    terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
+    return -1;
+#endif
+  }
 
   pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
   if (pData->dnodeHash == NULL) {
diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h
index 4769ef8538..504c405506 100644
--- a/source/dnode/mgmt/node_util/inc/dmUtil.h
+++ b/source/dnode/mgmt/node_util/inc/dmUtil.h
@@ -109,6 +109,7 @@ typedef struct {
   SMsgCb         msgCb;
   bool           validMnodeEps;
   int64_t        ipWhiteVer;
+  char           machineId[TSDB_MACHINE_ID_LEN + 1];
 } SDnodeData;
 
 typedef struct {
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index 6974c6b0e8..c65c228224 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -141,7 +141,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
     memcpy(dnodeObj.machineId, machineId, TSDB_MACHINE_ID_LEN);
     taosMemoryFreeClear(machineId);
   } else {
-#ifdef TD_ENTERPRISE
+#if defined(TD_ENTERPRISE) && !defined(GRANTS_CFG)
     terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
     goto _OVER;
 #endif

From e91586c484a305992ab1c8dcead539ce2fa3a485 Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Wed, 21 Feb 2024 09:47:42 +0800
Subject: [PATCH 06/37] fix: free fd to reuse the space

---
 source/os/src/osFile.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c
index bab9ba0cea..e6491639dc 100644
--- a/source/os/src/osFile.c
+++ b/source/os/src/osFile.c
@@ -1329,7 +1329,6 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
   char   *data = taosMemoryMalloc(compressSize);
   gzFile  dstFp = NULL;
 
-  TdFilePtr pFile = NULL;
   TdFilePtr pSrcFile = NULL;
 
   pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ | TD_FILE_STREAM);
@@ -1369,8 +1368,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
   }
 
 cmp_end:
-  if (pFile) {
-    taosCloseFile(&pFile);
+  if (fd >= 0) {
+    close(fd);
   }
   if (pSrcFile) {
     taosCloseFile(&pSrcFile);

From d0b26bf0370703d4d7ff3d9258b64de8bd26ae72 Mon Sep 17 00:00:00 2001
From: yihaoDeng <yhdeng@taosdata.com>
Date: Wed, 21 Feb 2024 14:07:46 +0800
Subject: [PATCH 07/37] add coveriry

---
 source/libs/index/test/CMakeLists.txt |  4 ++
 source/libs/index/test/fstTest.cc     | 53 +++++++++++++++++----------
 2 files changed, 38 insertions(+), 19 deletions(-)

diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt
index 1ee835573b..eff74f93d2 100644
--- a/source/libs/index/test/CMakeLists.txt
+++ b/source/libs/index/test/CMakeLists.txt
@@ -175,4 +175,8 @@ IF(NOT TD_DARWIN)
     NAME idxFstUT 
     COMMAND idxFstUT 
   )
+  add_test(
+    NAME idxFstTest
+    COMMAND idxFstTest 
+  )
 ENDIF ()
diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc
index b889a2209a..2b1744c28d 100644
--- a/source/libs/index/test/fstTest.cc
+++ b/source/libs/index/test/fstTest.cc
@@ -1,4 +1,5 @@
 
+#include <gtest/gtest.h>
 #include <algorithm>
 #include <iostream>
 #include <string>
@@ -14,6 +15,12 @@
 #include "tutil.h"
 void* callback(void* s) { return s; }
 
+class FstEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() {}
+  virtual void TearDown() {}
+};
+
 static std::string fileName = TD_TMP_DIR_PATH "tindex.tindex";
 class FstWriter {
  public:
@@ -154,7 +161,7 @@ class FstReadMemory {
   int32_t     _size;
 };
 
-#define L 100
+#define L 200
 #define M 100
 #define N 100
 
@@ -637,23 +644,31 @@ void iterTFileReader(char* path, char* uid, char* colName, char* ver) {
   tfileIteratorDestroy(iter);
 }
 
-int main(int argc, char* argv[]) {
-  // tool to check all kind of fst test
-  // if (argc > 1) { validateTFile(argv[1]); }
-  // if (argc > 4) {
-  // path suid colName ver
-  // iterTFileReader(argv[1], argv[2], argv[3], argv[4]);
-  //}
-  checkFstCheckIterator1();
-  // checkFstCheckIterator2();
-  // checkFstCheckIteratorPrefix();
-  // checkFstCheckIteratorRange1();
-  // checkFstCheckIteratorRange2();
-  // checkFstCheckIteratorRange3();
-  // checkFstLongTerm();
-  // checkFstPrefixSearch();
+// int main(int argc, char* argv[]) {
+//   // tool to check all kind of fst test
+//   // if (argc > 1) { validateTFile(argv[1]); }
+//   // if (argc > 4) {
+//   // path suid colName ver
+//   // iterTFileReader(argv[1], argv[2], argv[3], argv[4]);
+//   //}
+//   checkFstCheckIterator1();
+//   // checkFstCheckIterator2();
+//   // checkFstCheckIteratorPrefix();
+//   // checkFstCheckIteratorRange1();
+//   // checkFstCheckIteratorRange2();
+//   // checkFstCheckIteratorRange3();
+//   // checkFstLongTerm();
+//   // checkFstPrefixSearch();
 
-  // checkMillonWriteAndReadOfFst();
+//   // checkMillonWriteAndReadOfFst();
 
-  return 1;
-}
+//   return 1;
+// }
+TEST_F(FstEnv, checkIterator1) { checkFstCheckIterator1(); }
+TEST_F(FstEnv, checkItertor2) { checkFstCheckIterator2(); }
+TEST_F(FstEnv, checkPrefix) { checkFstCheckIteratorPrefix(); }
+TEST_F(FstEnv, checkRange1) { checkFstCheckIteratorRange1(); }
+TEST_F(FstEnv, checkRange2) { checkFstCheckIteratorRange2(); }
+TEST_F(FstEnv, checkRange3) { checkFstCheckIteratorRange3(); }
+TEST_F(FstEnv, checkLongTerm) { checkFstLongTerm(); }
+TEST_F(FstEnv, checkMillonWriteData) { checkMillonWriteAndReadOfFst(); }

From d28a450882e322e6efec1f6e532208898a78cca2 Mon Sep 17 00:00:00 2001
From: yihaoDeng <yhdeng@taosdata.com>
Date: Wed, 21 Feb 2024 14:39:41 +0800
Subject: [PATCH 08/37] add coveriry

---
 source/libs/index/test/fstTest.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc
index 2b1744c28d..adf783d2eb 100644
--- a/source/libs/index/test/fstTest.cc
+++ b/source/libs/index/test/fstTest.cc
@@ -207,7 +207,7 @@ void checkMillonWriteAndReadOfFst() {
   FstWriter* fw = new FstWriter;
   Performance_fstWriteRecords(fw);
   delete fw;
-  FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024);
+  FstReadMemory* fr = new FstReadMemory(1024 * 8 * 1024);
 
   if (fr->init()) {
     printf("success to init fst read");

From 6e13e4aa638dacfc34e6652355fa9ecf2deb5ed1 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 22 Feb 2024 10:58:32 +0800
Subject: [PATCH 09/37] fix:add excluded msg for meta in tmq

---
 include/common/tmsg.h                     | 16 ++++-----
 source/client/src/clientRawBlockWrite.c   | 41 ++++++++++++++++++-----
 source/client/src/clientTmq.c             |  2 +-
 source/common/src/tmsg.c                  | 22 ++++++++----
 source/dnode/mnode/impl/inc/mndDef.h      |  1 +
 source/dnode/mnode/impl/src/mndStb.c      | 10 ++++--
 source/dnode/vnode/src/sma/smaTimeRange.c |  2 +-
 source/dnode/vnode/src/tq/tqRead.c        |  2 +-
 source/dnode/vnode/src/tq/tqScan.c        |  4 +--
 source/dnode/vnode/src/tq/tqSink.c        |  4 +--
 source/dnode/vnode/src/tq/tqUtil.c        | 29 ++++++++++++++++
 source/libs/parser/src/parInsertUtil.c    |  3 +-
 12 files changed, 100 insertions(+), 36 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 1b438ba026..587e2f9f3e 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -207,9 +207,6 @@ typedef enum _mgmt_table {
 #define TD_CHILD_TABLE  TSDB_CHILD_TABLE
 #define TD_NORMAL_TABLE TSDB_NORMAL_TABLE
 
-#define TD_REQ_FROM_APP  0
-#define TD_REQ_FROM_TAOX 1
-
 typedef enum ENodeType {
   // Syntax nodes are used in parser and planner module, and some are also used in executor module, such as COLUMN,
   // VALUE, OPERATOR, FUNCTION and so on.
@@ -759,7 +756,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW
 typedef struct {
   char     name[TSDB_TABLE_FNAME_LEN];
   int8_t   igExists;
-  int8_t   source;  // 1-taosX or 0-taosClient
+  int8_t   source;  // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
   int8_t   reserved[6];
   tb_uid_t suid;
   int64_t  delay1;
@@ -802,7 +799,7 @@ void    tFreeSMCreateStbRsp(SMCreateStbRsp* pRsp);
 typedef struct {
   char     name[TSDB_TABLE_FNAME_LEN];
   int8_t   igNotExists;
-  int8_t   source;  // 1-taosX or 0-taosClient
+  int8_t   source;  // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
   int8_t   reserved[6];
   tb_uid_t suid;
   int32_t  sqlLen;
@@ -2661,6 +2658,7 @@ typedef struct SVCreateStbReq {
   SRSmaParam     rsmaParam;
   int32_t        alterOriDataLen;
   void*          alterOriData;
+  int8_t         source;
 } SVCreateStbReq;
 
 int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq);
@@ -2730,6 +2728,7 @@ typedef struct {
     SVCreateTbReq* pReqs;
     SArray*        pArray;
   };
+  int8_t   source;  // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
 } SVCreateTbBatchReq;
 
 int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq);
@@ -2822,6 +2821,7 @@ typedef struct {
   int32_t newCommentLen;
   char*   newComment;
   int64_t ctimeMs;  // fill by vnode
+  int8_t  source;  // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
 } SVAlterTbReq;
 
 int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
@@ -3919,12 +3919,13 @@ int32_t tDeatroySMqHbRsp(SMqHbRsp* pRsp);
 int32_t tSerializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
 int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
 
+#define TD_REQ_FROM_APP               0x0
 #define SUBMIT_REQ_AUTO_CREATE_TABLE  0x1
 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2
 #define SUBMIT_REQ_FROM_FILE          0x4
+#define TD_REQ_FROM_TAOX              0x8
 
-#define SOURCE_NULL  0
-#define SOURCE_TAOSX 1
+#define TD_REQ_FROM_TAOX_OLD          0x1     // for compatibility
 
 typedef struct {
   int32_t        flags;
@@ -3937,7 +3938,6 @@ typedef struct {
     SArray* aCol;
   };
   int64_t ctimeMs;
-  int8_t  source;
 } SSubmitTbData;
 
 typedef struct {
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 448243cc3d..f143624bab 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1001,6 +1001,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
 
       tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq));
       taosArrayPush(tBatch.req.pArray, pCreateReq);
+      tBatch.req.source = TD_REQ_FROM_TAOX;
 
       taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch));
     } else {  // add to the correct vgroup
@@ -1276,7 +1277,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
     return terrno;
   }
   SVAlterTbReq   req = {0};
-  SDecoder       coder = {0};
+  SDecoder       dcoder = {0};
   int32_t        code = TSDB_CODE_SUCCESS;
   SRequestObj*   pRequest = NULL;
   SQuery*        pQuery = NULL;
@@ -1297,8 +1298,8 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
   // decode and process req
   void*   data = POINTER_SHIFT(meta, sizeof(SMsgHead));
   int32_t len = metaLen - sizeof(SMsgHead);
-  tDecoderInit(&coder, data, len);
-  if (tDecodeSVAlterTbReq(&coder, &req) < 0) {
+  tDecoderInit(&dcoder, data, len);
+  if (tDecodeSVAlterTbReq(&dcoder, &req) < 0) {
     code = TSDB_CODE_INVALID_PARA;
     goto end;
   }
@@ -1340,14 +1341,36 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
     goto end;
   }
   pVgData->vg = pInfo;
-  pVgData->pData = taosMemoryMalloc(metaLen);
-  if (NULL == pVgData->pData) {
+
+  int         tlen = 0;
+  req.source = TD_REQ_FROM_TAOX;
+  tEncodeSize(tEncodeSVAlterTbReq, &req, tlen, code);
+  if(code != 0){
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto end;
   }
-  memcpy(pVgData->pData, meta, metaLen);
-  ((SMsgHead*)pVgData->pData)->vgId = htonl(pInfo.vgId);
-  pVgData->size = metaLen;
+  tlen += sizeof(SMsgHead);
+  void* pMsg = taosMemoryMalloc(tlen);
+  if (NULL == pMsg) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  ((SMsgHead*)pMsg)->vgId = htonl(pInfo.vgId);
+  ((SMsgHead*)pMsg)->contLen = htonl(tlen);
+  void*    pBuf = POINTER_SHIFT(pMsg, sizeof(SMsgHead));
+  SEncoder coder = {0};
+  tEncoderInit(&coder, pBuf, tlen - sizeof(SMsgHead));
+  code = tEncodeSVAlterTbReq(&coder, &req);
+  if(code != 0){
+    tEncoderClear(&coder);
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  tEncoderClear(&coder);
+
+  pVgData->pData = pMsg;
+  pVgData->size = tlen;
+
   pVgData->numOfTables = 1;
   taosArrayPush(pArray, &pVgData);
 
@@ -1387,7 +1410,7 @@ end:
   if (pVgData) taosMemoryFreeClear(pVgData->pData);
   taosMemoryFreeClear(pVgData);
   destroyRequest(pRequest);
-  tDecoderClear(&coder);
+  tDecoderClear(&dcoder);
   qDestroyQuery(pQuery);
   terrno = code;
   return code;
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 1904874a0b..b141921a77 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -389,7 +389,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
     }
   }
   if (strcasecmp(key, "msg.consume.excluded") == 0) {
-    conf->sourceExcluded = taosStr2int64(value);
+    conf->sourceExcluded = (taosStr2int64(value) != 0) ? TD_REQ_FROM_TAOX : 0;
     return TMQ_CONF_OK;
   }
 
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 2ffa12f2c1..85f5d462c7 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -7513,6 +7513,7 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) {
   if (pReq->alterOriDataLen > 0) {
     if (tEncodeBinary(pCoder, pReq->alterOriData, pReq->alterOriDataLen) < 0) return -1;
   }
+  if (tEncodeI8(pCoder, pReq->source) < 0) return -1;
 
   tEndEncode(pCoder);
   return 0;
@@ -7535,6 +7536,10 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
     if (tDecodeBinary(pCoder, (uint8_t **)&pReq->alterOriData, NULL) < 0) return -1;
   }
 
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI8(pCoder, &pReq->source) < 0) return -1;
+  }
+
   tEndDecode(pCoder);
   return 0;
 }
@@ -7663,6 +7668,8 @@ int tEncodeSVCreateTbBatchReq(SEncoder *pCoder, const SVCreateTbBatchReq *pReq)
     if (tEncodeSVCreateTbReq(pCoder, (SVCreateTbReq *)taosArrayGet(pReq->pArray, iReq)) < 0) return -1;
   }
 
+  if (tEncodeI8(pCoder, pReq->source) < 0) return -1;
+
   tEndEncode(pCoder);
   return 0;
 }
@@ -7677,6 +7684,10 @@ int tDecodeSVCreateTbBatchReq(SDecoder *pCoder, SVCreateTbBatchReq *pReq) {
     if (tDecodeSVCreateTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1;
   }
 
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI8(pCoder, &pReq->source) < 0) return -1;
+  }
+
   tEndDecode(pCoder);
   return 0;
 }
@@ -8034,6 +8045,7 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) {
       break;
   }
   if (tEncodeI64(pEncoder, pReq->ctimeMs) < 0) return -1;
+  if (tEncodeI8(pEncoder, pReq->source) < 0) return -1;
 
   tEndEncode(pEncoder);
   return 0;
@@ -8094,6 +8106,9 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) {
   if (!tDecodeIsEnd(pDecoder)) {
     if (tDecodeI64(pDecoder, &pReq->ctimeMs) < 0) return -1;
   }
+  if (!tDecodeIsEnd(pDecoder)) {
+    if (tDecodeI8(pDecoder, &pReq->source) < 0) return -1;
+  }
 
   tEndDecode(pDecoder);
   return 0;
@@ -8670,7 +8685,6 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm
     }
   }
   if (tEncodeI64(pCoder, pSubmitTbData->ctimeMs) < 0) return -1;
-  if (tEncodeI8(pCoder, pSubmitTbData->source) < 0) return -1;
 
   tEndEncode(pCoder);
   return 0;
@@ -8758,12 +8772,6 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa
       goto _exit;
     }
   }
-  if (!tDecodeIsEnd(pCoder)) {
-    if (tDecodeI8(pCoder, &pSubmitTbData->source) < 0) {
-      code = TSDB_CODE_INVALID_MSG;
-      goto _exit;
-    }
-  }
 
   tEndDecode(pCoder);
 
diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h
index fc58fabad5..4a092057ce 100644
--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@@ -464,6 +464,7 @@ typedef struct {
   char*    pAst1;
   char*    pAst2;
   SRWLatch lock;
+  int8_t   source;
 } SStbObj;
 
 typedef struct {
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 1ddd2f34e6..aad1dc881b 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -458,6 +458,7 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3
   req.rollup = pStb->ast1Len > 0 ? 1 : 0;
   req.alterOriData = alterOriData;
   req.alterOriDataLen = alterOriDataLen;
+  req.source = pStb->source;
   // todo
   req.schemaRow.nCols = pStb->numOfColumns;
   req.schemaRow.version = pStb->colVer;
@@ -774,7 +775,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
   pDst->createdTime = taosGetTimestampMs();
   pDst->updateTime = pDst->createdTime;
   pDst->uid =
-      (pCreate->source == TD_REQ_FROM_TAOX) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
+      (pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX)
+          ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
   pDst->dbUid = pDb->uid;
   pDst->tagVer = 1;
   pDst->colVer = 1;
@@ -790,6 +792,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
   pDst->numOfFuncs = pCreate->numOfFuncs;
   pDst->commentLen = pCreate->commentLen;
   pDst->pFuncs = pCreate->pFuncs;
+  pDst->source = pCreate->source;
   pCreate->pFuncs = NULL;
 
   if (pDst->commentLen > 0) {
@@ -1033,6 +1036,7 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
   memcpy(pDst, pStb, sizeof(SStbObj));
   taosRUnLockLatch(&pStb->lock);
 
+  pDst->source = createReq->source;
   pDst->updateTime = taosGetTimestampMs();
   pDst->numOfColumns = createReq->numOfColumns;
   pDst->numOfTags = createReq->numOfTags;
@@ -1141,7 +1145,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
     }
   } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) {
     goto _OVER;
-  } else if (createReq.source == TD_REQ_FROM_TAOX && (createReq.tagVer != 1 || createReq.colVer != 1)) {
+  } else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) && (createReq.tagVer != 1 || createReq.colVer != 1)) {
     mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name);
     code = 0;
     goto _OVER;
@@ -2572,7 +2576,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
     }
   }
 
-  if (dropReq.source == TD_REQ_FROM_TAOX && pStb->uid != dropReq.suid) {
+  if ((dropReq.source == TD_REQ_FROM_TAOX_OLD || dropReq.source == TD_REQ_FROM_TAOX) && pStb->uid != dropReq.suid) {
     code = 0;
     goto _OVER;
   }
diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c
index 54d859992c..fb898c02f8 100644
--- a/source/dnode/vnode/src/sma/smaTimeRange.c
+++ b/source/dnode/vnode/src/sma/smaTimeRange.c
@@ -193,7 +193,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
       continue;
     }
 
-    SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE, .source = SOURCE_NULL};
+    SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE};
 
     int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
     tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true);
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 727157a2f8..7a737fbb5d 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -392,7 +392,7 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
               pReader->msg.ver);
 
       SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
-      if ((pSubmitTbData->source & sourceExcluded) != 0) {
+      if ((pSubmitTbData->flags & sourceExcluded) != 0) {
         pReader->nextBlk += 1;
         continue;
       }
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index f108cedcf5..103007eb57 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -267,7 +267,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
         if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_table;
       }
 
-      if ((pSubmitTbDataRet->source & sourceExcluded) != 0) {
+      if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) {
         goto loop_table;
       }
       if (pRsp->withTbName) {
@@ -335,7 +335,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
         if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_db;
       }
 
-      if ((pSubmitTbDataRet->source & sourceExcluded) != 0) {
+      if ((pSubmitTbDataRet->flags & sourceExcluded) != 0) {
         goto loop_db;
       }
       if (pRsp->withTbName) {
diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c
index b56bf3e0fe..42bd1bc59c 100644
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@@ -815,7 +815,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
           return;
         }
 
-        SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL};
+        SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP};
         code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData);
         if (code != TSDB_CODE_SUCCESS) {
           continue;
@@ -860,7 +860,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
       pTask->execInfo.sink.numOfBlocks += 1;
       uint64_t groupId = pDataBlock->info.id.groupId;
 
-      SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL};
+      SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP};
 
       int32_t* index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId));
       if (index == NULL) {  // no data yet, append it
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index 31267dbf52..dad1211326 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -171,6 +171,22 @@ end : {
 }
 }
 
+#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC) \
+  SDecoder           decoder = {0};\
+  TYPE               req = {0}; \
+  void*   data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \
+  int32_t len = pHead->bodyLen - sizeof(SMsgHead); \
+  tDecoderInit(&decoder, data, len); \
+  if (DECODE_FUNC(&decoder, &req) == 0 && (req.source & TD_REQ_FROM_TAOX) != 0) { \
+    tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 " msgType %d",  \
+            pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \
+    fetchVer++; \
+    tDecoderClear(&decoder); \
+    continue; \
+  } \
+  tDecoderClear(&decoder);
+
+
 static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
                                                   SRpcMsg* pMsg, STqOffsetVal* offset) {
   int        code = 0;
@@ -239,6 +255,19 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
           goto end;
         }
 
+        if ((pRequest->sourceExcluded & TD_REQ_FROM_TAOX) != 0) {
+          if (pHead->msgType == TDMT_VND_CREATE_TABLE) {
+            PROCESS_EXCLUDED_MSG(SVCreateTbBatchReq, tDecodeSVCreateTbBatchReq)
+          } else if (pHead->msgType == TDMT_VND_ALTER_TABLE) {
+            PROCESS_EXCLUDED_MSG(SVAlterTbReq, tDecodeSVAlterTbReq)
+          } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) {
+            PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq)
+          } else if (pHead->msgType == TDMT_VND_DELETE) {
+            fetchVer++;
+            continue;
+          }
+        }
+
         tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType));
         tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1);
         metaRsp.resMsgType = pHead->msgType;
diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c
index 8900ef8e7f..cd38b67fe9 100644
--- a/source/libs/parser/src/parInsertUtil.c
+++ b/source/libs/parser/src/parInsertUtil.c
@@ -285,7 +285,6 @@ static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) {
     pTmp->suid = pSrc->suid;
     pTmp->uid = pSrc->uid;
     pTmp->sver = pSrc->sver;
-    pTmp->source = pSrc->source;
     pTmp->pCreateTbReq = NULL;
     if (pTmp->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) {
       if (pSrc->pCreateTbReq) {
@@ -653,7 +652,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
     goto end;
   }
 
-  pTableCxt->pData->source = SOURCE_TAOSX;
+  pTableCxt->pData->flags |= TD_REQ_FROM_TAOX;
   if(tmp == NULL){
     ret = initTableColSubmitData(pTableCxt);
     if (ret != TSDB_CODE_SUCCESS) {

From c22360a529ef92e3773d3399684ff3b5a9fe2831 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 22 Feb 2024 11:32:45 +0800
Subject: [PATCH 10/37] fix:add excluded msg for meta in tmq

---
 tests/system-test/7-tmq/tmq_taosx.py |  4 ++--
 utils/test/c/tmq_taosx_ci.c          | 10 +++++++++-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py
index ac8a6f377c..c547385d70 100644
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@@ -323,7 +323,7 @@ class TDTestCase:
         tdSql.query("select * from st")
         tdSql.checkRows(8)
 
-        tdSql.execute(f'create topic topic_excluded with meta as database d1')
+        tdSql.execute(f'create topic topic_all with meta as database d1')
         consumer_dict = {
             "group.id": "g1",
             "td.connect.user": "root",
@@ -333,7 +333,7 @@ class TDTestCase:
         consumer = Consumer(consumer_dict)
 
         try:
-            consumer.subscribe(["topic_excluded"])
+            consumer.subscribe(["topic_all"])
         except TmqError:
             tdLog.exit(f"subscribe error")
 
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 056b7dc6cf..2257089f06 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -20,6 +20,7 @@
 #include <time.h>
 #include "taos.h"
 #include "types.h"
+#include "tmsg.h"
 
 static int running = 1;
 TdFilePtr  g_fp = NULL;
@@ -966,7 +967,14 @@ void testConsumeExcluded(int topic_type){
       tmq_raw_data raw = {0};
       tmq_get_raw(msg, &raw);
       if(topic_type == 1){
-        assert(raw.raw_type != 2 && raw.raw_type != 4);
+        assert(raw.raw_type != 2 && raw.raw_type != 4 &&
+               raw.raw_type != TDMT_VND_CREATE_STB &&
+               raw.raw_type != TDMT_VND_ALTER_STB &&
+               raw.raw_type != TDMT_VND_CREATE_TABLE &&
+               raw.raw_type != TDMT_VND_ALTER_TABLE &&
+               raw.raw_type != TDMT_VND_DELETE);
+        assert(raw.raw_type == TDMT_VND_DROP_STB ||
+               raw.raw_type == TDMT_VND_DROP_TABLE);
       }else if(topic_type == 2){
         assert(0);
       }

From 8c6ea4079c0c72918c8494d54f82f55a78454451 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 22 Feb 2024 14:15:35 +0800
Subject: [PATCH 11/37] delete invalid result

---
 .../executor/src/streamcountwindowoperator.c     | 16 +++++++++++++---
 source/libs/stream/src/streamSessionState.c      |  3 +--
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c
index 080f9d4e2b..689ba54ca6 100644
--- a/source/libs/executor/src/streamcountwindowoperator.c
+++ b/source/libs/executor/src/streamcountwindowoperator.c
@@ -115,8 +115,16 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
   }
 }
 
-static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs, int32_t start, int32_t rows, int32_t maxRows,
-                              SSHashObj* pStDeleted, bool* pRebuild) {
+static void removeCountResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) {
+  SSessionKey key = {0};
+  getSessionHashKey(pKey, &key);
+  tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));
+  tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey));
+}
+
+static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowInfo* pWinInfo, TSKEY* pTs,
+                                     int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated,
+                                     SSHashObj* pStDeleted, bool* pRebuild) {
   SSessionKey sWinKey = pWinInfo->winInfo.sessionWin;
   int32_t num = 0;
   for (int32_t i = start; i < rows; i++) {
@@ -148,6 +156,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI
 
   if (needDelState) {
     memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));
+    removeCountResult(pStUpdated, pAggSup->pResultRows, &sWinKey);
     if (pWinInfo->winInfo.pStatePos->needFree) {
       pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &sWinKey);
     }
@@ -242,7 +251,8 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
     setSessionWinOutputInfo(pStUpdated, &curWin.winInfo);
     slidingRows = *curWin.pWindowCount;
     if (!buffInfo.rebuildWindow) {
-      winRows = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStDeleted, &buffInfo.rebuildWindow);
+      winRows = updateCountWindowInfo(pAggSup, &curWin, startTsCols, i, rows, pAggSup->windowCount, pStUpdated,
+                                      pStDeleted, &buffInfo.rebuildWindow);
     }
     if (buffInfo.rebuildWindow) {
       SSessionKey range = {0};
diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c
index bd28d2bca9..e2aae130e5 100644
--- a/source/libs/stream/src/streamSessionState.c
+++ b/source/libs/stream/src/streamSessionState.c
@@ -736,6 +736,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
     void* pRockVal = NULL;
     SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileStore, pKey->groupId);
     code = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &pRockVal, pVLen);
+    streamStateFreeCur(pCur);
     if (code == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) {
       qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code);
       if (code == TSDB_CODE_SUCCESS) {
@@ -743,7 +744,6 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
         COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)( (char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE)) );
         if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) {
           (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen);
-          streamStateFreeCur(pCur);
           goto _end;
         }
       }
@@ -751,7 +751,6 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
       pWinKey->win.ekey = endTs;
       (*pVal) = createSessionWinBuff(pFileState, pWinKey, NULL, NULL);
       taosMemoryFree(pRockVal);
-      streamStateFreeCur(pCur);
     } else {
       (*pVal) = addNewSessionWindow(pFileState, pWinStates, pWinKey);
       code = TSDB_CODE_FAILED;

From 7c8eb21ee541cf73d632e2e97ecaf0b8e6122407 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 22 Feb 2024 15:01:04 +0800
Subject: [PATCH 12/37] fix(stream): halt the correct task.

---
 source/dnode/mnode/impl/src/mndScheduler.c | 24 ++++++++++++----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c
index cbc0ace75d..ef9a7205e1 100644
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@@ -44,9 +44,8 @@ static bool hasCountWindowNode(SPhysiNode* pNode) {
   }
 }
 
-static bool countWindowStreamTask(SSubplan* pPlan) {
-  SPhysiNode* pNode = pPlan->pNode;
-  return hasCountWindowNode(pNode);
+static bool isCountWindowStreamTask(SSubplan* pPlan) {
+  return hasCountWindowNode((SPhysiNode*)pPlan->pNode);
 }
 
 int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
@@ -342,13 +341,13 @@ static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVe
   }
 }
 
-static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan) {
-  bool hasCountWindowNode = countWindowStreamTask(pPlan);
-  bool isRelStreamTask = (pTask->hTaskInfo.id.taskId != 0);
-  if (hasCountWindowNode && isRelStreamTask) {
+static void haltInitialTaskStatus(SStreamTask* pTask, SSubplan* pPlan, bool isFillhistoryTask) {
+  bool hasCountWindowNode = isCountWindowStreamTask(pPlan);
+
+  if (hasCountWindowNode && (!isFillhistoryTask)) {
     SStreamStatus* pStatus = &pTask->status;
-    mDebug("s-task:0x%x status is set to %s from %s for count window agg task with fill-history option set",
-           pTask->id.taskId, streamTaskGetStatusStr(pStatus->taskStatus), streamTaskGetStatusStr(TASK_STATUS__HALT));
+    mDebug("s-task:0x%x status set %s from %s for count window agg task with fill-history option set",
+           pTask->id.taskId, streamTaskGetStatusStr(TASK_STATUS__HALT), streamTaskGetStatusStr(pStatus->taskStatus));
     pStatus->taskStatus = TASK_STATUS__HALT;
   }
 }
@@ -398,15 +397,17 @@ static void setHTasksId(SStreamObj* pStream) {
 
 static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset, int64_t skey,
                                SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam) {
-  // new stream task
   SStreamTask* pTask = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam);
   if (pTask == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return terrno;
   }
+
   mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);
 
-  haltInitialTaskStatus(pTask, plan);
+  if (pStream->conf.fillHistory) {
+    haltInitialTaskStatus(pTask, plan, isFillhistory);
+  }
 
   streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
 
@@ -415,6 +416,7 @@ static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStre
     terrno = code;
     return terrno;
   }
+
   return TDB_CODE_SUCCESS;
 }
 

From c859213ce54a9faba038ca89ddbb0a6e90d7b89d Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 22 Feb 2024 15:34:12 +0800
Subject: [PATCH 13/37] fix: free datablocks that is skipped decided by firt
 row ts

---
 source/libs/executor/src/tsort.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index fde5f22aaa..9ff903cdb9 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1125,8 +1125,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
       int64_t firstRowTs = *(int64_t*)tsCol->pData;
       if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs)  ||
           (pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) {
-            continue;
-          }
+        if (bExtractedBlock) {
+          blockDataDestroy(pBlk);
+	}
+        continue;
+      }
     }
 
     if (pBlk != NULL) {
@@ -1149,10 +1152,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
       tSimpleHashClear(mUidBlk);
       code = sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc);
       if (code != TSDB_CODE_SUCCESS) {
-        tSimpleHashCleanup(mUidBlk);
-        taosArrayDestroy(aBlkSort);
-        taosArrayDestroy(aExtSrc);
-        return code;
+        for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) {
+          blockDataDestroy(taosArrayGetP(aBlkSort, i));
+        }
+        taosArrayClear(aBlkSort);
+        break;	
       }
 
       int64_t el = taosGetTimestampUs() - p;
@@ -1165,6 +1169,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
       szSort = 0;
       qDebug("source %zu created", taosArrayGetSize(aExtSrc));
     }
+
     if (pBlk == NULL) {
       break;
     }
@@ -1180,6 +1185,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   }
 
   tSimpleHashCleanup(mUidBlk);
+  for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) {
+    blockDataDestroy(taosArrayGetP(aBlkSort, i));
+  }
   taosArrayDestroy(aBlkSort);
   tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL);
   if (!tsortIsClosed(pHandle)) {
@@ -1188,7 +1196,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   taosArrayDestroy(aExtSrc);
   tSimpleHashCleanup(mTableNumRows);
   pHandle->type = SORT_SINGLESOURCE_SORT;
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {

From 4037a7a1a5070666b5bb312e995a10ce90b3b6bb Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 22 Feb 2024 15:44:50 +0800
Subject: [PATCH 14/37] fix:change datablock to old version for compatibility

---
 include/common/tdatablock.h                 |  5 ++++-
 source/client/src/clientImpl.c              | 10 +++++-----
 source/client/src/clientMsgHandler.c        |  4 ++--
 source/common/src/tdatablock.c              | 10 ++++++----
 source/dnode/mgmt/mgmt_dnode/src/dmHandle.c |  2 +-
 source/dnode/mnode/impl/src/mndShow.c       |  2 +-
 source/dnode/mnode/impl/src/mndStb.c        |  2 +-
 source/dnode/vnode/src/tq/tqScan.c          |  2 +-
 source/libs/command/src/command.c           |  2 +-
 source/libs/command/src/explain.c           |  2 +-
 source/libs/executor/src/dataDispatcher.c   |  2 +-
 source/libs/parser/src/parInsertUtil.c      |  4 ++--
 source/libs/stream/src/streamDispatch.c     |  4 ++--
 13 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index bcb40f4175..e305f4c2ec 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -32,6 +32,9 @@ typedef struct SBlockOrderInfo {
   SColumnInfoData* pColData;
 } SBlockOrderInfo;
 
+#define BLOCK_VERSION_1          1
+#define BLOCK_VERSION_2          2
+
 #define NBIT                     (3u)
 #define BitPos(_n)               ((_n) & ((1 << NBIT) - 1))
 #define BMCharPos(bm_, r_)       ((bm_)[(r_) >> NBIT])
@@ -253,7 +256,7 @@ SColumnInfoData  createColumnInfoData(int16_t type, int32_t bytes, int16_t colId
 SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index);
 
 int32_t blockGetEncodeSize(const SSDataBlock* pBlock);
-int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols);
+int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion);
 const char* blockDecode(SSDataBlock* pBlock, const char* pData);
 
 // for debug
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 9800d233e9..a8ebef763a 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -1811,7 +1811,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i
 
   char* pStart = p + len;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i];
+    int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i];
 
     if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) {
       int32_t* offset = (int32_t*)pStart;
@@ -1910,8 +1910,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
   char* pStart = p;
   char* pStart1 = p1;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i];
-    int32_t colLen1 = (blockVersion == 1) ? htonl(colLength1[i]) : colLength1[i];
+    int32_t colLen = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength[i]) : colLength[i];
+    int32_t colLen1 = (blockVersion == BLOCK_VERSION_1) ? htonl(colLength1[i]) : colLength1[i];
     if (ASSERT(colLen < dataLen)) {
       tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen);
       return TSDB_CODE_TSC_INTERNAL_ERROR;
@@ -1970,7 +1970,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int
       }
       colLen1 = len;
       totalLen += colLen1;
-      colLength1[i] = (blockVersion == 1) ? htonl(len) : len;
+      colLength1[i] = (blockVersion == BLOCK_VERSION_1) ? htonl(len) : len;
     } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
       len = numOfRows * sizeof(int32_t);
       memcpy(pStart1, pStart, len);
@@ -2059,7 +2059,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
 
   char* pStart = p;
   for (int32_t i = 0; i < numOfCols; ++i) {
-    if(blockVersion == 1){
+    if(blockVersion == BLOCK_VERSION_1){
       colLength[i] = htonl(colLength[i]);
     }
     if (colLength[i] >= dataLen) {
diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c
index e0cedb9924..3aa46d135e 100644
--- a/source/client/src/clientMsgHandler.c
+++ b/source/client/src/clientMsgHandler.c
@@ -499,7 +499,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
   (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
   (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS);
 
-  int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS);
+  int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS, BLOCK_VERSION_1);
   blockDataDestroy(pBlock);
 
   if (len != rspSize - sizeof(SRetrieveTableRsp)) {
@@ -611,7 +611,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr
   (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
   (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS);
 
-  int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS);
+  int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS, BLOCK_VERSION_1);
   blockDataDestroy(pBlock);
 
   if (len != rspSize - sizeof(SRetrieveTableRsp)) {
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 5382259899..c6f004f83b 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -2191,12 +2191,12 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
+int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion) {
   int32_t dataLen = 0;
 
   // todo extract method
   int32_t* version = (int32_t*)data;
-  *version = 2;
+  *version = bVersion;
   data += sizeof(int32_t);
 
   int32_t* actualLen = (int32_t*)data;
@@ -2277,7 +2277,9 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
       data += colSizes[col];
     }
 
-//    colSizes[col] = htonl(colSizes[col]);
+    if(bVersion == BLOCK_VERSION_1){
+      colSizes[col] = htonl(colSizes[col]);
+    }
     //    uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type,
     //    htonl(colSizes[col]), colSizes[col]);
   }
@@ -2338,7 +2340,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) {
   pStart += sizeof(int32_t) * numOfCols;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
-    if(version == 1){
+    if(version == BLOCK_VERSION_1){
       colLen[i] = htonl(colLen[i]);
     }
     ASSERT(colLen[i] >= 0);
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
index 14853009e0..b36dc71322 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
@@ -401,7 +401,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     pStart += sizeof(SSysTableSchema);
   }
 
-  int32_t len = blockEncode(pBlock, pStart, numOfCols);
+  int32_t len = blockEncode(pBlock, pStart, numOfCols, BLOCK_VERSION_1);
 
   pRsp->numOfRows = htonl(pBlock->info.rows);
   pRsp->precision = TSDB_TIME_PRECISION_MILLI;  // millisecond time precision
diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c
index 8e7e72aa0e..e46c057129 100644
--- a/source/dnode/mnode/impl/src/mndShow.c
+++ b/source/dnode/mnode/impl/src/mndShow.c
@@ -328,7 +328,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
       pStart += sizeof(SSysTableSchema);
     }
 
-    int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns);
+    int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1);
   }
 
   pRsp->numOfRows = htonl(rowsRead);
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 1ddd2f34e6..ed1192c490 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -3150,7 +3150,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t
 //      pStart += sizeof(SSysTableSchema);
 //    }
 //
-//    int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns);
+//    int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1);
 //  }
 //
 //  pRsp->numOfRows = htonl(rowsRead);
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index 5432637482..4e4e1481f9 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -28,7 +28,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t
   pRetrieve->compressed = 0;
   pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows);
 
-  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols);
+  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_2);
   actualLen += sizeof(SRetrieveTableRspForTmq);
   taosArrayPush(pRsp->blockDataLen, &actualLen);
   taosArrayPush(pRsp->blockData, &buf);
diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c
index 6bae0e1022..e74c735497 100644
--- a/source/libs/command/src/command.c
+++ b/source/libs/command/src/command.c
@@ -40,7 +40,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe
   (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
   (*pRsp)->numOfCols = htonl(numOfCols);
 
-  int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols);
+  int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols, BLOCK_VERSION_1);
 
   return TSDB_CODE_SUCCESS;
 }
diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c
index 66b50bcb47..d2d14bb9f3 100644
--- a/source/libs/command/src/explain.c
+++ b/source/libs/command/src/explain.c
@@ -1817,7 +1817,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
   rsp->completed = 1;
   rsp->numOfRows = htobe64((int64_t)rowNum);
 
-  int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock));
+  int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock), BLOCK_VERSION_1);
 
   rsp->compLen = htonl(len);
 
diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c
index abe566473f..fb57d6b531 100644
--- a/source/libs/executor/src/dataDispatcher.c
+++ b/source/libs/executor/src/dataDispatcher.c
@@ -76,7 +76,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
   pEntry->dataLen = 0;
 
   pBuf->useSize = sizeof(SDataCacheEntry);
-  pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols);
+  pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols, BLOCK_VERSION_1);
   //  ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8));
   //  ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4));
 
diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c
index 6b655bfae6..514f2ca64e 100644
--- a/source/libs/parser/src/parInsertUtil.c
+++ b/source/libs/parser/src/parInsertUtil.c
@@ -718,7 +718,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
         goto end;
       }
       fields += sizeof(int8_t) + sizeof(int32_t);
-      if (needChangeLength && version == 1) {
+      if (needChangeLength && version == BLOCK_VERSION_1) {
         pStart += htonl(colLength[j]);
       } else {
         pStart += colLength[j];
@@ -749,7 +749,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
             goto end;
           }
           fields += sizeof(int8_t) + sizeof(int32_t);
-          if (needChangeLength && version == 1) {
+          if (needChangeLength && version == BLOCK_VERSION_1) {
             pStart += htonl(colLength[i]);
           } else {
             pStart += colLength[i];
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index b51845d152..080c2c9556 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -188,7 +188,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
   pRetrieve->ekey = htobe64(pBlock->info.window.ekey);
   pRetrieve->version = htobe64(pBlock->info.version);
 
-  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols);
+  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1);
 
   SStreamRetrieveReq req = {
       .streamId = pTask->id.streamId,
@@ -772,7 +772,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch
   int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
   pRetrieve->numOfCols = htonl(numOfCols);
 
-  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols);
+  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1);
   actualLen += sizeof(SRetrieveTableRsp);
   ASSERT(actualLen <= dataStrLen);
   taosArrayPush(pReq->dataLen, &actualLen);

From da0265ddde86bdb0136ab5ba14ff1ee4a39f8e26 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 22 Feb 2024 15:52:47 +0800
Subject: [PATCH 15/37] fix(tsdb): set correct error code info extract method.

---
 source/dnode/vnode/src/tsdb/tsdbRead2.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c
index 86f58717e2..90a26d17dc 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c
@@ -2543,7 +2543,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr
   while (1) {
     // only check here, since the iterate data in memory is very fast.
     if (pReader->code != TSDB_CODE_SUCCESS) {
-      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr);
+      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);
       return pReader->code;
     }
 
@@ -2694,7 +2694,7 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) {
 
   while (1) {
     if (pReader->code != TSDB_CODE_SUCCESS) {
-      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr);
+      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);
       return pReader->code;
     }
 
@@ -2909,7 +2909,7 @@ static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_
 
   while (1) {
     if (pReader->code != TSDB_CODE_SUCCESS) {
-      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr);
+      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);
       return pReader->code;
     }
 
@@ -2951,7 +2951,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en
 
   while (1) {
     if (pReader->code != TSDB_CODE_SUCCESS) {
-      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr);
+      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);
       return pReader->code;
     }
 

From ac36dd46d5e5f4b27b5639b6bc0bffa29afcaa78 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 22 Feb 2024 16:32:09 +0800
Subject: [PATCH 16/37] fix: add test case

---
 tests/parallel_test/cases.task           |  1 +
 tests/system-test/2-query/tms_memleak.py | 51 ++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 tests/system-test/2-query/tms_memleak.py

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index cc15df33a1..12f1f62f63 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -49,6 +49,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_interval.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/compact-col.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tms_memleak.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 3
diff --git a/tests/system-test/2-query/tms_memleak.py b/tests/system-test/2-query/tms_memleak.py
new file mode 100644
index 0000000000..0d5cdd8272
--- /dev/null
+++ b/tests/system-test/2-query/tms_memleak.py
@@ -0,0 +1,51 @@
+import sys 
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+from math import inf
+
+class TDTestCase:
+    def caseDescription(self):
+        '''
+        case1<shenglian zhou>: [TD-] 
+        ''' 
+        return
+    
+    def init(self, conn, logSql, replicaVer=1):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), True)
+        self.conn = conn
+        
+    def restartTaosd(self, index=1, dbname="db"):
+        tdDnodes.stop(index)
+        tdDnodes.startWithoutSleep(index)
+        tdSql.execute(f"use tms_memleak")
+
+    def run(self):
+        print("running {}".format(__file__))
+        tdSql.execute("drop database if exists tms_memleak")
+        tdSql.execute("create database if not exists tms_memleak")
+        tdSql.execute('use tms_memleak')
+
+        tdSql.execute('create table st(ts timestamp, f int) tags (t int);')
+
+        tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1)('2021-04-19 00:00:02', 2)('2021-04-19 00:00:03', 3)('2021-04-19 00:00:04', 4)")
+
+        tdSql.execute("insert into ct2 using st tags(2) values('2021-04-20 00:00:01', 5)('2021-04-20 00:00:02', 6)('2021-04-20 00:00:03', 7)('2021-04-20 00:00:04', 8)")
+
+        tdSql.execute("insert into ct3 using st tags(3) values('2021-04-21 00:00:01', 5)('2021-04-21 00:00:02', 6)('2021-04-21 00:00:03', 7)('2021-04-21 00:00:04', 8)")
+
+        tdSql.execute("insert into ct4 using st tags(4) values('2021-04-22 00:00:01', 5)('2021-04-22 00:00:02', 6)('2021-04-22 00:00:03', 7)('2021-04-22 00:00:04', 8)")
+
+        tdSql.query("select * from st order by ts  limit 1 ");
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 1, 1);
+
+        tdSql.execute('drop database tms_memleak')
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

From d8361e979c50911b281ae1b6237b75ab65e79f84 Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Thu, 22 Feb 2024 16:47:27 +0800
Subject: [PATCH 17/37] fix: add check s3 upload over logic

---
 tests/army/enterprise/s3/s3_basic.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py
index 58dbd12d7b..5d9a52ebc4 100644
--- a/tests/army/enterprise/s3/s3_basic.py
+++ b/tests/army/enterprise/s3/s3_basic.py
@@ -81,15 +81,23 @@ class TDTestCase(TBase):
         cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data"
         tdLog.info(cmd)
         loop = 0
-        while len(eos.runRetList(cmd)) > 0 and loop < 100:
-            time.sleep(5)
+        rets = []
+        while loop < 500:
+            rets = eos.runRetList(cmd)
+            cnt = len(rets)
+            if cnt == 0:
+                tdLog.info("All data file upload to server over.")
+                break
+            time.sleep(3)
             self.trimDb(True)
             loop += 1
-            tdLog.info(f"loop={loop} wait 5s...")
+            tdLog.info(f"loop={loop} data files have {cnt} wait 5s...")
             if loop == 4:
                 sc.dnodeStop(1)
                 time.sleep(2)
                 sc.dnodeStart(1)
+        if len(rets) > 0:
+            tdLog.exit(f"s3 can not upload all data to server. data files={rets}")
 
     def checkStreamCorrect(self):
         sql = f"select count(*) from {self.db}.stm1"

From 6ac6e8f2bd382c2e3b41dd604e43f197135a770c Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Thu, 22 Feb 2024 17:46:36 +0800
Subject: [PATCH 18/37] enh: unit test and optimization for base58

---
 include/util/tbase58.h              |  3 ++
 source/util/src/tbase58.c           | 34 +++++++++---
 source/util/test/CMakeLists.txt     |  8 +++
 source/util/test/tbaseCodecTest.cpp | 80 +++++++++++++++++++++++++++++
 4 files changed, 117 insertions(+), 8 deletions(-)
 create mode 100644 source/util/test/tbaseCodecTest.cpp

diff --git a/include/util/tbase58.h b/include/util/tbase58.h
index e1b03f8a8f..ab62e1926e 100644
--- a/include/util/tbase58.h
+++ b/include/util/tbase58.h
@@ -22,6 +22,9 @@
 extern "C" {
 #endif
 
+#define TBASE_MAX_ILEN 4096
+#define TBASE_MAX_OLEN 5653
+
 uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen);
 char    *base58_encode(const uint8_t *value, int32_t vlen);
 
diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c
index fa3ecadd14..e1cee72b47 100644
--- a/source/util/src/tbase58.c
+++ b/source/util/src/tbase58.c
@@ -18,24 +18,29 @@
 #include <math.h>
 #include <stdbool.h>
 
-#define BASE_BUF_SIZE 256
+#define TBASE_BUF_SIZE 256
 static const char *basis_58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
 
 char *base58_encode(const uint8_t *value, int32_t vlen) {
   const uint8_t *pb = value;
   const uint8_t *pe = pb + vlen;
-  uint8_t        buf[BASE_BUF_SIZE] = {0};
+  uint8_t        buf[TBASE_BUF_SIZE] = {0};
   uint8_t       *pbuf = &buf[0];
   bool           bfree = false;
   int32_t        nz = 0, size = 0, len = 0;
 
+  if (vlen > TBASE_MAX_ILEN) {
+    terrno = TSDB_CODE_INVALID_PARA;
+    return NULL;
+  }
+
   while (pb != pe && *pb == 0) {
     ++pb;
     ++nz;
   }
 
   size = (pe - pb) * 69 / 50 + 1;
-  if (size > BASE_BUF_SIZE) {
+  if (size > TBASE_BUF_SIZE) {
     if (!(pbuf = taosMemoryCalloc(1, size))) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;
@@ -47,7 +52,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) {
     int32_t num = *pb;
     int32_t i = 0;
     for (int32_t j = (int32_t)size - 1; (num != 0 || i < len) && j >= 0; --j, ++i) {
-      num += ((int32_t)buf[j]) << 8;
+      num += ((int32_t)pbuf[j]) << 8;
       pbuf[j] = num % 58;
       num /= 58;
     }
@@ -57,7 +62,7 @@ char *base58_encode(const uint8_t *value, int32_t vlen) {
 
   const uint8_t *pi = pbuf + (size - len);
   while (pi != pbuf + size && *pi == 0) ++pi;
-  uint8_t *result = taosMemoryCalloc(1, size + 1);
+  uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - pi) + 1);
   if (!result) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     if (bfree) taosMemoryFree(pbuf);
@@ -83,11 +88,23 @@ static const signed char index_58[256] = {
 
 uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
   const char *pe = value + inlen;
-  uint8_t     buf[BASE_BUF_SIZE] = {0};
+  uint8_t     buf[TBASE_BUF_SIZE] = {0};
   uint8_t    *pbuf = &buf[0];
   bool        bfree = false;
   int32_t     nz = 0, size = 0, len = 0;
 
+  if (inlen > TBASE_MAX_OLEN) {
+    terrno = TSDB_CODE_INVALID_PARA;
+    return NULL;
+  }
+
+  for (int32_t i = 0; i < inlen; ++i) {
+    if (value[i] == 0) {
+      terrno = TSDB_CODE_INVALID_PARA;
+      return NULL;
+    }
+  }
+
   while (*value && isspace(*value)) ++value;
   while (*value == '1') {
     ++nz;
@@ -95,7 +112,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
   }
 
   size = (int32_t)(pe - value) * 733 / 1000 + 1;
-  if (size > BASE_BUF_SIZE) {
+  if (size > TBASE_BUF_SIZE) {
     if (!(pbuf = taosMemoryCalloc(1, size))) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return NULL;
@@ -106,6 +123,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
   while (*value && !isspace(*value)) {
     int32_t num = index_58[(uint8_t)*value];
     if (num == -1) {
+      terrno = TSDB_CODE_INVALID_PARA;
       if (bfree) taosMemoryFree(pbuf);
       return NULL;
     }
@@ -127,7 +145,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
   const uint8_t *it = pbuf + (size - len);
   while (it != pbuf + size && *it == 0) ++it;
 
-  uint8_t *result = taosMemoryCalloc(1, size + 1);
+  uint8_t *result = taosMemoryCalloc(1, inlen + 1);
   if (!result) {
     if (bfree) taosMemoryFree(pbuf);
     terrno = TSDB_CODE_OUT_OF_MEMORY;
diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt
index f4f3880388..5e32bd82b7 100644
--- a/source/util/test/CMakeLists.txt
+++ b/source/util/test/CMakeLists.txt
@@ -100,3 +100,11 @@ add_test(
     NAME talgoTest
     COMMAND talgoTest
 )
+
+# tbaseCodecTest
+add_executable(tbaseCodecTest "tbaseCodecTest.cpp")
+target_link_libraries(tbaseCodecTest os util common gtest_main)
+add_test(
+    NAME tbaseCodecTest
+    COMMAND tbaseCodecTest
+
diff --git a/source/util/test/tbaseCodecTest.cpp b/source/util/test/tbaseCodecTest.cpp
new file mode 100644
index 0000000000..70639e7ec9
--- /dev/null
+++ b/source/util/test/tbaseCodecTest.cpp
@@ -0,0 +1,80 @@
+#include <gtest/gtest.h>
+#include <cassert>
+
+#include <iostream>
+#include "os.h"
+#include "taos.h"
+#include "taoserror.h"
+#include "tbase58.h"
+#include "tbase64.h"
+#include "tglobal.h"
+
+using namespace std;
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
+
+static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) {
+  char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen);
+  ASSERT_NE(nullptr, pEnc);
+
+  int32_t encLen = strlen(pEnc);
+  std::cout << "index:" << index << ", encLen is " << encLen << std::endl;
+  int32_t decLen = 0;
+  char   *pDec = (char *)base58_decode((const char *)pEnc, encLen, &decLen);
+  std::cout << "index:" << index << ", decLen is " << decLen << std::endl;
+  ASSERT_NE(nullptr, pDec);
+  ASSERT_EQ(rawLen, decLen);
+  ASSERT_LE(rawLen, encLen);
+  ASSERT_EQ(0, strncmp((char *)pRaw, pDec, rawLen));
+  taosMemoryFreeClear(pDec);
+  taosMemoryFreeClear(pEnc);
+}
+
+TEST(TD_BASE_CODEC_TEST, tbase58_test) {
+  const int32_t TEST_LEN_MAX = TBASE_MAX_ILEN;
+  const int32_t TEST_LEN_STEP = 10;
+  int32_t       rawLen = 0;
+  uint8_t      *pRaw = NULL;
+
+  pRaw = (uint8_t *)taosMemoryCalloc(1, TEST_LEN_MAX);
+  ASSERT_NE(nullptr, pRaw);
+
+  // 1. normal case
+  // string blend with char and '\0'
+  rawLen = TEST_LEN_MAX;
+  for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) {
+    checkBase58Codec(pRaw, rawLen, i);
+    pRaw[i] = i & 127;
+  }
+
+  // string without '\0'
+  for (int32_t i = 0; i < TEST_LEN_MAX; ++i) {
+    pRaw[i] = i & 127;
+  }
+  checkBase58Codec(pRaw, TEST_LEN_MAX, 0);
+  for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) {
+    rawLen = i;
+    checkBase58Codec(pRaw, rawLen, i);
+  }
+  taosMemoryFreeClear(pRaw);
+  ASSERT_EQ(nullptr, pRaw);
+
+  // 2. overflow case
+  char  tmp[1];
+  char *pEnc = base58_encode((const uint8_t *)tmp, TBASE_MAX_ILEN + 1);
+  ASSERT_EQ(nullptr, pEnc);
+  char *pDec = (char *)base58_decode((const char *)tmp, TBASE_MAX_OLEN + 1, NULL);
+  ASSERT_EQ(nullptr, pDec);
+
+  taosMemoryFreeClear(pRaw);
+  ASSERT_EQ(nullptr, pRaw);
+}
\ No newline at end of file

From 604ae220f306d5a794d4082560aa3e2367f1aaf8 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 22 Feb 2024 17:48:33 +0800
Subject: [PATCH 19/37] fix:change datablock to old version for compatibility

---
 include/common/tdatablock.h                 |  2 +-
 source/client/src/clientMsgHandler.c        |  4 +-
 source/client/src/clientTmq.c               | 67 ++++++++++++++-------
 source/common/src/tdatablock.c              | 12 ++--
 source/dnode/mgmt/mgmt_dnode/src/dmHandle.c |  2 +-
 source/dnode/mnode/impl/src/mndShow.c       |  2 +-
 source/dnode/mnode/impl/src/mndStb.c        |  2 +-
 source/dnode/vnode/src/tq/tqScan.c          |  2 +-
 source/libs/command/src/command.c           |  2 +-
 source/libs/command/src/explain.c           |  2 +-
 source/libs/executor/src/dataDispatcher.c   |  2 +-
 source/libs/stream/src/streamDispatch.c     |  4 +-
 12 files changed, 62 insertions(+), 41 deletions(-)

diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index e305f4c2ec..292e9a3181 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -256,7 +256,7 @@ SColumnInfoData  createColumnInfoData(int16_t type, int32_t bytes, int16_t colId
 SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index);
 
 int32_t blockGetEncodeSize(const SSDataBlock* pBlock);
-int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion);
+int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols);
 const char* blockDecode(SSDataBlock* pBlock, const char* pData);
 
 // for debug
diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c
index 3aa46d135e..e0cedb9924 100644
--- a/source/client/src/clientMsgHandler.c
+++ b/source/client/src/clientMsgHandler.c
@@ -499,7 +499,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
   (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
   (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS);
 
-  int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS, BLOCK_VERSION_1);
+  int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS);
   blockDataDestroy(pBlock);
 
   if (len != rspSize - sizeof(SRetrieveTableRsp)) {
@@ -611,7 +611,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr
   (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
   (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS);
 
-  int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS, BLOCK_VERSION_1);
+  int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS);
   blockDataDestroy(pBlock);
 
   if (len != rspSize - sizeof(SRetrieveTableRsp)) {
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 69681b9ae0..ebd4e1c7b6 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1562,17 +1562,39 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
   return pRspObj;
 }
 
-SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
-  SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
-  pRspObj->resType = RES_TYPE__TMQ;
 
+void changeByteEndian(char* pData){
+  char* p = pData;
+
+  // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length |
+  // version:
+  int32_t blockVersion = *(int32_t*)p;
+  ASSERT(blockVersion == BLOCK_VERSION_1);
+  *(int32_t*)p = BLOCK_VERSION_2;
+
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  int32_t cols = *(int32_t*)p;
+  p += sizeof(int32_t);
+  p += sizeof(int32_t);
+  p += sizeof(uint64_t);
+  // check fields
+  p += cols * (sizeof(int8_t) + sizeof(int32_t));
+
+  int32_t* colLength = (int32_t*)p;
+  for (int32_t i = 0; i < cols; ++i) {
+    colLength[i] = htonl(colLength[i]);
+  }
+}
+
+static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) {
   (*numOfRows) = 0;
   tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
   tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
 
   pRspObj->vgId = pWrapper->vgHandle->vgId;
   pRspObj->resIter = -1;
-  memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
 
   pRspObj->resInfo.totalRows = 0;
   pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
@@ -1584,11 +1606,21 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg,
   }
     // extract the rows in this data packet
   for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
-    SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i);
-    int64_t            rows = htobe64(pRetrieve->numOfRows);
+    void* pRetrieve = taosArrayGetP(pRspObj->rsp.blockData, i);
+    void* rawData = NULL;
+    int64_t rows = 0;
+    // deal with compatibility
+    if(*(int64_t*)pRetrieve == 0){
+      rawData = ((SRetrieveTableRsp*)pRetrieve)->data;
+      rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows);
+    }else if(*(int64_t*)pRetrieve == 1){
+      rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data;
+      rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows);
+    }
+
     pVg->numOfRows += rows;
     (*numOfRows) += rows;
-
+    changeByteEndian(rawData);
     if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable
       SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema);
       if(schema){
@@ -1596,29 +1628,22 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg,
       }
     }
   }
+}
 
+SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
+  SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
+  pRspObj->resType = RES_TYPE__TMQ;
+  memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp));
+  tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, pRspObj);
   return pRspObj;
 }
 
 SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) {
   SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj));
   pRspObj->resType = RES_TYPE__TMQ_METADATA;
-  tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
-  tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
-  pRspObj->vgId = pWrapper->vgHandle->vgId;
-  pRspObj->resIter = -1;
   memcpy(&pRspObj->rsp, &pWrapper->taosxRsp, sizeof(STaosxRsp));
 
-  pRspObj->resInfo.totalRows = 0;
-  pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
-
-  // extract the rows in this data packet
-  for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) {
-    SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i);
-    int64_t            rows = htobe64(pRetrieve->numOfRows);
-    pVg->numOfRows += rows;
-    (*numOfRows) += rows;
-  }
+  tmqBuildRspFromWrapperInner(pWrapper, pVg, numOfRows, (SMqRspObj*)pRspObj);
   return pRspObj;
 }
 
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index c6f004f83b..49f1f1eb0e 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -2191,12 +2191,12 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, int32_t bVersion) {
+int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
   int32_t dataLen = 0;
 
   // todo extract method
   int32_t* version = (int32_t*)data;
-  *version = bVersion;
+  *version = BLOCK_VERSION_1;
   data += sizeof(int32_t);
 
   int32_t* actualLen = (int32_t*)data;
@@ -2277,9 +2277,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols, in
       data += colSizes[col];
     }
 
-    if(bVersion == BLOCK_VERSION_1){
-      colSizes[col] = htonl(colSizes[col]);
-    }
+    colSizes[col] = htonl(colSizes[col]);
     //    uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type,
     //    htonl(colSizes[col]), colSizes[col]);
   }
@@ -2340,9 +2338,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) {
   pStart += sizeof(int32_t) * numOfCols;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
-    if(version == BLOCK_VERSION_1){
-      colLen[i] = htonl(colLen[i]);
-    }
+    colLen[i] = htonl(colLen[i]);
     ASSERT(colLen[i] >= 0);
 
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
index b36dc71322..14853009e0 100644
--- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
+++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
@@ -401,7 +401,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     pStart += sizeof(SSysTableSchema);
   }
 
-  int32_t len = blockEncode(pBlock, pStart, numOfCols, BLOCK_VERSION_1);
+  int32_t len = blockEncode(pBlock, pStart, numOfCols);
 
   pRsp->numOfRows = htonl(pBlock->info.rows);
   pRsp->precision = TSDB_TIME_PRECISION_MILLI;  // millisecond time precision
diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c
index e46c057129..8e7e72aa0e 100644
--- a/source/dnode/mnode/impl/src/mndShow.c
+++ b/source/dnode/mnode/impl/src/mndShow.c
@@ -328,7 +328,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
       pStart += sizeof(SSysTableSchema);
     }
 
-    int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1);
+    int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns);
   }
 
   pRsp->numOfRows = htonl(rowsRead);
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index ed1192c490..1ddd2f34e6 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -3150,7 +3150,7 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t
 //      pStart += sizeof(SSysTableSchema);
 //    }
 //
-//    int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns, BLOCK_VERSION_1);
+//    int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns);
 //  }
 //
 //  pRsp->numOfRows = htonl(rowsRead);
diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c
index 4e4e1481f9..5432637482 100644
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@@ -28,7 +28,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t
   pRetrieve->compressed = 0;
   pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows);
 
-  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_2);
+  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols);
   actualLen += sizeof(SRetrieveTableRspForTmq);
   taosArrayPush(pRsp->blockDataLen, &actualLen);
   taosArrayPush(pRsp->blockData, &buf);
diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c
index e74c735497..6bae0e1022 100644
--- a/source/libs/command/src/command.c
+++ b/source/libs/command/src/command.c
@@ -40,7 +40,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe
   (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
   (*pRsp)->numOfCols = htonl(numOfCols);
 
-  int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols, BLOCK_VERSION_1);
+  int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols);
 
   return TSDB_CODE_SUCCESS;
 }
diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c
index d2d14bb9f3..66b50bcb47 100644
--- a/source/libs/command/src/explain.c
+++ b/source/libs/command/src/explain.c
@@ -1817,7 +1817,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
   rsp->completed = 1;
   rsp->numOfRows = htobe64((int64_t)rowNum);
 
-  int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock), BLOCK_VERSION_1);
+  int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock));
 
   rsp->compLen = htonl(len);
 
diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c
index fb57d6b531..abe566473f 100644
--- a/source/libs/executor/src/dataDispatcher.c
+++ b/source/libs/executor/src/dataDispatcher.c
@@ -76,7 +76,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
   pEntry->dataLen = 0;
 
   pBuf->useSize = sizeof(SDataCacheEntry);
-  pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols, BLOCK_VERSION_1);
+  pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols);
   //  ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8));
   //  ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4));
 
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 080c2c9556..b51845d152 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -188,7 +188,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
   pRetrieve->ekey = htobe64(pBlock->info.window.ekey);
   pRetrieve->version = htobe64(pBlock->info.version);
 
-  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1);
+  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols);
 
   SStreamRetrieveReq req = {
       .streamId = pTask->id.streamId,
@@ -772,7 +772,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch
   int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
   pRetrieve->numOfCols = htonl(numOfCols);
 
-  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols, BLOCK_VERSION_1);
+  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols);
   actualLen += sizeof(SRetrieveTableRsp);
   ASSERT(actualLen <= dataStrLen);
   taosArrayPush(pReq->dataLen, &actualLen);

From 5b0fc1544d08e74487deca8f5300bb4e99ba23f7 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Thu, 22 Feb 2024 18:00:12 +0800
Subject: [PATCH 20/37] enh: unit test and optimization for base58

---
 source/util/test/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt
index 5e32bd82b7..3514c578e9 100644
--- a/source/util/test/CMakeLists.txt
+++ b/source/util/test/CMakeLists.txt
@@ -107,4 +107,4 @@ target_link_libraries(tbaseCodecTest os util common gtest_main)
 add_test(
     NAME tbaseCodecTest
     COMMAND tbaseCodecTest
-
+)

From 75caea4fbf6b47a17b87528960f69ce5d79e40d9 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Thu, 22 Feb 2024 19:04:21 +0800
Subject: [PATCH 21/37] enh: code optimization for tbase58

---
 source/util/src/tbase58.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c
index e1cee72b47..2566d54431 100644
--- a/source/util/src/tbase58.c
+++ b/source/util/src/tbase58.c
@@ -145,7 +145,7 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
   const uint8_t *it = pbuf + (size - len);
   while (it != pbuf + size && *it == 0) ++it;
 
-  uint8_t *result = taosMemoryCalloc(1, inlen + 1);
+  uint8_t *result = taosMemoryCalloc(1, nz + (pbuf + size - it) + 1);
   if (!result) {
     if (bfree) taosMemoryFree(pbuf);
     terrno = TSDB_CODE_OUT_OF_MEMORY;

From 7c9ae199899df75abfa10b9233cd1c9fc56776a5 Mon Sep 17 00:00:00 2001
From: dmchen <cademfly@hotmail.com>
Date: Thu, 22 Feb 2024 12:31:33 +0000
Subject: [PATCH 22/37] core at show user

---
 source/dnode/mnode/impl/src/mndUser.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c
index 5e5a3626a4..8a2313370a 100644
--- a/source/dnode/mnode/impl/src/mndUser.c
+++ b/source/dnode/mnode/impl/src/mndUser.c
@@ -2484,7 +2484,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)privilege, false);
 
-      char objName[20] = {0};
+      char objName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
       STR_WITH_MAXSIZE_TO_VARSTR(objName, "all", pShow->pMeta->pSchemas[cols].bytes);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)objName, false);

From c7a50acd632af1789f2a195ce4652c268d359a32 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Fri, 23 Feb 2024 09:05:30 +0800
Subject: [PATCH 23/37] rebuild sliding window

---
 source/libs/executor/src/streamcountwindowoperator.c | 3 +++
 source/libs/stream/src/streamSessionState.c          | 1 +
 2 files changed, 4 insertions(+)

diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c
index 689ba54ca6..294c2730df 100644
--- a/source/libs/executor/src/streamcountwindowoperator.c
+++ b/source/libs/executor/src/streamcountwindowoperator.c
@@ -99,6 +99,9 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
                                                    (void**)&pCurWin->winInfo.pStatePos, &size);
       }
     }
+    if (ts < pCurWin->winInfo.sessionWin.win.ekey) {
+      pBuffInfo->rebuildWindow = true;
+    }
   } else {
     code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(
         pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size);
diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c
index e2aae130e5..3d0241df75 100644
--- a/source/libs/stream/src/streamSessionState.c
+++ b/source/libs/stream/src/streamSessionState.c
@@ -488,6 +488,7 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS
   void* pFileStore = getStateFileStore(pFileState);
   SStreamStateCur* pCur = streamStateSessionSeekKeyPrev_rocksdb(pFileStore, pWinKey);
   if (pCur) {
+    pCur->pStreamFileState = pFileState;
     SSessionKey key = {0};
     void* pVal = NULL;
     int len = 0;

From e80ac5a4a69e42fa6ccc48cbee67a6513c5945cb Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 23 Feb 2024 10:14:59 +0800
Subject: [PATCH 24/37] fix(vnd/query): not release reader if not initialized

---
 source/dnode/vnode/src/vnd/vnodeQuery.c | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c
index 4fc7a88494..d1c811858a 100644
--- a/source/dnode/vnode/src/vnd/vnodeQuery.c
+++ b/source/dnode/vnode/src/vnd/vnodeQuery.c
@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnd.h"
 #include "tsdb.h"
+#include "vnd.h"
 
 #define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags)                                                    \
   do {                                                                                                        \
@@ -49,7 +49,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
   // decode req
   if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) {
     code = TSDB_CODE_INVALID_MSG;
-    goto _exit;
+    goto _exit4;
   }
 
   metaRsp.dbId = pVnode->config.dbId;
@@ -59,7 +59,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
   sprintf(tableFName, "%s.%s", infoReq.dbFName, infoReq.tbName);
   code = vnodeValidateTableHash(pVnode, tableFName);
   if (code) {
-    goto _exit;
+    goto _exit4;
   }
 
   // query meta
@@ -67,7 +67,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
 
   if (metaGetTableEntryByName(&mer1, infoReq.tbName) < 0) {
     code = terrno;
-    goto _exit;
+    goto _exit3;
   }
 
   metaRsp.tableType = mer1.me.type;
@@ -81,7 +81,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
     metaRsp.suid = mer1.me.uid;
   } else if (mer1.me.type == TSDB_CHILD_TABLE) {
     metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK);
-    if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
+    if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit2;
 
     strcpy(metaRsp.stbName, mer2.me.name);
     metaRsp.suid = mer2.me.uid;
@@ -125,6 +125,12 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
   tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp);
 
 _exit:
+  taosMemoryFree(metaRsp.pSchemas);
+_exit2:
+  metaReaderClear(&mer2);
+_exit3:
+  metaReaderClear(&mer1);
+_exit4:
   rpcMsg.info = pMsg->info;
   rpcMsg.pCont = pRsp;
   rpcMsg.contLen = rspLen;
@@ -141,9 +147,6 @@ _exit:
     *pMsg = rpcMsg;
   }
 
-  taosMemoryFree(metaRsp.pSchemas);
-  metaReaderClear(&mer2);
-  metaReaderClear(&mer1);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -706,5 +709,5 @@ void *vnodeGetIvtIdx(void *pVnode) {
 }
 
 int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid) {
-  return tsdbGetTableSchema(((SVnode*)pVnode)->pMeta, uid, pSchema, suid);
+  return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid);
 }

From 31cf877f50c6b2221daa48b3e252323b81588938 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Fri, 23 Feb 2024 10:16:27 +0800
Subject: [PATCH 25/37] enh: code optimization for tbase58

---
 source/util/src/tbase58.c | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/source/util/src/tbase58.c b/source/util/src/tbase58.c
index 2566d54431..5eb72879c3 100644
--- a/source/util/src/tbase58.c
+++ b/source/util/src/tbase58.c
@@ -87,6 +87,7 @@ static const signed char index_58[256] = {
     -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
 
 uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
+  const char *pb = value;
   const char *pe = value + inlen;
   uint8_t     buf[TBASE_BUF_SIZE] = {0};
   uint8_t    *pbuf = &buf[0];
@@ -98,20 +99,22 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
     return NULL;
   }
 
-  for (int32_t i = 0; i < inlen; ++i) {
-    if (value[i] == 0) {
+  while (pb != pe) {
+    if (*pb == 0) {
       terrno = TSDB_CODE_INVALID_PARA;
       return NULL;
     }
+    ++pb;
   }
 
-  while (*value && isspace(*value)) ++value;
-  while (*value == '1') {
+  pb = value;
+  while (pb != pe && *pb && isspace(*pb)) ++pb;
+  while (pb != pe && *pb == '1') {
     ++nz;
-    ++value;
+    ++pb;
   }
 
-  size = (int32_t)(pe - value) * 733 / 1000 + 1;
+  size = (int32_t)(pe - pb) * 733 / 1000 + 1;
   if (size > TBASE_BUF_SIZE) {
     if (!(pbuf = taosMemoryCalloc(1, size))) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -120,8 +123,8 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
     bfree = true;
   }
 
-  while (*value && !isspace(*value)) {
-    int32_t num = index_58[(uint8_t)*value];
+  while (pb != pe && *pb && !isspace(*pb)) {
+    int32_t num = index_58[(uint8_t)*pb];
     if (num == -1) {
       terrno = TSDB_CODE_INVALID_PARA;
       if (bfree) taosMemoryFree(pbuf);
@@ -134,11 +137,11 @@ uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen) {
       num >>= 8;
     }
     len = i;
-    ++value;
+    ++pb;
   }
 
-  while (isspace(*value)) ++value;
-  if (*value != 0) {
+  while (pb != pe && isspace(*pb)) ++pb;
+  if (*pb != 0) {
     if (bfree) taosMemoryFree(pbuf);
     return NULL;
   }

From 03d68a037693120001a69d6653d38c8da5e1464f Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Wed, 21 Feb 2024 13:37:15 +0800
Subject: [PATCH 26/37] enh: change redo logs of trans create-stb as prepare
 logs

---
 source/dnode/mnode/impl/src/mndStb.c | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index aad1dc881b..a1aed7b735 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -617,10 +617,10 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
   return 0;
 }
 
-static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
+static int32_t mndSetCreateStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -629,18 +629,6 @@ static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
   return 0;
 }
 
-static int32_t mndSetCreateStbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
-  SSdbRaw *pUndoRaw = mndStbActionEncode(pStb);
-  if (pUndoRaw == NULL) return -1;
-  if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
-    sdbFreeRaw(pUndoRaw);
-    return -1;
-  }
-  if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
-
-  return 0;
-}
-
 static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   SSdbRaw *pCommitRaw = mndStbActionEncode(pStb);
   if (pCommitRaw == NULL) return -1;
@@ -913,8 +901,7 @@ _OVER:
 int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   mndTransSetDbName(pTrans, pDb->name, pStb->name);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) return -1;
-  if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
-  if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
+  if (mndSetCreateStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;

From 3a81c9777ca3297a6e7ca682a84707154add14b3 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Wed, 21 Feb 2024 20:00:04 +0800
Subject: [PATCH 27/37] enh: change redo logs of trans alter-stb and drop-stb
 as prepare logs

---
 source/dnode/mnode/impl/src/mndStb.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index a1aed7b735..b6e49b4d8b 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -1733,10 +1733,10 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO
   return 0;
 }
 
-static int32_t mndSetAlterStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
+static int32_t mndSetAlterStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -2142,7 +2142,7 @@ static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbOb
     mndTransSetRpcRsp(pTrans, pCont, contLen);
   }
 
-  if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@@ -2179,11 +2179,11 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO
     if (mndGetIdxsByTagName(pMnode, pStb, pField0->name, &idxObj) == 0) {
       exist = true;
     }
-    if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+    if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
     if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
 
     if (exist == true) {
-      if (mndSetDropIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
+      if (mndSetDropIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
       if (mndSetDropIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
     }
 
@@ -2202,13 +2202,13 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbO
       exist = true;
     }
 
-    if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+    if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
     if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
 
     if (exist == true) {
       memcpy(idxObj.colName, nTagName, strlen(nTagName));
       idxObj.colName[strlen(nTagName)] = 0;
-      if (mndSetAlterIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
+      if (mndSetAlterIdxPrepareLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
       if (mndSetAlterIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER;
     }
 
@@ -2341,10 +2341,10 @@ _OVER:
   return code;
 }
 
-static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
+static int32_t mndSetDropStbPrepareLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
   SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -2414,7 +2414,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p
   mndTransSetDbName(pTrans, pDb->name, pStb->name);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
-  if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
+  if (mndSetDropStbPrepareLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
   if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
   if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndDropIdxsByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
@@ -3534,7 +3534,7 @@ static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) {
   mndTransSetDbName(pTrans, pDb->name, pStb->name);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
-  if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
+  if (mndSetAlterStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER;
   if (mndSetAlterStbRedoActions2(pMnode, pTrans, pDb, pStb, sql, len) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;

From 8f3b6a717df0eb8a4ae4b464ff2c00ac3f9ebbfe Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Wed, 21 Feb 2024 20:03:21 +0800
Subject: [PATCH 28/37] enh: change redo logs of trans alter-db and drop-db as
 prepare logs

---
 source/dnode/mnode/impl/src/mndDb.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 077c0a9c2a..2ea1cdfe2a 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -884,10 +884,10 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
   return terrno;
 }
 
-static int32_t mndSetAlterDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
+static int32_t mndSetAlterDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
   SSdbRaw *pRedoRaw = mndDbActionEncode(pOld);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -943,7 +943,7 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p
   mndTransSetDbName(pTrans, pOld->name, NULL);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
-  if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
+  if (mndSetAlterDbPrepareLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
@@ -1120,10 +1120,10 @@ _OVER:
   return code;
 }
 
-static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
+static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
   SSdbRaw *pRedoRaw = mndDbActionEncode(pDb);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
   if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
 
   return 0;
@@ -1257,7 +1257,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
     goto _OVER;
   }
 
-  if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
+  if (mndSetDropDbPrepareLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
   if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER;
   /*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/
   /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/

From 94f7e0c811af2a7ce4ab4e8ed9ebc853230b7341 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Wed, 21 Feb 2024 20:07:45 +0800
Subject: [PATCH 29/37] enh: change redo logs of trans create-stb-index and
 drop-index as prepare logs

---
 source/dnode/mnode/impl/inc/mndIndex.h |  8 ++++----
 source/dnode/mnode/impl/src/mndIndex.c | 16 ++++++++--------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/source/dnode/mnode/impl/inc/mndIndex.h b/source/dnode/mnode/impl/inc/mndIndex.h
index 2d5479bc9b..3b9b68a1b5 100644
--- a/source/dnode/mnode/impl/inc/mndIndex.h
+++ b/source/dnode/mnode/impl/inc/mndIndex.h
@@ -19,16 +19,16 @@ int32_t  mndGetTableIdx(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool
 int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq);
 
-int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
-int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 
-int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
+int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_MND_IDX_H_*/
\ No newline at end of file
+#endif /*_TD_MND_IDX_H_*/
diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c
index 041cc664e5..622ed0080f 100644
--- a/source/dnode/mnode/impl/src/mndIndex.c
+++ b/source/dnode/mnode/impl/src/mndIndex.c
@@ -331,10 +331,10 @@ SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) {
   return mndAcquireDb(pMnode, db);
 }
 
-int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetCreateIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
   SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
   if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
 
   return 0;
@@ -349,10 +349,10 @@ int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx)
   return 0;
 }
 
-int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetAlterIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
   SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) {
     sdbFreeRaw(pRedoRaw);
     return -1;
   }
@@ -482,10 +482,10 @@ _OVER:
   return code;
 }
 
-int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
+int32_t mndSetDropIdxPrepareLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) {
   SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx);
   if (pRedoRaw == NULL) return -1;
-  if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
+  if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1;
   if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
 
   return 0;
@@ -652,7 +652,7 @@ int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pSt
 
   mndTransSetSerial(pTrans);
 
-  if (mndSetCreateIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
+  if (mndSetCreateIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
   if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
 
   if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER;
@@ -771,7 +771,7 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p
   if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
 
   mndTransSetSerial(pTrans);
-  if (mndSetDropIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
+  if (mndSetDropIdxPrepareLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
   if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER;
 
   if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER;

From fbd2a0035c2a6122dc30f8332905b2ec0f9d8b3b Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Wed, 21 Feb 2024 20:09:01 +0800
Subject: [PATCH 30/37] enh: change type of trans create-snode and drop-snode
 as serial

---
 source/dnode/mnode/impl/src/mndSnode.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c
index 1275ba7962..4243ccb77c 100644
--- a/source/dnode/mnode/impl/src/mndSnode.c
+++ b/source/dnode/mnode/impl/src/mndSnode.c
@@ -257,6 +257,7 @@ static int32_t mndCreateSnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-snode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId);
 
@@ -383,6 +384,7 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) {
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-snode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to drop snode:%d", pTrans->id, pObj->id);
   if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;

From a371afa58e054d02fe62975ea7ff3984a33ee66f Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Wed, 21 Feb 2024 20:12:29 +0800
Subject: [PATCH 31/37] enh: change type of trans create-qnode and drop-qnode
 as serial

---
 source/dnode/mnode/impl/src/mndQnode.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c
index a7d8c81d3f..2f4ad69e2a 100644
--- a/source/dnode/mnode/impl/src/mndQnode.c
+++ b/source/dnode/mnode/impl/src/mndQnode.c
@@ -257,6 +257,7 @@ static int32_t mndCreateQnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-qnode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId);
   if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) goto _OVER;
@@ -380,6 +381,7 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) {
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-qnode");
   if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
 
   mInfo("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id);
   if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj, false) != 0) goto _OVER;

From ef7b38d4f69257df739a6e602deca68482e4bd81 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Fri, 23 Feb 2024 10:30:47 +0800
Subject: [PATCH 32/37] enh: code optimization for tbase58

---
 source/util/test/tbaseCodecTest.cpp | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/source/util/test/tbaseCodecTest.cpp b/source/util/test/tbaseCodecTest.cpp
index 70639e7ec9..4c56979885 100644
--- a/source/util/test/tbaseCodecTest.cpp
+++ b/source/util/test/tbaseCodecTest.cpp
@@ -3,10 +3,10 @@
 
 #include <iostream>
 #include "os.h"
+#include "osTime.h"
 #include "taos.h"
 #include "taoserror.h"
 #include "tbase58.h"
-#include "tbase64.h"
 #include "tglobal.h"
 
 using namespace std;
@@ -23,14 +23,17 @@ int main(int argc, char **argv) {
 }
 
 static void checkBase58Codec(uint8_t *pRaw, int32_t rawLen, int32_t index) {
-  char *pEnc = base58_encode((const uint8_t *)pRaw, rawLen);
+  int64_t start = taosGetTimestampUs();
+  char   *pEnc = base58_encode((const uint8_t *)pRaw, rawLen);
   ASSERT_NE(nullptr, pEnc);
 
   int32_t encLen = strlen(pEnc);
-  std::cout << "index:" << index << ", encLen is " << encLen << std::endl;
+  int64_t endOfEnc = taosGetTimestampUs();
+  std::cout << "index:" << index << ", encLen is " << encLen << ", cost:" << endOfEnc - start << " us" << std::endl;
   int32_t decLen = 0;
   char   *pDec = (char *)base58_decode((const char *)pEnc, encLen, &decLen);
-  std::cout << "index:" << index << ", decLen is " << decLen << std::endl;
+  std::cout << "index:" << index << ", decLen is " << decLen << ", cost:" << taosGetTimestampUs() - endOfEnc << " us"
+            << std::endl;
   ASSERT_NE(nullptr, pDec);
   ASSERT_EQ(rawLen, decLen);
   ASSERT_LE(rawLen, encLen);
@@ -51,7 +54,7 @@ TEST(TD_BASE_CODEC_TEST, tbase58_test) {
   // 1. normal case
   // string blend with char and '\0'
   rawLen = TEST_LEN_MAX;
-  for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) {
+  for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) {
     checkBase58Codec(pRaw, rawLen, i);
     pRaw[i] = i & 127;
   }
@@ -61,7 +64,7 @@ TEST(TD_BASE_CODEC_TEST, tbase58_test) {
     pRaw[i] = i & 127;
   }
   checkBase58Codec(pRaw, TEST_LEN_MAX, 0);
-  for (int32_t i = 0; i < TEST_LEN_MAX; i += 1000) {
+  for (int32_t i = 0; i < TEST_LEN_MAX; i += 500) {
     rawLen = i;
     checkBase58Codec(pRaw, rawLen, i);
   }

From dfef26866a4ac267457d5207b9dc16ffc6198430 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Thu, 22 Feb 2024 10:34:00 +0800
Subject: [PATCH 33/37] enh: decode sdb raw object in SdbValidateFp instead

---
 source/dnode/mnode/impl/src/mndDb.c     | 20 +++++++++++++-----
 source/dnode/mnode/impl/src/mndSync.c   | 17 ++--------------
 source/dnode/mnode/impl/src/mndVgroup.c | 27 +++++++++++++++++--------
 source/dnode/mnode/sdb/inc/sdb.h        |  2 +-
 4 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 2ea1cdfe2a..9f644fb290 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -41,7 +41,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
 static int32_t  mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
 static int32_t  mndDbActionDelete(SSdb *pSdb, SDbObj *pDb);
 static int32_t  mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew);
-static int32_t  mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj);
+static int32_t  mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
 
 static int32_t  mndProcessCreateDbReq(SRpcMsg *pReq);
 static int32_t  mndProcessAlterDbReq(SRpcMsg *pReq);
@@ -256,17 +256,27 @@ _OVER:
   return pRow;
 }
 
-static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) {
-  SDbObj *pNewDb = pObj;
+static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) {
+  SSdb    *pSdb = pMnode->pSdb;
+  SSdbRow *pRow = NULL;
+  int      code = -1;
+
+  pRow = mndDbActionDecode(pRaw);
+  if (pRow == NULL) goto _OVER;
+  SDbObj *pNewDb = sdbGetRowObj(pRow);
+  if (pNewDb == NULL) goto _OVER;
 
   SDbObj *pOldDb = sdbAcquire(pMnode->pSdb, SDB_DB, pNewDb->name);
   if (pOldDb != NULL) {
     mError("trans:%d, db name already in use. name: %s", pTrans->id, pNewDb->name);
     sdbRelease(pMnode->pSdb, pOldDb);
-    return -1;
+    goto _OVER;
   }
 
-  return 0;
+  code = 0;
+_OVER:
+  taosMemoryFree(pRow);
+  return code;
 }
 
 static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb) {
diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c
index 0fc8dad420..a1490a3813 100644
--- a/source/dnode/mnode/impl/src/mndSync.c
+++ b/source/dnode/mnode/impl/src/mndSync.c
@@ -77,29 +77,16 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
 static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) {
   SSdbRaw *pRaw = pAction->pRaw;
   SSdb    *pSdb = pMnode->pSdb;
-  SSdbRow *pRow = NULL;
-  void    *pObj = NULL;
-  int      code = -1;
+  int      code = 0;
 
   if (pRaw->status != SDB_STATUS_CREATING) goto _OUT;
 
-  pRow = (pSdb->decodeFps[pRaw->type])(pRaw);
-  if (pRow == NULL) goto _OUT;
-  pObj = sdbGetRowObj(pRow);
-  if (pObj == NULL) goto _OUT;
-
   SdbValidateFp validateFp = pSdb->validateFps[pRaw->type];
-  code = 0;
   if (validateFp) {
-    code = validateFp(pMnode, pTrans, pObj);
+    code = validateFp(pMnode, pTrans, pRaw);
   }
 
 _OUT:
-  if (pRow) {
-    SdbDeleteFp deleteFp = pSdb->deleteFps[pRaw->type];
-    if (deleteFp) (*deleteFp)(pSdb, pRow->pObj, false);
-    taosMemoryFreeClear(pRow);
-  }
   return code;
 }
 
diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c
index a5df9ad820..6bd29f7910 100644
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@@ -31,10 +31,10 @@
 #define VGROUP_VER_NUMBER   1
 #define VGROUP_RESERVE_SIZE 64
 
-static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
-static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
-static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
-static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj);
+static int32_t  mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
+static int32_t  mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
+static int32_t  mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
+static int32_t  mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
 
 static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextVgroup(SMnode *pMnode, void *pIter);
@@ -181,15 +181,26 @@ _OVER:
   return pRow;
 }
 
-static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, void *pObj) {
-  SVgObj *pVgroup = pObj;
+static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) {
+  SSdb    *pSdb = pMnode->pSdb;
+  SSdbRow *pRow = NULL;
+  int      code = -1;
+
+  pRow = mndVgroupActionDecode(pRaw);
+  if (pRow == NULL) goto _OVER;
+  SVgObj *pVgroup = sdbGetRowObj(pRow);
+  if (pVgroup == NULL) goto _OVER;
 
   int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
   if (maxVgId > pVgroup->vgId) {
     mError("trans:%d, vgroup id %d already in use. maxVgId:%d", pTrans->id, pVgroup->vgId, maxVgId);
-    return -1;
+    goto _OVER;
   }
-  return 0;
+
+  code = 0;
+_OVER:
+  taosMemoryFree(pRow);
+  return code;
 }
 
 static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup) {
diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h
index 9d1125dad2..e2d4a1f6fc 100644
--- a/source/dnode/mnode/sdb/inc/sdb.h
+++ b/source/dnode/mnode/sdb/inc/sdb.h
@@ -106,7 +106,7 @@ typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj);
 typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj);
 typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc);
 typedef int32_t (*SdbDeployFp)(SMnode *pMnode);
-typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, void *pObj);
+typedef int32_t (*SdbValidateFp)(SMnode *pMnode, void *pTrans, SSdbRaw *pRaw);
 typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw);
 typedef SSdbRaw *(*SdbEncodeFp)(void *pObj);
 typedef bool (*sdbTraverseFp)(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3);

From 1b2bccd1d9866c78ec9c80002e530ef515bfed76 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Thu, 22 Feb 2024 15:49:30 +0800
Subject: [PATCH 34/37] enh: not use prepare log in trans create-stb anymore

---
 source/dnode/mnode/impl/src/mndStb.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index b6e49b4d8b..7ee1b36916 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -901,7 +901,6 @@ _OVER:
 int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
   mndTransSetDbName(pTrans, pDb->name, pStb->name);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) return -1;
-  if (mndSetCreateStbPrepareLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;
   if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, pStb) != 0) return -1;

From ec093cb47732f3fc63e439014d44d68c43960e60 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Fri, 23 Feb 2024 11:25:42 +0800
Subject: [PATCH 35/37] enh: ensure decoded obj cleared in new obj action
 validate

---
 source/dnode/mnode/impl/src/mndDb.c     | 6 ++++--
 source/dnode/mnode/impl/src/mndVgroup.c | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 9f644fb290..37c2d19bd4 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -259,11 +259,12 @@ _OVER:
 static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) {
   SSdb    *pSdb = pMnode->pSdb;
   SSdbRow *pRow = NULL;
+  SDbObj  *pNewDb = NULL;
   int      code = -1;
 
   pRow = mndDbActionDecode(pRaw);
   if (pRow == NULL) goto _OVER;
-  SDbObj *pNewDb = sdbGetRowObj(pRow);
+  pNewDb = sdbGetRowObj(pRow);
   if (pNewDb == NULL) goto _OVER;
 
   SDbObj *pOldDb = sdbAcquire(pMnode->pSdb, SDB_DB, pNewDb->name);
@@ -275,7 +276,8 @@ static int32_t mndNewDbActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *p
 
   code = 0;
 _OVER:
-  taosMemoryFree(pRow);
+  if (pNewDb) mndDbActionDelete(pSdb, pNewDb);
+  taosMemoryFreeClear(pRow);
   return code;
 }
 
diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c
index 6bd29f7910..b0290191bc 100644
--- a/source/dnode/mnode/impl/src/mndVgroup.c
+++ b/source/dnode/mnode/impl/src/mndVgroup.c
@@ -184,11 +184,12 @@ _OVER:
 static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw) {
   SSdb    *pSdb = pMnode->pSdb;
   SSdbRow *pRow = NULL;
+  SVgObj  *pVgroup = NULL;
   int      code = -1;
 
   pRow = mndVgroupActionDecode(pRaw);
   if (pRow == NULL) goto _OVER;
-  SVgObj *pVgroup = sdbGetRowObj(pRow);
+  pVgroup = sdbGetRowObj(pRow);
   if (pVgroup == NULL) goto _OVER;
 
   int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
@@ -199,7 +200,8 @@ static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *p
 
   code = 0;
 _OVER:
-  taosMemoryFree(pRow);
+  if (pVgroup) mndVgroupActionDelete(pSdb, pVgroup);
+  taosMemoryFreeClear(pRow);
   return code;
 }
 

From acdc21ff6c3aaadf0d68cbbe94b2e692126dae09 Mon Sep 17 00:00:00 2001
From: Alex Duan <417921451@qq.com>
Date: Fri, 23 Feb 2024 11:38:04 +0800
Subject: [PATCH 36/37] fix: modify retry to upload to 8 minutes

---
 tests/army/enterprise/s3/s3_basic.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py
index 5d9a52ebc4..45519d925f 100644
--- a/tests/army/enterprise/s3/s3_basic.py
+++ b/tests/army/enterprise/s3/s3_basic.py
@@ -28,12 +28,12 @@ from frame import *
 from frame.eos import *
 
 #  
-# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa
+# 192.168.1.52 MINIO S3 
 #
 
 '''
 s3EndPoint     http://192.168.1.52:9000
-s3AccessKey    MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa
+s3AccessKey    'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX'
 s3BucketName   ci-bucket
 s3UploadDelaySec 60
 '''
@@ -42,7 +42,7 @@ s3UploadDelaySec 60
 class TDTestCase(TBase):
     updatecfgDict = {
         's3EndPoint': 'http://192.168.1.52:9000', 
-        's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', 
+        's3AccessKey': 'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX', 
         's3BucketName': 'ci-bucket',
         's3BlockSize': '10240',
         's3BlockCacheSize': '320',
@@ -78,26 +78,27 @@ class TDTestCase(TBase):
         self.trimDb(True)
 
         rootPath = sc.clusterRootPath()
-        cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data"
+        cmd = f"ls {rootPath}/dnode1/data2*/vnode/vnode*/tsdb/*.data"
         tdLog.info(cmd)
         loop = 0
         rets = []
-        while loop < 500:
+        while loop < 180:
+            time.sleep(3)
             rets = eos.runRetList(cmd)
             cnt = len(rets)
             if cnt == 0:
                 tdLog.info("All data file upload to server over.")
-                break
-            time.sleep(3)
+                break            
             self.trimDb(True)
-            loop += 1
-            tdLog.info(f"loop={loop} data files have {cnt} wait 5s...")
-            if loop == 4:
+            tdLog.info(f"loop={loop} no upload {cnt} data files wait 3s retry ...")
+            if loop == 0:
                 sc.dnodeStop(1)
                 time.sleep(2)
                 sc.dnodeStart(1)
+            loop += 1
+                
         if len(rets) > 0:
-            tdLog.exit(f"s3 can not upload all data to server. data files={rets}")
+            tdLog.exit(f"s3 can not upload all data to server. data files cnt={len(rets)} list={rets}")
 
     def checkStreamCorrect(self):
         sql = f"select count(*) from {self.db}.stm1"

From f2c1b7da05eaca83d30b3ed5f5a8d490be78d097 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Fri, 23 Feb 2024 13:16:59 +0800
Subject: [PATCH 37/37] enh: keep in redo stage on context switch for rollback
 trans

---
 source/dnode/mnode/impl/src/mndTrans.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 1adc4ed4bf..5c76baff08 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -1419,7 +1419,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool
     pTrans->stage = TRN_STAGE_COMMIT;
     mInfo("trans:%d, stage from redoAction to commit", pTrans->id);
     continueExec = true;
-  } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
+  } else if (code == TSDB_CODE_ACTION_IN_PROGRESS || code == TSDB_CODE_MND_TRANS_CTX_SWITCH) {
     mInfo("trans:%d, stage keep on redoAction since %s", pTrans->id, tstrerror(code));
     continueExec = false;
   } else {