From 5a4cc36079728eeacdbed652ccc41600a3a146a7 Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Wed, 11 Jan 2023 20:44:29 +0800
Subject: [PATCH 01/40] enh: skip WAL forceSync for single replica vgroup

---
 include/libs/sync/sync.h                 |  2 +-
 source/libs/sync/src/syncAppendEntries.c |  4 ++--
 source/libs/sync/src/syncMain.c          |  4 ++--
 source/libs/sync/src/syncPipeline.c      | 11 ++++++++---
 source/libs/sync/src/syncRaftLog.c       |  7 ++-----
 5 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h
index 4c23c1f557..defafce30e 100644
--- a/include/libs/sync/sync.h
+++ b/include/libs/sync/sync.h
@@ -193,7 +193,7 @@ typedef struct SSyncLogStore {
   SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore);
   SyncTerm (*syncLogLastTerm)(struct SSyncLogStore* pLogStore);
 
-  int32_t (*syncLogAppendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry);
+  int32_t (*syncLogAppendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, bool forcSync);
   int32_t (*syncLogGetEntry)(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry);
   int32_t (*syncLogTruncate)(struct SSyncLogStore* pLogStore, SyncIndex fromIndex);
 
diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c
index e77a8d4be3..948d2f53e7 100644
--- a/source/libs/sync/src/syncAppendEntries.c
+++ b/source/libs/sync/src/syncAppendEntries.c
@@ -357,7 +357,7 @@ int32_t syncNodeOnAppendEntriesOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
         ASSERT(pAppendEntry->index == appendIndex);
 
         // append
-        code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
+        code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry, false);
         if (code != 0) {
           char logBuf[128];
           snprintf(logBuf, sizeof(logBuf), "ignore, append error, append-index:%" PRId64, appendIndex);
@@ -398,7 +398,7 @@ int32_t syncNodeOnAppendEntriesOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
         }
 
         // append
-        code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
+        code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry, false);
         if (code != 0) {
           char logBuf[128];
           snprintf(logBuf, sizeof(logBuf), "ignore, log not exist, append error, append-index:%" PRId64, appendIndex);
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index a339cb9857..51a7fa8118 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -2478,7 +2478,7 @@ static int32_t syncNodeAppendNoopOld(SSyncNode* ths) {
   LRUHandle* h = NULL;
 
   if (ths->state == TAOS_SYNC_STATE_LEADER) {
-    int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
+    int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry, false);
     if (code != 0) {
       sError("append noop error");
       return -1;
@@ -2721,7 +2721,7 @@ int32_t syncNodeOnClientRequestOld(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRe
 
   if (ths->state == TAOS_SYNC_STATE_LEADER) {
     // append entry
-    code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
+    code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry, false);
     if (code != 0) {
       if (ths->replicaNum == 1) {
         if (h) {
diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c
index b61fc2e90d..1d2b0b57fe 100644
--- a/source/libs/sync/src/syncPipeline.c
+++ b/source/libs/sync/src/syncPipeline.c
@@ -364,7 +364,11 @@ _out:
   return ret;
 }
 
-int32_t syncLogStorePersist(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
+static inline bool syncLogStoreNeedFlush(SSyncRaftEntry* pEntry, int32_t replicaNum) {
+  return (replicaNum > 1) && (pEntry->originalRpcType == TDMT_VND_COMMIT);
+}
+
+int32_t syncLogStorePersist(SSyncLogStore* pLogStore, SSyncNode* pNode, SSyncRaftEntry* pEntry) {
   ASSERT(pEntry->index >= 0);
   SyncIndex lastVer = pLogStore->syncLogLastIndex(pLogStore);
   if (lastVer >= pEntry->index && pLogStore->syncLogTruncate(pLogStore, pEntry->index) < 0) {
@@ -374,7 +378,8 @@ int32_t syncLogStorePersist(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
   lastVer = pLogStore->syncLogLastIndex(pLogStore);
   ASSERT(pEntry->index == lastVer + 1);
 
-  if (pLogStore->syncLogAppendEntry(pLogStore, pEntry) < 0) {
+  bool doFsync = syncLogStoreNeedFlush(pEntry, pNode->replicaNum);
+  if (pLogStore->syncLogAppendEntry(pLogStore, pEntry, doFsync) < 0) {
     sError("failed to append sync log entry since %s. index:%" PRId64 ", term:%" PRId64 "", terrstr(), pEntry->index,
            pEntry->term);
     return -1;
@@ -436,7 +441,7 @@ int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* p
     (void)syncNodeReplicateWithoutLock(pNode);
 
     // persist
-    if (syncLogStorePersist(pLogStore, pEntry) < 0) {
+    if (syncLogStorePersist(pLogStore, pNode, pEntry) < 0) {
       sError("vgId:%d, failed to persist sync log entry from buffer since %s. index:%" PRId64, pNode->vgId, terrstr(),
              pEntry->index);
       goto _out;
diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c
index ca6d3c314f..e6569d9974 100644
--- a/source/libs/sync/src/syncRaftLog.c
+++ b/source/libs/sync/src/syncRaftLog.c
@@ -23,7 +23,7 @@
 
 // public function
 static int32_t   raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncIndex snapshotIndex);
-static int32_t   raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry);
+static int32_t   raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, bool forceSync);
 static int32_t   raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex);
 static bool      raftLogExist(struct SSyncLogStore* pLogStore, SyncIndex index);
 static int32_t   raftLogUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index);
@@ -192,9 +192,7 @@ SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
   return SYNC_TERM_INVALID;
 }
 
-static inline bool raftLogForceSync(SSyncRaftEntry* pEntry) { return (pEntry->originalRpcType == TDMT_VND_COMMIT); }
-
-static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
+static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, bool forceSync) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
 
@@ -221,7 +219,6 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
 
   ASSERT(pEntry->index == index);
 
-  bool forceSync = raftLogForceSync(pEntry);
   walFsync(pWal, forceSync);
 
   sNTrace(pData->pSyncNode, "write index:%" PRId64 ", type:%s, origin type:%s, elapsed:%" PRId64, pEntry->index,

From 06f863fbad5a60c2b756cdd9a79f93a5dec9a87c Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 12 Jan 2023 17:38:07 +0800
Subject: [PATCH 02/40] fix:memory leak

---
 source/client/src/clientRawBlockWrite.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 8ceb2a3380..9a3838a6b4 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1448,6 +1448,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
   end:
   taosMemoryFreeClear(pTableMeta);
   qDestroyQuery(pQuery);
+  destroyRequest(pRequest);
   taosMemoryFree(subReq);
   return code;
 }
@@ -1639,6 +1640,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
   end:
   taosMemoryFreeClear(pTableMeta);
   qDestroyQuery(pQuery);
+  destroyRequest(pRequest);
   taosMemoryFree(subReq);
   return code;
 }

From 9997362cf7f15575a4e6c9f5c602ce5e04c554c8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 14 Jan 2023 01:12:15 +0800
Subject: [PATCH 03/40] fix(query): use the recycled blocks to reduce the
 cached buffer.

---
 source/libs/executor/src/exchangeoperator.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c
index 9873c52006..0de2836f55 100644
--- a/source/libs/executor/src/exchangeoperator.c
+++ b/source/libs/executor/src/exchangeoperator.c
@@ -584,7 +584,13 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa
   int32_t index = 0;
   int32_t code = 0;
   while (index++ < pRetrieveRsp->numOfBlocks) {
-    SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false);
+    SSDataBlock* pb = NULL;
+    if (taosArrayGetSize(pExchangeInfo->pRecycledBlocks) > 0) {
+      pb = *(SSDataBlock**)taosArrayPop(pExchangeInfo->pRecycledBlocks);
+      blockDataCleanup(pb);
+    } else {
+      pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false);
+    }
 
     code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart);
     if (code != 0) {

From 51cdd4af19f1a000987d71ce5944f3c642c5d713 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Mon, 16 Jan 2023 10:28:34 +0800
Subject: [PATCH 04/40] fix: add stt_trigger to show create database processing

---
 include/common/tmsg.h               | 1 +
 source/common/src/tmsg.c            | 3 ++-
 source/dnode/mnode/impl/src/mndDb.c | 2 +-
 source/libs/command/src/command.c   | 4 ++--
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index ad6077db09..0cc9fb8619 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -907,6 +907,7 @@ typedef struct {
   int32_t numOfRetensions;
   SArray* pRetensions;
   int8_t  schemaless;
+  int16_t sstTrigger;
 } SDbCfgRsp;
 
 int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp);
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 95625e8d93..891f5e21ab 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -2821,8 +2821,8 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) {
     if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1;
   }
   if (tEncodeI8(&encoder, pRsp->schemaless) < 0) return -1;
+  if (tEncodeI16(&encoder, pRsp->sstTrigger) < 0) return -1;
   tEndEncode(&encoder);
-
   int32_t tlen = encoder.pos;
   tEncoderClear(&encoder);
   return tlen;
@@ -2873,6 +2873,7 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) {
     }
   }
   if (tDecodeI8(&decoder, &pRsp->schemaless) < 0) return -1;
+  if (tDecodeI16(&decoder, &pRsp->sstTrigger) < 0) return -1;
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c
index 7e5c29d56f..bdfda14a32 100644
--- a/source/dnode/mnode/impl/src/mndDb.c
+++ b/source/dnode/mnode/impl/src/mndDb.c
@@ -889,7 +889,7 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) {
   cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions;
   cfgRsp.pRetensions = pDb->cfg.pRetensions;
   cfgRsp.schemaless = pDb->cfg.schemaless;
-
+  cfgRsp.sstTrigger = pDb->cfg.sstTrigger;
   int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &cfgRsp);
   void   *pRsp = rpcMallocCont(contLen);
   if (pRsp == NULL) {
diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c
index a179ec24f9..3f10ed7388 100644
--- a/source/libs/command/src/command.c
+++ b/source/libs/command/src/command.c
@@ -264,10 +264,10 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S
   len += sprintf(
       buf2 + VARSTR_HEADER_SIZE,
       "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm "
-      "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
+      "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
       "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d",
       dbFName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile,
-      pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
+      pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows,  pCfg->sstTrigger, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
       pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups,
       1 == pCfg->numOfStables);
 

From 29bccd26240697bd127896615c394f1329c2b5b2 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Mon, 16 Jan 2023 11:40:45 +0800
Subject: [PATCH 05/40] fix: taosbenchmark schemaless refine main (#19567)

* fix: taos-tools 143b9e4 for taosbenchmark schemaless refine for main

* fix: update taos-tools 723f696
---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index bc3fe87884..cddc5aee4e 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG f80dd7e
+        GIT_TAG 723f696
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From 04a3b1d5984d86306f5618bb1356bb2348eec33f Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Mon, 16 Jan 2023 14:23:40 +0800
Subject: [PATCH 06/40] fix: add ci test

---
 tests/develop-test/2-query/show_create_db.py | 82 ++++++++++++++++++++
 tests/parallel_test/cases.task               |  1 +
 2 files changed, 83 insertions(+)
 create mode 100644 tests/develop-test/2-query/show_create_db.py

diff --git a/tests/develop-test/2-query/show_create_db.py b/tests/develop-test/2-query/show_create_db.py
new file mode 100644
index 0000000000..e5a79074ef
--- /dev/null
+++ b/tests/develop-test/2-query/show_create_db.py
@@ -0,0 +1,82 @@
+import sys 
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import tdDnodes
+from math import inf
+
+class TDTestCase:
+    def caseDescription(self):
+        '''
+        case1<shenglian zhou>: [TD-11204]Difference improvement that can ignore negative 
+        ''' 
+        return
+    
+    def init(self, conn, logSql, replicaVer=1):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), False)
+        self._conn = conn
+        
+    def restartTaosd(self, index=1, dbname="db"):
+        tdDnodes.stop(index)
+        tdDnodes.startWithoutSleep(index)
+        tdSql.execute(f"use scd")
+
+    def run(self):
+        print("running {}".format(__file__))
+        tdSql.execute("drop database if exists scd")
+        tdSql.execute("create database if not exists scd")
+        tdSql.execute('use scd')
+        tdSql.execute('create table stb1 (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);')
+
+        tdSql.execute("create table tb1 using stb1 tags(1,'1',1.0);")
+
+        tdSql.execute("create table tb2 using stb1 tags(2,'2',2.0);")
+
+        tdSql.execute("create table tb3 using stb1 tags(3,'3',3.0);")
+
+        tdSql.execute('create database scd2 stt_trigger 3;')
+
+        tdSql.execute('create database scd4 stt_trigger 13;')
+
+        tdSql.query('show create database scd;')
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'scd')
+        tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 1 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0")
+
+        tdSql.query('show create database scd2;')
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'scd2')
+        tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0")
+
+        tdSql.query('show create database scd4')
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'scd4')
+        tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0")
+
+
+        self.restartTaosd(1, dbname='scd')
+
+        tdSql.query('show create database scd;')
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'scd')
+        tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 1 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0")
+
+        tdSql.query('show create database scd2;')
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'scd2')
+        tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0")
+
+        tdSql.query('show create database scd4')
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 'scd4')
+        tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0")
+
+
+        tdSql.execute('drop database scd')
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index da81c2f59f..b002753b7a 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -1051,6 +1051,7 @@
 
 #develop test
 ,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py
+,,n,develop-test,python3 ./test.py -f 2-query/show_create_db.py
 ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py
 ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py
 ,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py

From 20b4c1ca832ddf94e3d6b7e91dbe8bf5def9c8f0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 16 Jan 2023 15:16:06 +0800
Subject: [PATCH 07/40] test: add new sim to CI.

---
 tests/parallel_test/cases.task | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 1205da31b3..cadfe2cff7 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -145,6 +145,7 @@
 ,,y,script,./test.sh -f tsim/parser/precision_ns.sim
 ,,y,script,./test.sh -f tsim/parser/projection_limit_offset.sim
 ,,y,script,./test.sh -f tsim/parser/regex.sim
+,,y,script,./test.sh -f tsim/parser/regressiontest.sim
 ,,y,script,./test.sh -f tsim/parser/select_across_vnodes.sim
 ,,y,script,./test.sh -f tsim/parser/select_distinct_tag.sim
 ,,y,script,./test.sh -f tsim/parser/select_from_cache_disk.sim

From e3248d005369a68dac78dca6a28823c88ff6bf82 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 16 Jan 2023 16:02:23 +0800
Subject: [PATCH 08/40] refactor: disable warning.

---
 source/libs/executor/src/executil.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index a5468008aa..36981f501e 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -954,7 +954,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
           return -1;
         }
       } else {
-        qWarn("failed to get tableIds from by table name: %s, reason: %s", name, tstrerror(terrno));
+//        qWarn("failed to get tableIds from by table name: %s, reason: %s", name, tstrerror(terrno));
         terrno = 0;
       }
     }

From ab7761e64ed33016e5cc637da7c0d016cebc9b27 Mon Sep 17 00:00:00 2001
From: chenhaoran <haoran920c@163.com>
Date: Mon, 16 Jan 2023 18:35:26 +0800
Subject: [PATCH 09/40] ci:modify the filename of the error log

---
 tests/parallel_test/run.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh
index b5d57265be..43533d4f36 100755
--- a/tests/parallel_test/run.sh
+++ b/tests/parallel_test/run.sh
@@ -184,6 +184,10 @@ function run_thread() {
         if [ $? -eq 0 ]; then
             case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'`
         fi
+        echo "$case_cmd"|grep -q "^./pytest.sh"
+        if [ $? -eq 0 ]; then
+            case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'`
+        fi
         echo "$case_cmd"|grep -q "\.sim"
         if [ $? -eq 0 ]; then
             case_file=`echo "$case_cmd"|grep -o ".*\.sim"|awk '{print $NF}'`

From 215864c5cd8884911f7cdfe5b3c6913c1e95234a Mon Sep 17 00:00:00 2001
From: chenhaoran <haoran920c@163.com>
Date: Mon, 16 Jan 2023 19:29:50 +0800
Subject: [PATCH 10/40] ci:delete para of 'make -j'

---
 Jenkinsfile2                           |  2 +-
 tests/parallel_test/container_build.sh | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile2 b/Jenkinsfile2
index 80f6b8e9e7..dd15807308 100644
--- a/Jenkinsfile2
+++ b/Jenkinsfile2
@@ -430,7 +430,7 @@ pipeline {
                                         date
                                         rm -rf ${WKC}/debug
                                         cd ${WKC}/tests/parallel_test
-                                        time ./container_build.sh -w ${WKDIR} -t 10 -e
+                                        time ./container_build.sh -w ${WKDIR} -e
                                     '''
                                     def extra_param = ""
                                     def log_server_file = "/home/log_server.json"
diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh
index 5059630a3f..221e549056 100755
--- a/tests/parallel_test/container_build.sh
+++ b/tests/parallel_test/container_build.sh
@@ -37,9 +37,9 @@ if [ -z "$WORKDIR" ]; then
     usage
     exit 1
 fi
-if [ -z "$THREAD_COUNT" ]; then
-    THREAD_COUNT=1
-fi
+# if [ -z "$THREAD_COUNT" ]; then
+#     THREAD_COUNT=1
+# fi
 
 ulimit -c unlimited
 
@@ -55,7 +55,7 @@ fi
 date 
 docker run \
     -v $REP_MOUNT_PARAM \
-    --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true;make -j $THREAD_COUNT || exit 1"
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true;make -j || exit 1"
 
 if [[ -d ${WORKDIR}/debugNoSan  ]] ;then
     echo "delete  ${WORKDIR}/debugNoSan"
@@ -70,7 +70,7 @@ mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugNoSan
 date
 docker run \
     -v $REP_MOUNT_PARAM \
-    --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true   -DBUILD_SANITIZER=1  -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug;make -j $THREAD_COUNT || exit 1 "
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true   -DBUILD_SANITIZER=1  -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug;make -j || exit 1 "
 
 mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugSan
 

From 63a23a3562011ff438c84e55b852587099075543 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Tue, 17 Jan 2023 09:12:51 +0800
Subject: [PATCH 11/40] fix: insert into select can't be stopped issue

---
 source/client/src/clientMain.c         |  3 +--
 source/client/src/clientStmt.c         |  6 +-----
 source/libs/parser/src/parInsertStmt.c | 23 ++++++++++++++++++++++-
 source/util/src/tlog.c                 |  1 +
 tests/script/api/batchprepare.c        |  2 +-
 5 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 15c1d65162..e5f677637e 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -509,9 +509,8 @@ void taos_stop_query(TAOS_RES *res) {
   SRequestObj *pRequest = (SRequestObj *)res;
   pRequest->killed = true;
 
-  int32_t numOfFields = taos_num_fields(pRequest);
   // It is not a query, no need to stop.
-  if (numOfFields == 0) {
+  if (NULL == pRequest->pQuery || QUERY_EXEC_MODE_SCHEDULE != pRequest->pQuery->execMode) {
     tscDebug("request 0x%" PRIx64 " no need to be killed since not query", pRequest->requestId);
     return;
   }
diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c
index 82ea9e0d8f..1ec6450228 100644
--- a/source/client/src/clientStmt.c
+++ b/source/client/src/clientStmt.c
@@ -300,11 +300,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) {
       continue;
     }
 
-    if (STMT_TYPE_MULTI_INSERT == pStmt->sql.type) {
-      qFreeStmtDataBlock(pBlocks);
-    } else {
-      qDestroyStmtDataBlock(pBlocks);
-    }
+    qDestroyStmtDataBlock(pBlocks);
     taosHashRemove(pStmt->exec.pBlockHash, key, keyLen);
 
     pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter);
diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c
index 4ed72e6c14..1f437e8a8c 100644
--- a/source/libs/parser/src/parInsertStmt.c
+++ b/source/libs/parser/src/parInsertStmt.c
@@ -425,6 +425,27 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) {
     pBlock->pTableMeta = pNewMeta;
   }
 
+  if (pBlock->boundColumnInfo.boundColumns) {
+    int32_t size = pBlock->boundColumnInfo.numOfCols * sizeof(col_id_t);
+    void* tmp = taosMemoryMalloc(size);
+    memcpy(tmp, pBlock->boundColumnInfo.boundColumns, size);
+    pBlock->boundColumnInfo.boundColumns = tmp;
+  }
+
+  if (pBlock->boundColumnInfo.cols) {
+    int32_t size = pBlock->boundColumnInfo.numOfCols * sizeof(SBoundColumn);
+    void* tmp = taosMemoryMalloc(size);
+    memcpy(tmp, pBlock->boundColumnInfo.cols, size);
+    pBlock->boundColumnInfo.cols = tmp;
+  }
+
+  if (pBlock->boundColumnInfo.colIdxInfo) {
+    int32_t size = pBlock->boundColumnInfo.numOfBound * sizeof(SBoundIdxInfo);
+    void* tmp = taosMemoryMalloc(size);
+    memcpy(tmp, pBlock->boundColumnInfo.colIdxInfo, size);
+    pBlock->boundColumnInfo.colIdxInfo = tmp;
+  }
+
   return qResetStmtDataBlock(*pDst, false);
 }
 
@@ -437,7 +458,7 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgI
   STableDataBlocks* pBlock = (STableDataBlocks*)*pDst;
   pBlock->pData = taosMemoryMalloc(pBlock->nAllocSize);
   if (NULL == pBlock->pData) {
-    qFreeStmtDataBlock(pBlock);
+    qDestroyStmtDataBlock(pBlock);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c
index 34ad9ae6bc..62f074db5b 100644
--- a/source/util/src/tlog.c
+++ b/source/util/src/tlog.c
@@ -897,6 +897,7 @@ void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, vo
 
     pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
     if (pFile == NULL) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
       taosPrintLog(flags, level, dflag, "failed to open file:%s since %s", filepath, terrstr());
       goto _return;
     }
diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c
index 60df188d7b..d1a80d9683 100644
--- a/tests/script/api/batchprepare.c
+++ b/tests/script/api/batchprepare.c
@@ -2828,7 +2828,7 @@ void runAll(TAOS *taos) {
   printf("%s Begin\n", gCaseCtrl.caseCatalog);
   runCaseList(taos);
 
-#if 0
+#if 1
   strcpy(gCaseCtrl.caseCatalog, "Micro DB precision Test");
   printf("%s Begin\n", gCaseCtrl.caseCatalog);
   gCaseCtrl.precision = TIME_PRECISION_MICRO;

From 6b259777adfd7500c4b3ea642e20e90793562052 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Tue, 17 Jan 2023 11:18:49 +0800
Subject: [PATCH 12/40] fix: move crash report to shell

---
 source/client/src/clientEnv.c  | 66 ++++++++++------------------------
 source/client/src/clientImpl.c |  2 +-
 tools/shell/inc/shellInt.h     |  1 +
 tools/shell/src/shellEngine.c  |  4 +--
 tools/shell/src/shellMain.c    | 30 ++++++++++++++++
 5 files changed, 51 insertions(+), 52 deletions(-)

diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 495c2cca9a..3670d3862c 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -388,45 +388,6 @@ void destroyRequest(SRequestObj *pRequest) {
   removeRequest(pRequest->self);
 }
 
-void taosClientCrash(int signum, void *sigInfo, void *context) {
-  taosIgnSignal(SIGTERM);
-  taosIgnSignal(SIGHUP);
-  taosIgnSignal(SIGINT);
-  taosIgnSignal(SIGBREAK);
-
-#if !defined(WINDOWS)
-  taosIgnSignal(SIGBUS);
-#endif  
-  taosIgnSignal(SIGABRT);
-  taosIgnSignal(SIGFPE);
-  taosIgnSignal(SIGSEGV);
-
-  char *pMsg = NULL;
-  const char *flags = "UTL FATAL ";
-  ELogLevel   level = DEBUG_FATAL;
-  int32_t     dflag = 255;
-  int64_t     msgLen= -1;
-
-  if (tsEnableCrashReport) {
-    if (taosGenCrashJsonMsg(signum, &pMsg, lastClusterId, appInfo.startTime)) {
-      taosPrintLog(flags, level, dflag, "failed to generate crash json msg");
-      goto _return;
-    } else {
-      msgLen = strlen(pMsg);  
-    }
-  }
-  
-_return:
-
-  taosLogCrashInfo("taos", pMsg, msgLen, signum, sigInfo);
-
-#ifdef _TD_DARWIN_64
-  exit(signum);
-#elif defined(WINDOWS)
-  exit(signum);
-#endif
-}
-
 void crashReportThreadFuncUnexpectedStopped(void) { atomic_store_32(&clientStop, -1); }
 
 static void *tscCrashReportThreadFp(void *param) {
@@ -523,15 +484,26 @@ void tscStopCrashReport() {
   }
 }
 
-static void tscSetSignalHandle() {
-#if !defined(WINDOWS)
-  taosSetSignal(SIGBUS, taosClientCrash);
-#endif
-  taosSetSignal(SIGABRT, taosClientCrash);
-  taosSetSignal(SIGFPE, taosClientCrash);
-  taosSetSignal(SIGSEGV, taosClientCrash);
+
+void tscWriteCrashInfo(int signum, void *sigInfo, void *context) {
+  char *pMsg = NULL;
+  const char *flags = "UTL FATAL ";
+  ELogLevel   level = DEBUG_FATAL;
+  int32_t     dflag = 255;
+  int64_t     msgLen= -1;
+
+  if (tsEnableCrashReport) {
+    if (taosGenCrashJsonMsg(signum, &pMsg, lastClusterId, appInfo.startTime)) {
+      taosPrintLog(flags, level, dflag, "failed to generate crash json msg");
+    } else {
+      msgLen = strlen(pMsg);  
+    }
+  }
+
+  taosLogCrashInfo("taos", pMsg, msgLen, signum, sigInfo);
 }
 
+
 void taos_init_imp(void) {
   // In the APIs of other program language, taos_cleanup is not available yet.
   // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning.
@@ -555,8 +527,6 @@ void taos_init_imp(void) {
     return;
   }
 
-  tscSetSignalHandle();
-
   initQueryModuleMsgHandle();
 
   if (taosConvInit() != 0) {
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 53acafeeaa..f36036fd0a 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -1253,7 +1253,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
 
   int64_t transporterId = 0;
   asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
-  
+
   tsem_wait(&pRequest->body.rspSem);
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     const char* errorMsg =
diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h
index af724c1533..e2da695c92 100644
--- a/tools/shell/inc/shellInt.h
+++ b/tools/shell/inc/shellInt.h
@@ -147,5 +147,6 @@ void shellRunSingleCommandWebsocketImp(char *command);
 
 // shellMain.c
 extern SShellObj shell;
+extern void tscWriteCrashInfo(int signum, void *sigInfo, void *context);
 
 #endif /*_TD_SHELL_INT_H_*/
diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index 986806fdd8..479c2cf39a 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -1136,10 +1136,8 @@ int32_t shellExecute() {
 
   taosSetSignal(SIGTERM, shellQueryInterruptHandler);
   taosSetSignal(SIGHUP, shellQueryInterruptHandler);
-  taosSetSignal(SIGABRT, shellQueryInterruptHandler);
-
   taosSetSignal(SIGINT, shellQueryInterruptHandler);
-
+  
 #ifdef WEBSOCKET
   if (!shell.args.restful && !shell.args.cloud) {
 #endif
diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c
index fa3c0f2585..22b8e89959 100644
--- a/tools/shell/src/shellMain.c
+++ b/tools/shell/src/shellMain.c
@@ -19,6 +19,29 @@
 
 SShellObj shell = {0};
 
+
+void shellCrashHandler(int signum, void *sigInfo, void *context) {
+  taosIgnSignal(SIGTERM);
+  taosIgnSignal(SIGHUP);
+  taosIgnSignal(SIGINT);
+  taosIgnSignal(SIGBREAK);
+
+#if !defined(WINDOWS)
+  taosIgnSignal(SIGBUS);
+#endif  
+  taosIgnSignal(SIGABRT);
+  taosIgnSignal(SIGFPE);
+  taosIgnSignal(SIGSEGV);
+
+  tscWriteCrashInfo(signum, sigInfo, context);
+
+#ifdef _TD_DARWIN_64
+  exit(signum);
+#elif defined(WINDOWS)
+  exit(signum);
+#endif
+}
+
 int main(int argc, char *argv[]) {
   shell.exit = false;
 #ifdef WEBSOCKET
@@ -26,6 +49,13 @@ int main(int argc, char *argv[]) {
   shell.args.cloud = true;
 #endif
 
+#if !defined(WINDOWS)
+  taosSetSignal(SIGBUS, shellCrashHandler);
+#endif
+  taosSetSignal(SIGABRT, shellCrashHandler);
+  taosSetSignal(SIGFPE, shellCrashHandler);
+  taosSetSignal(SIGSEGV, shellCrashHandler);
+
   if (shellCheckIntSize() != 0) {
     return -1;
   }

From 6284e872178522f78af725d377f79889ff19bd43 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Tue, 17 Jan 2023 14:36:00 +0800
Subject: [PATCH 13/40] fix: fix vgroup dead lock issue

---
 source/libs/catalog/src/catalog.c  | 8 +++++++-
 source/libs/catalog/src/ctgAsync.c | 2 ++
 source/libs/catalog/src/ctgCache.c | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c
index c7af0411be..f9a218835e 100644
--- a/source/libs/catalog/src/catalog.c
+++ b/source/libs/catalog/src/catalog.c
@@ -598,10 +598,16 @@ int32_t ctgGetCachedTbVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInf
 
   CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pTableName, pVgroup));
 
+  ctgRUnlockVgInfo(dbCache);
+
   SCtgTbMetaCtx ctx = {0};
   ctx.pName = (SName*)pTableName;
   ctx.flag = CTG_FLAG_UNKNOWN_STB;
-  CTG_ERR_JRET(ctgCopyTbMeta(pCtg, &ctx, &dbCache, &tbCache, pTableMeta, db));
+  code = ctgCopyTbMeta(pCtg, &ctx, &dbCache, &tbCache, pTableMeta, db);
+
+  ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache);
+
+  CTG_RET(code);
 
 _return:
   
diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c
index 438128203e..325d6e0e46 100644
--- a/source/libs/catalog/src/ctgAsync.c
+++ b/source/libs/catalog/src/ctgAsync.c
@@ -999,6 +999,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf
             CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq));
 
             ctgReleaseVgInfoToCache(pCtg, dbCache);
+            dbCache = NULL;
           } else {
             SBuildUseDBInput input = {0};
 
@@ -1168,6 +1169,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu
             CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq));
 
             ctgReleaseVgInfoToCache(pCtg, dbCache);
+            dbCache = NULL;
           } else {
             SBuildUseDBInput input = {0};
 
diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c
index c266cc1df9..6e4077eae0 100644
--- a/source/libs/catalog/src/ctgCache.c
+++ b/source/libs/catalog/src/ctgCache.c
@@ -2118,7 +2118,7 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) {
 
 _return:
 
-  if (dbCache) {
+  if (code == TSDB_CODE_SUCCESS && dbCache) {
     ctgWUnlockVgInfo(dbCache);
   }
 

From 69ecd6e50a05b87a082ae05b2894ea5925670851 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Tue, 17 Jan 2023 14:46:33 +0800
Subject: [PATCH 14/40] fix: compile issue

---
 source/libs/catalog/inc/catalogInt.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h
index 3e8300e05d..30593fbaab 100644
--- a/source/libs/catalog/inc/catalogInt.h
+++ b/source/libs/catalog/inc/catalogInt.h
@@ -805,6 +805,7 @@ int32_t ctgMakeVgArray(SDBVgInfo* dbInfo);
 int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb);
 int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta, char* dbFName);
 void    ctgReleaseVgMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache);
+void    ctgReleaseTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache);
 
 extern SCatalogMgmt gCtgMgmt;
 extern SCtgDebug    gCTGDebug;

From e92bf83986f4cb51e891e08a851ada89e5f20236 Mon Sep 17 00:00:00 2001
From: freemine <freemine@yeah.net>
Date: Tue, 17 Jan 2023 15:05:33 +0800
Subject: [PATCH 15/40] bugfix: stack-buffer-overflow (#19584)

---
 source/os/src/osSysinfo.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c
index 7521ae4e0f..aeaa4fcafd 100644
--- a/source/os/src/osSysinfo.c
+++ b/source/os/src/osSysinfo.c
@@ -834,7 +834,11 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) {
   uuid_generate(uuid);
   // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null
   uuid_unparse_lower(uuid, buf);
-  memcpy(uid, buf, uidlen);
+  int n = snprintf(uid, uidlen, "%.*s", (int)sizeof(buf), buf);  // though less performance, much safer
+  if (n >= uidlen) {
+    // target buffer is too small
+    return -1;
+  }
   return 0;
 #else
   int len = 0;

From 70fb18e61195254e983a5193ed4b77287f1a038b Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Tue, 17 Jan 2023 15:11:11 +0800
Subject: [PATCH 16/40] fix: tsdb ReadSnap take/untake

---
 source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 12 ++++++++----
 source/dnode/vnode/src/tsdb/tsdbRead.c      |  4 ++--
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
index f05f5d5c88..a837543e62 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c
@@ -164,7 +164,7 @@ void* tsdbCacherowsReaderClose(void* pReader) {
 
   destroyLastBlockLoadInfo(p->pLoadInfo);
 
-  taosMemoryFree((void*) p->idstr);
+  taosMemoryFree((void*)p->idstr);
   taosMemoryFree(pReader);
   return NULL;
 }
@@ -241,7 +241,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
     taosArrayPush(pLastCols, &p);
   }
 
-  tsdbTakeReadSnap(pr->pVnode->pTsdb, &pr->pReadSnap, "cache-l");
+  code = tsdbTakeReadSnap(pr->pVnode->pTsdb, &pr->pReadSnap, "cache-l");
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _end;
+  }
+
   pr->pDataFReader = NULL;
   pr->pDataFReaderLast = NULL;
 
@@ -252,7 +256,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
 
       code = doExtractCacheRow(pr, lruCache, pKeyInfo->uid, &pRow, &h);
       if (code != TSDB_CODE_SUCCESS) {
-        return code;
+        goto _end;
       }
 
       if (h == NULL) {
@@ -321,7 +325,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
       STableKeyInfo* pKeyInfo = &pr->pTableList[i];
       code = doExtractCacheRow(pr, lruCache, pKeyInfo->uid, &pRow, &h);
       if (code != TSDB_CODE_SUCCESS) {
-        return code;
+        goto _end;
       }
 
       if (h == NULL) {
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index b21050b2ae..ee93756f0e 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -3884,7 +3884,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
     if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) {
       code = doOpenReaderImpl(pReader);
       if (code != TSDB_CODE_SUCCESS) {
-        return code;
+        goto _err;
       }
     } else {
       STsdbReader* pPrevReader = pReader->innerReader[0];
@@ -3905,7 +3905,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
 
       code = doOpenReaderImpl(pPrevReader);
       if (code != TSDB_CODE_SUCCESS) {
-        return code;
+        goto _err;
       }
     }
   }

From f966197cea582f9a5c4ffb156ac9cb39a7a3b262 Mon Sep 17 00:00:00 2001
From: xinsheng Ren <285808407@qq.com>
Date: Tue, 17 Jan 2023 16:30:06 +0800
Subject: [PATCH 17/40] Fix/xsren/td 21762/sem mem leak basemain (#19587)

* TD-21762 sem_init cause mem leak on windows

* fix/memleak

Co-authored-by: facetosea <25808407@qq.com>
---
 source/client/src/clientEnv.c  |  4 ++++
 source/client/src/clientImpl.c | 21 ++++++++++++++-------
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 3670d3862c..593a8fd20a 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -357,6 +357,7 @@ void doDestroyRequest(void *p) {
   taosMemoryFreeClear(pRequest->pDb);
 
   doFreeReqResultInfo(&pRequest->body.resInfo);
+  tsem_destroy(&pRequest->body.rspSem);
 
   taosArrayDestroy(pRequest->tableList);
   taosArrayDestroy(pRequest->dbList);
@@ -371,6 +372,9 @@ void doDestroyRequest(void *p) {
   }
 
   if (pRequest->syncQuery) {
+      if (pRequest->body.param){
+        tsem_destroy(&((SSyncQueryParam*)pRequest->body.param)->sem);
+      }
     taosMemoryFree(pRequest->body.param);
   }
 
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index f36036fd0a..b5b99e92b0 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -159,6 +159,12 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
   return taosConnectImpl(user, &secretEncrypt[0], localDb, NULL, NULL, *pInst, connType);
 }
 
+void freeQueryParam(SSyncQueryParam* param) {
+  if (param == NULL) return;
+  tsem_destroy(&param->sem);
+  taosMemoryFree(param);
+}
+
 int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, bool validateSql,
                      SRequestObj** pRequest, int64_t reqid) {
   *pRequest = createRequest(connId, TSDB_SQL_SELECT, reqid);
@@ -180,17 +186,18 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
   (*pRequest)->sqlLen = sqlLen;
   (*pRequest)->validateOnly = validateSql;
 
+  SSyncQueryParam* newpParam;
   if (param == NULL) {
-    SSyncQueryParam* pParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
-    if (pParam == NULL) {
+    newpParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+    if (newpParam == NULL) {
       destroyRequest(*pRequest);
       *pRequest = NULL;
       return TSDB_CODE_OUT_OF_MEMORY;
     }
 
-    tsem_init(&pParam->sem, 0, 0);
-    pParam->pRequest = (*pRequest);
-    param = pParam;
+    tsem_init(&newpParam->sem, 0, 0);
+    newpParam->pRequest = (*pRequest);
+    param = newpParam;
   }
 
   (*pRequest)->body.param = param;
@@ -201,8 +208,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
   if (err) {
     tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
              (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
-
-    taosMemoryFree(param);
+    freeQueryParam(newpParam);
     destroyRequest(*pRequest);
     *pRequest = NULL;
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -214,6 +220,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
         nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) {
       tscError("%" PRId64 " failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
                (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
+      freeQueryParam(newpParam);
       destroyRequest(*pRequest);
       *pRequest = NULL;
       return TSDB_CODE_OUT_OF_MEMORY;

From da8d80dfeb935ac5acbab66804e2e94ade600a59 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Tue, 17 Jan 2023 17:45:42 +0800
Subject: [PATCH 18/40] fix: packaging/release.sh don't catch error temporarily
 for main (#19611)

---
 packaging/release.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/release.sh b/packaging/release.sh
index 1dfbf2b112..e442e07c6a 100755
--- a/packaging/release.sh
+++ b/packaging/release.sh
@@ -2,7 +2,7 @@
 #
 # Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os
 
-set -e
+# set -e
 # set -x
 
 # release.sh  -v [cluster | edge]

From e70f767647d5de6585acd76a84a57483c74b8bd1 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Tue, 17 Jan 2023 17:53:56 +0800
Subject: [PATCH 19/40] fix: crash caused of wrong task phase issue

---
 source/libs/qworker/inc/qwInt.h   | 7 ++++++-
 source/libs/qworker/src/qworker.c | 6 ++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h
index 35b2479a51..aa1ce80903 100644
--- a/source/libs/qworker/inc/qwInt.h
+++ b/source/libs/qworker/inc/qwInt.h
@@ -228,9 +228,14 @@ typedef struct SQWorkerMgmt {
       case QW_PHASE_POST_FETCH:                                              \
         ctx->inFetch = 0;                                                    \
         break;                                                               \
-      default:                                                               \
+      case QW_PHASE_PRE_QUERY:                                               \
+      case QW_PHASE_POST_QUERY:                                              \
+      case QW_PHASE_PRE_CQUERY:                                              \
+      case QW_PHASE_POST_CQUERY:                                             \
         atomic_store_8(&(ctx)->phase, _value);                               \
         break;                                                               \
+      default:                                                               \
+        break;                                                               \
     }                                                                        \
   } while (0)
 
diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c
index dcb7c02580..e38361d87f 100644
--- a/source/libs/qworker/src/qworker.c
+++ b/source/libs/qworker/src/qworker.c
@@ -551,7 +551,9 @@ _return:
   if (ctx) {
     QW_UPDATE_RSP_CODE(ctx, code);
 
-    QW_SET_PHASE(ctx, phase);
+    if (QW_PHASE_POST_CQUERY != phase) {
+      QW_SET_PHASE(ctx, phase);
+    }
 
     QW_UNLOCK(QW_WRITE, &ctx->lock);
     qwReleaseTaskCtx(mgmt, ctx);
@@ -758,7 +760,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
     QW_LOCK(QW_WRITE, &ctx->lock);
     if (qComplete || (queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code) {
       // Note: query is not running anymore
-      QW_SET_PHASE(ctx, 0);
+      QW_SET_PHASE(ctx, QW_PHASE_POST_CQUERY);
       QW_UNLOCK(QW_WRITE, &ctx->lock);
       break;
     }

From d9ae5a755c683e3ff828adb46ba5234d8ed42550 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Wed, 18 Jan 2023 11:05:39 +0800
Subject: [PATCH 20/40] fix: tsdb fs head/data/sma nRef

---
 source/dnode/vnode/src/tsdb/tsdbFS.c | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c
index 7dc839773f..51fdc69a95 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFS.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS.c
@@ -458,9 +458,8 @@ static int32_t tsdbMergeFileSet(STsdb *pTsdb, SDFileSet *pSetOld, SDFileSet *pSe
       taosMemoryFree(pHeadF);
     }
   } else {
-    nRef = pHeadF->nRef;
-    *pHeadF = *pSetNew->pHeadF;
-    pHeadF->nRef = nRef;
+    ASSERT(pHeadF->offset == pSetNew->pHeadF->offset);
+    ASSERT(pHeadF->size == pSetNew->pHeadF->size);
   }
 
   // data
@@ -481,9 +480,7 @@ static int32_t tsdbMergeFileSet(STsdb *pTsdb, SDFileSet *pSetOld, SDFileSet *pSe
       taosMemoryFree(pDataF);
     }
   } else {
-    nRef = pDataF->nRef;
-    *pDataF = *pSetNew->pDataF;
-    pDataF->nRef = nRef;
+    pDataF->size = pSetNew->pDataF->size;
   }
 
   // sma
@@ -504,9 +501,7 @@ static int32_t tsdbMergeFileSet(STsdb *pTsdb, SDFileSet *pSetOld, SDFileSet *pSe
       taosMemoryFree(pSmaF);
     }
   } else {
-    nRef = pSmaF->nRef;
-    *pSmaF = *pSetNew->pSmaF;
-    pSmaF->nRef = nRef;
+    pSmaF->size = pSetNew->pSmaF->size;
   }
 
   // stt

From 8082077c18355d8a6e49cc5589f8898af015d296 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Wed, 18 Jan 2023 11:41:33 +0800
Subject: [PATCH 21/40] chore: release script add comp for taostools formain
 (#19619)

* chore: add comp postfix for taos-tools

* fix: update taos-tools 5c53cc8 for main
---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index cddc5aee4e..1e2247eedc 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 723f696
+        GIT_TAG 5c53cc8
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From c9a1b3ba011878265d1e2f08bd4933ee8385a711 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 18 Jan 2023 16:24:29 +0800
Subject: [PATCH 22/40] fix(query):  handle the multi-group limit/offset in
 table group merge scan/ multiway-merge executor.

---
 source/libs/executor/inc/executorimpl.h     |  3 ++-
 source/libs/executor/src/exchangeoperator.c | 11 ++++++---
 source/libs/executor/src/executil.c         |  5 ++++
 source/libs/executor/src/projectoperator.c  | 12 ++++++---
 source/libs/executor/src/scanoperator.c     | 27 +++++++++++++++------
 source/libs/executor/src/sortoperator.c     | 11 +++++----
 6 files changed, 48 insertions(+), 21 deletions(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index fffe687ff6..c68f7c4697 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -705,7 +705,8 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
 
 bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
 void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);
-void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator);
+void resetLimitInfoForNextGroup(SLimitInfo* pLimitInfo);
+bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator);
 
 void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData,
                                      int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput);
diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c
index 0de2836f55..037b33dc9f 100644
--- a/source/libs/executor/src/exchangeoperator.c
+++ b/source/libs/executor/src/exchangeoperator.c
@@ -738,9 +738,7 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa
     }
 
     // reset the value for a new group data
-    pLimitInfo->numOfOutputRows = 0;
-    pLimitInfo->remainOffset = pLimitInfo->limit.offset;
-
+    resetLimitInfoForNextGroup(pLimitInfo);
     // existing rows that belongs to previous group.
     if (pBlock->info.rows > 0) {
       return PROJECT_RETRIEVE_DONE;
@@ -766,7 +764,12 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa
     int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows);
     blockDataKeepFirstNRows(pBlock, keepRows);
     if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) {
-      pOperator->status = OP_EXEC_DONE;
+      setOperatorCompleted(pOperator);
+    } else {
+      // current group limitation is reached, and future blocks of this group need to be discarded.
+      if (pBlock->info.rows == 0) {
+        return PROJECT_RETRIEVE_CONTINUE;
+      }
     }
 
     return PROJECT_RETRIEVE_DONE;
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 36981f501e..757324a773 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1759,6 +1759,11 @@ void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimit
   pLimitInfo->remainGroupOffset = slimit.offset;
 }
 
+void resetLimitInfoForNextGroup(SLimitInfo* pLimitInfo) {
+  pLimitInfo->numOfOutputRows = 0;
+  pLimitInfo->remainOffset = pLimitInfo->limit.offset;
+}
+
 uint64_t tableListGetSize(const STableListInfo* pTableList) {
   ASSERT(taosArrayGetSize(pTableList->pTableList) == taosHashGetSize(pTableList->map));
   return taosArrayGetSize(pTableList->pTableList);
diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c
index 4d38f2c8e9..3e3610827b 100644
--- a/source/libs/executor/src/projectoperator.c
+++ b/source/libs/executor/src/projectoperator.c
@@ -175,8 +175,7 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S
 
     // reset the value for a new group data
     // existing rows that belongs to previous group.
-    pLimitInfo->numOfOutputRows = 0;
-    pLimitInfo->remainOffset = pLimitInfo->limit.offset;
+    resetLimitInfoForNextGroup(pLimitInfo);
   }
 
   return PROJECT_RETRIEVE_DONE;
@@ -200,10 +199,18 @@ static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SS
   if (pLimitInfo->limit.limit >= 0 && pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) {
     int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows);
     blockDataKeepFirstNRows(pBlock, keepRows);
+
     // TODO: optimize it later when partition by + limit
+    // all retrieved requirement has been fulfilled, let's finish this
     if ((pLimitInfo->slimit.limit == -1 && pLimitInfo->currentGroupId == 0) ||
         (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
       setOperatorCompleted(pOperator);
+    } else {
+      // Even current group is done, there may be many vgroups remain existed, and we need to continue to retrieve data
+      // from next group. So let's continue this retrieve process
+      if (keepRows == 0) {
+        return PROJECT_RETRIEVE_CONTINUE;
+      }
     }
   }
 
@@ -357,7 +364,6 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
     pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
   }
 
-  // printDataBlock1(p, "project");
   return (p->info.rows > 0) ? p : NULL;
 }
 
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index eb38299938..813763fffa 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -257,7 +257,7 @@ static void doSetTagColumnData(STableScanBase* pTableScanInfo, SSDataBlock* pBlo
 }
 
 // todo handle the slimit info
-void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator) {
+bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator) {
   SLimit*     pLimit = &pLimitInfo->limit;
   const char* id = GET_TASKID(pTaskInfo);
 
@@ -266,6 +266,7 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
       pLimitInfo->remainOffset -= pBlock->info.rows;
       blockDataEmpty(pBlock);
       qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset, id);
+      return false;
     } else {
       blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset);
       pLimitInfo->remainOffset = 0;
@@ -274,13 +275,14 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
 
   if (pLimit->limit != -1 && pLimit->limit <= (pLimitInfo->numOfOutputRows + pBlock->info.rows)) {
     // limit the output rows
-    int32_t overflowRows = pLimitInfo->numOfOutputRows + pBlock->info.rows - pLimit->limit;
-    int32_t keep = pBlock->info.rows - overflowRows;
+    int32_t keep = (int32_t)(pLimit->limit - pLimitInfo->numOfOutputRows);
 
     blockDataKeepFirstNRows(pBlock, keep);
     qDebug("output limit %" PRId64 " has reached, %s", pLimit->limit, id);
-    pOperator->status = OP_EXEC_DONE;
+    return true;
   }
+
+  return false;
 }
 
 static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
@@ -391,7 +393,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
     }
   }
 
-  applyLimitOffset(&pTableScanInfo->limitInfo, pBlock, pTaskInfo, pOperator);
+  bool limitReached = applyLimitOffset(&pTableScanInfo->limitInfo, pBlock, pTaskInfo, pOperator);
+  if (limitReached) { // set operator flag is done
+    setOperatorCompleted(pOperator);
+  }
 
   pCost->totalRows += pBlock->info.rows;
   pTableScanInfo->limitInfo.numOfOutputRows = pCost->totalRows;
@@ -768,8 +773,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
 
     // reset value for the next group data output
     pOperator->status = OP_OPENED;
-    pInfo->base.limitInfo.numOfOutputRows = 0;
-    pInfo->base.limitInfo.remainOffset = pInfo->base.limitInfo.limit.offset;
+    resetLimitInfoForNextGroup(&pInfo->base.limitInfo);
 
     int32_t        num = 0;
     STableKeyInfo* pList = NULL;
@@ -2685,9 +2689,12 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
   taosArrayDestroy(pInfo->queryConds);
   pInfo->queryConds = NULL;
 
+  resetLimitInfoForNextGroup(&pInfo->limitInfo);
   return TSDB_CODE_SUCCESS;
 }
 
+// all data produced by this function only belongs to one group
+// slimit/soffset does not need to be concerned here, since this function only deal with data within one group.
 SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* pResBlock, int32_t capacity,
                                               SOperatorInfo* pOperator) {
   STableMergeScanInfo* pInfo = pOperator->info;
@@ -2707,10 +2714,12 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock*
     }
   }
 
-  qDebug("%s get sorted row blocks, rows:%d", GET_TASKID(pTaskInfo), pResBlock->info.rows);
   applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo, pOperator);
   pInfo->limitInfo.numOfOutputRows += pResBlock->info.rows;
 
+  qDebug("%s get sorted row block, rows:%d, limit:%"PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows,
+      pInfo->limitInfo.numOfOutputRows);
+
   return (pResBlock->info.rows > 0) ? pResBlock : NULL;
 }
 
@@ -2749,11 +2758,13 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
       pOperator->resultInfo.totalRows += pBlock->info.rows;
       return pBlock;
     } else {
+      // Data of this group are all dumped, let's try the next group
       stopGroupTableMergeScan(pOperator);
       if (pInfo->tableEndIndex >= tableListSize - 1) {
         setOperatorCompleted(pOperator);
         break;
       }
+
       pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
       pInfo->groupId = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->tableStartIndex)->groupId;
       startGroupTableMergeScan(pOperator);
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index f5dc6cc623..97b4fd9dc4 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -680,11 +680,13 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
       break;
     }
 
+    bool limitReached = applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo, pOperator);
+    if (limitReached) {
+      resetLimitInfoForNextGroup(&pInfo->limitInfo);
+    }
+
     if (p->info.rows > 0) {
-      applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo, pOperator);
-      if (p->info.rows > 0) {
-        break;
-      }
+      break;
     }
   }
 
@@ -698,7 +700,6 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
       colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
     }
 
-    pInfo->limitInfo.numOfOutputRows += p->info.rows;
     pDataBlock->info.rows = p->info.rows;
     pDataBlock->info.id.groupId = pInfo->groupId;
     pDataBlock->info.dataLoad = 1;

From f1d62626303e6dd33bcde6a3005186829db385fc Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Wed, 18 Jan 2023 12:26:01 +0800
Subject: [PATCH 23/40] enh: optimize execute plan when 'tail' function is used
 with 'partition by' clause

---
 source/libs/planner/src/planOptimizer.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index 03e0275d7d..e901297f4d 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -1080,29 +1080,29 @@ static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) {
     return false;
   }
   SSortLogicNode* pSort = (SSortLogicNode*)pNode;
-  if (pSort->groupSort || !sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) {
+  if (!sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) {
     return false;
   }
   return true;
 }
 
-static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool* pNotOptimize,
+static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, bool* pNotOptimize,
                                                    SNodeList** pSequencingNodes) {
   switch (nodeType(pNode)) {
     case QUERY_NODE_LOGIC_PLAN_SCAN: {
       SScanLogicNode* pScan = (SScanLogicNode*)pNode;
-      if (NULL != pScan->pGroupTags || TSDB_SYSTEM_TABLE == pScan->tableType) {
+      if ((!groupSort && NULL != pScan->pGroupTags) || TSDB_SYSTEM_TABLE == pScan->tableType) {
         *pNotOptimize = true;
         return TSDB_CODE_SUCCESS;
       }
       return nodesListMakeAppend(pSequencingNodes, (SNode*)pNode);
     }
     case QUERY_NODE_LOGIC_PLAN_JOIN: {
-      int32_t code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0),
+      int32_t code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), groupSort,
                                                          pNotOptimize, pSequencingNodes);
       if (TSDB_CODE_SUCCESS == code) {
-        code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), pNotOptimize,
-                                                   pSequencingNodes);
+        code = sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 1), groupSort,
+                                                   pNotOptimize, pSequencingNodes);
       }
       return code;
     }
@@ -1121,13 +1121,13 @@ static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool* pNot
     return TSDB_CODE_SUCCESS;
   }
 
-  return sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), pNotOptimize,
-                                             pSequencingNodes);
+  return sortPriKeyOptGetSequencingNodesImpl((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), groupSort,
+                                             pNotOptimize, pSequencingNodes);
 }
 
-static int32_t sortPriKeyOptGetSequencingNodes(SLogicNode* pNode, SNodeList** pSequencingNodes) {
+static int32_t sortPriKeyOptGetSequencingNodes(SLogicNode* pNode, bool groupSort, SNodeList** pSequencingNodes) {
   bool    notOptimize = false;
-  int32_t code = sortPriKeyOptGetSequencingNodesImpl(pNode, &notOptimize, pSequencingNodes);
+  int32_t code = sortPriKeyOptGetSequencingNodesImpl(pNode, groupSort, &notOptimize, pSequencingNodes);
   if (TSDB_CODE_SUCCESS != code || notOptimize) {
     NODES_CLEAR_LIST(*pSequencingNodes);
   }
@@ -1175,8 +1175,8 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS
 
 static int32_t sortPrimaryKeyOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SSortLogicNode* pSort) {
   SNodeList* pSequencingNodes = NULL;
-  int32_t    code =
-      sortPriKeyOptGetSequencingNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0), &pSequencingNodes);
+  int32_t    code = sortPriKeyOptGetSequencingNodes((SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0),
+                                                    pSort->groupSort, &pSequencingNodes);
   if (TSDB_CODE_SUCCESS == code && NULL != pSequencingNodes) {
     code = sortPriKeyOptApply(pCxt, pLogicSubplan, pSort, pSequencingNodes);
   }

From 17bfb8228a25f3be6c684067d2bff3c27cb69c6c Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 18 Jan 2023 15:55:24 +0800
Subject: [PATCH 24/40] fix: rows number exceeds block capacity issue

---
 source/dnode/vnode/inc/vnode.h             |  2 ++
 source/dnode/vnode/src/meta/metaQuery.c    | 27 +++++++++++++++++++++-
 source/libs/executor/src/sysscanoperator.c | 14 ++++++++---
 tests/script/tsim/query/sys_tbname.sim     | 19 +++++++++++++++
 4 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index e5e7fea1cf..a4cbfe60f7 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -153,6 +153,8 @@ typedef struct SMTbCursor SMTbCursor;
 SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
 void        metaCloseTbCursor(SMTbCursor *pTbCur);
 int32_t     metaTbCursorNext(SMTbCursor *pTbCur);
+int32_t     metaTbCursorPrev(SMTbCursor *pTbCur);
+
 #endif
 
 // tsdb
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index cfdb4ab8d1..28bac3e8d0 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -311,7 +311,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) {
   }
 }
 
-int metaTbCursorNext(SMTbCursor *pTbCur) {
+int32_t metaTbCursorNext(SMTbCursor *pTbCur) {
   int    ret;
   void  *pBuf;
   STbCfg tbCfg;
@@ -335,6 +335,31 @@ int metaTbCursorNext(SMTbCursor *pTbCur) {
   return 0;
 }
 
+int32_t metaTbCursorPrev(SMTbCursor *pTbCur) {
+  int    ret;
+  void  *pBuf;
+  STbCfg tbCfg;
+
+  for (;;) {
+    ret = tdbTbcPrev(pTbCur->pDbc, &pTbCur->pKey, &pTbCur->kLen, &pTbCur->pVal, &pTbCur->vLen);
+    if (ret < 0) {
+      return -1;
+    }
+
+    tDecoderClear(&pTbCur->mr.coder);
+
+    metaGetTableEntryByVersion(&pTbCur->mr, ((SUidIdxVal *)pTbCur->pVal)[0].version, *(tb_uid_t *)pTbCur->pKey);
+    if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) {
+      continue;
+    }
+
+    break;
+  }
+
+  return 0;
+}
+
+
 SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) {
   void           *pData = NULL;
   int             nData = 0;
diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c
index 05570eda2f..b4da3ae710 100644
--- a/source/libs/executor/src/sysscanoperator.c
+++ b/source/libs/executor/src/sysscanoperator.c
@@ -491,6 +491,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
     pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta);
   }
 
+  bool blockFull = false;
   while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) {
     if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) {
       continue;
@@ -512,17 +513,24 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
       T_LONG_JMP(pTaskInfo->env, terrno);
     }
 
-    sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock);
-
+    if ((smrSuperTable.me.stbEntry.schemaTag.nCols + numOfRows) > pOperator->resultInfo.capacity) {
+      metaTbCursorPrev(pInfo->pCur);
+      blockFull = true;
+    } else {
+      sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock);
+    }
+    
     metaReaderClear(&smrSuperTable);
 
-    if (numOfRows >= pOperator->resultInfo.capacity) {
+    if (blockFull || numOfRows >= pOperator->resultInfo.capacity) {
       relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo);
       numOfRows = 0;
 
       if (pInfo->pRes->info.rows > 0) {
         break;
       }
+
+      blockFull = false;
     }
   }
 
diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim
index 045e908a57..4587dcd4f7 100644
--- a/tests/script/tsim/query/sys_tbname.sim
+++ b/tests/script/tsim/query/sys_tbname.sim
@@ -86,4 +86,23 @@ if $data00 != @ins_tags@ then
 	return -1
 endi
 
+sql create stable stb(ts timestamp, f int) tags(t1 int, t2 int, t3 int, t4 int, t5 int);
+
+$i = 0
+$tbNum = 1000
+$tbPrefix = stb_tb
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+  sql create table $tb using stb tags( $i , $i , $i , $i , $i )
+
+  $i = $i + 1
+endw
+
+sql select tag_value from information_schema.ins_tags where stable_name='stb';
+if $rows != 5000 then
+   print $rows
+        return -1
+endi
+
+
 #system sh/exec.sh -n dnode1 -s stop -x SIGINT

From d130d1a84af664b1620049ffe5415e328a2ade79 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 18 Jan 2023 17:13:35 +0800
Subject: [PATCH 25/40] fix: atexit memory leak issue

---
 source/libs/executor/src/executor.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 814ead57f0..9f5f7eca98 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -24,7 +24,10 @@
 static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT;
 int32_t             exchangeObjRefPool = -1;
 
-static void initRefPool() { exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo); }
+static void initRefPool() { 
+  exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo);   
+  atexit(cleanupRefPool);
+}
 static void cleanupRefPool() {
   int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0);
   taosCloseRef(ref);
@@ -442,7 +445,6 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
   SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo;
 
   taosThreadOnce(&initPoolOnce, initRefPool);
-  atexit(cleanupRefPool);
 
   qDebug("start to create subplan task, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId);
 

From 2d579cfc16d60d35764007a832bca566557c698c Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 18 Jan 2023 17:27:27 +0800
Subject: [PATCH 26/40] fix: fix compile issue

---
 source/libs/executor/src/executor.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 9f5f7eca98..6c354c3d61 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -24,15 +24,16 @@
 static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT;
 int32_t             exchangeObjRefPool = -1;
 
-static void initRefPool() { 
-  exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo);   
-  atexit(cleanupRefPool);
-}
 static void cleanupRefPool() {
   int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0);
   taosCloseRef(ref);
 }
 
+static void initRefPool() { 
+  exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo);   
+  atexit(cleanupRefPool);
+}
+
 static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) {
   ASSERT(pOperator != NULL);
   if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

From 130036269e608e0410e4d666eb4601448b5bb631 Mon Sep 17 00:00:00 2001
From: xiaolei li <85657333+xleili@users.noreply.github.com>
Date: Wed, 18 Jan 2023 21:23:41 +0800
Subject: [PATCH 27/40] release: update main version to 3.0.2.4 (#19635)

---
 cmake/cmake.version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/cmake.version b/cmake/cmake.version
index ba85a3d99b..a4c783b6c8 100644
--- a/cmake/cmake.version
+++ b/cmake/cmake.version
@@ -2,7 +2,7 @@
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "3.0.2.2")
+  SET(TD_VER_NUMBER "3.0.2.4")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

From 0251750c895a24117dc8a1652277b38bb9468235 Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Fri, 20 Jan 2023 16:45:09 +0800
Subject: [PATCH 28/40] fix: packaging/tools/install.sh for main (#19649)

---
 packaging/tools/install.sh | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh
index 2a078b5eab..4be179b04d 100755
--- a/packaging/tools/install.sh
+++ b/packaging/tools/install.sh
@@ -743,6 +743,34 @@ function is_version_compatible() {
   esac
 }
 
+deb_erase() {
+  confirm=""
+  while [ "" == "${confirm}" ]; do
+    echo -e -n "${RED}Exist tdengine deb detected, do you want to remove it? [yes|no] ${NC}:"
+    read confirm
+    if [ "yes" == "$confirm" ]; then
+      ${csudo}dpkg --remove tdengine ||:
+      break
+    elif [ "no" == "$confirm" ]; then
+      break
+    fi
+  done
+}
+
+rpm_erase() {
+  confirm=""
+  while [ "" == "${confirm}" ]; do
+    echo -e -n "${RED}Exist tdengine rpm detected, do you want to remove it? [yes|no] ${NC}:"
+    read confirm
+    if [ "yes" == "$confirm" ]; then
+      ${csudo}rpm -e tdengine ||:
+      break
+    elif [ "no" == "$confirm" ]; then
+      break
+    fi
+  done
+}
+
 function updateProduct() {
   # Check if version compatible
   if ! is_version_compatible; then
@@ -755,6 +783,13 @@ function updateProduct() {
     echo "File ${tarName} does not exist"
     exit 1
   fi
+
+  if echo $osinfo | grep -qwi "centos"; then
+    rpm -q tdengine 2>&1 > /dev/null && rpm_erase tdengine ||:
+  elif echo $osinfo | grep -qwi "ubuntu"; then
+    dpkg -l tdengine 2>&1 > /dev/null && deb_erase tdengine ||:
+  fi
+
   tar -zxf ${tarName}
   install_jemalloc
 

From 490cbaf55b8be37e0395317869244ef5211bc86c Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Fri, 20 Jan 2023 22:09:37 +0800
Subject: [PATCH 29/40] fix: reenable -e in release.sh for main (#19650)

* fix: taos-tools deb/rpm compn for main

* fix: update taos-tools 5aa25e9

* fix: reenable -e in release.sh
---
 packaging/release.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/release.sh b/packaging/release.sh
index e442e07c6a..1dfbf2b112 100755
--- a/packaging/release.sh
+++ b/packaging/release.sh
@@ -2,7 +2,7 @@
 #
 # Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os
 
-# set -e
+set -e
 # set -x
 
 # release.sh  -v [cluster | edge]

From 39b69ba39e8f25c3cafd367f0f08f12580413d2b Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Sat, 21 Jan 2023 13:44:25 +0800
Subject: [PATCH 30/40] fix: row schema version for row merger

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index ee93756f0e..85282a2340 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -1758,11 +1758,14 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
     }
 
     if (minKey == k.ts) {
+      STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
+      if (pSchema == NULL) {
+        return terrno;
+      }
       if (init) {
-        tRowMerge(&merge, pRow);
+        tRowMergerAdd(&merge, pRow, pSchema);
       } else {
         init = true;
-        STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
         int32_t   code = tRowMergerInit(&merge, pRow, pSchema);
         if (code != TSDB_CODE_SUCCESS) {
           return code;

From 2416d34a80068e721fdf25c349ec020c7d4e600f Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Sat, 21 Jan 2023 16:07:27 +0800
Subject: [PATCH 31/40] fix: memory leak for row merge

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 49 +++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 86adc1dc80..030e482bf1 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -722,7 +722,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
     pTColumn = &pMerger->pTSchema->columns[iCol];
     if (pTSchema->columns[jCol].colId < pTColumn->colId) {
       ++jCol;
-      --iCol;
+      // --iCol;
       continue;
     } else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
       continue;
@@ -731,6 +731,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
     tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal);
 
     if (key.version > pMerger->version) {
+#if 0
       if (!COL_VAL_IS_NONE(pColVal)) {
         if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
           SColVal *tColVal = taosArrayGet(pMerger->pArray, iCol);
@@ -743,10 +744,34 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
           }
           tColVal->flag = 0;
         } else {
+
+          taosArraySet(pMerger->pArray, iCol, pColVal);
+        }
+      }
+#endif
+      if (!COL_VAL_IS_NONE(pColVal)) {
+        if (IS_VAR_DATA_TYPE(pColVal->type)) {
+          SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol);
+          if (!COL_VAL_IS_NULL(pColVal)) {
+            code = tRealloc(&pTColVal->value.pData, pColVal->value.nData);
+            if (code) return code;
+
+            pTColVal->value.nData = pColVal->value.nData;
+            if (pTColVal->value.nData) {
+              memcpy(pTColVal->value.pData,  pColVal->value.pData, pTColVal->value.nData);
+            }
+            pTColVal->flag = 0;
+          } else {
+            tFree(pTColVal->value.pData);
+            pTColVal->value.pData = NULL;
+            taosArraySet(pMerger->pArray, iCol, pColVal);
+          }
+        } else {
           taosArraySet(pMerger->pArray, iCol, pColVal);
         }
       }
     } else if (key.version < pMerger->version) {
+#if 0
       SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
       if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
         if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
@@ -762,6 +787,28 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
           taosArraySet(pMerger->pArray, iCol, pColVal);
         }
       }
+#endif
+      SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
+      if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
+        if (IS_VAR_DATA_TYPE(pColVal->type)) {
+          if (!COL_VAL_IS_NULL(pColVal)) {
+            code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
+            if (code) return code;
+
+            tColVal->value.nData = pColVal->value.nData;
+            if (tColVal->value.nData) {
+              memcpy(tColVal->value.pData,  pColVal->value.pData, tColVal->value.nData);
+            }
+            tColVal->flag = 0;
+          } else {
+            tFree(tColVal->value.pData);
+            tColVal->value.pData = NULL;
+            taosArraySet(pMerger->pArray, iCol, pColVal);
+          }
+        } else {
+          taosArraySet(pMerger->pArray, iCol, pColVal);
+        }
+      }
     } else {
       ASSERT(0 && "dup versions not allowed");
     }

From 338a2a74cb48f6a6d266b52c9702fc3160730422 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Sat, 21 Jan 2023 16:08:33 +0800
Subject: [PATCH 32/40] chore: revert the code

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 030e482bf1..8da0b268fb 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -722,7 +722,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
     pTColumn = &pMerger->pTSchema->columns[iCol];
     if (pTSchema->columns[jCol].colId < pTColumn->colId) {
       ++jCol;
-      // --iCol;
+      --iCol;
       continue;
     } else if (pTSchema->columns[jCol].colId > pTColumn->colId) {
       continue;

From 08a5ed4a076d74bcb5ca7ec0110b6bfe7b24824a Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Sat, 21 Jan 2023 16:11:33 +0800
Subject: [PATCH 33/40] chore: revert the code

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 8da0b268fb..6cc8540d38 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -744,7 +744,6 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
           }
           tColVal->flag = 0;
         } else {
-
           taosArraySet(pMerger->pArray, iCol, pColVal);
         }
       }

From 1e06fb17836c7f8f5f030a1c8ede8ad654032e40 Mon Sep 17 00:00:00 2001
From: kailixu <klxu@taosdata.com>
Date: Sat, 21 Jan 2023 16:15:12 +0800
Subject: [PATCH 34/40] chore: code optimization

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 6cc8540d38..ab10960970 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -770,7 +770,6 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
         }
       }
     } else if (key.version < pMerger->version) {
-#if 0
       SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
       if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
         if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) {
@@ -786,28 +785,6 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
           taosArraySet(pMerger->pArray, iCol, pColVal);
         }
       }
-#endif
-      SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol);
-      if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) {
-        if (IS_VAR_DATA_TYPE(pColVal->type)) {
-          if (!COL_VAL_IS_NULL(pColVal)) {
-            code = tRealloc(&tColVal->value.pData, pColVal->value.nData);
-            if (code) return code;
-
-            tColVal->value.nData = pColVal->value.nData;
-            if (tColVal->value.nData) {
-              memcpy(tColVal->value.pData,  pColVal->value.pData, tColVal->value.nData);
-            }
-            tColVal->flag = 0;
-          } else {
-            tFree(tColVal->value.pData);
-            tColVal->value.pData = NULL;
-            taosArraySet(pMerger->pArray, iCol, pColVal);
-          }
-        } else {
-          taosArraySet(pMerger->pArray, iCol, pColVal);
-        }
-      }
     } else {
       ASSERT(0 && "dup versions not allowed");
     }

From 21d57a36240c506b47313911375b4e82c906c354 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 22 Jan 2023 01:45:29 +0800
Subject: [PATCH 35/40] fix(query): check for failure during add new buf pages.

---
 source/libs/executor/src/groupoperator.c |  10 ++-
 source/util/src/tpagedbuf.c              | 104 +++++++++++------------
 2 files changed, 60 insertions(+), 54 deletions(-)

diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c
index 5676e19cdf..bf4b9a2599 100644
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@@ -593,8 +593,11 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf
 
     int32_t pageId = 0;
     pPage = getNewBufPage(pInfo->pBuf, &pageId);
-    taosArrayPush(p->pPageList, &pageId);
+    if (pPage == NULL) {
+      return pPage;
+    }
 
+    taosArrayPush(p->pPageList, &pageId);
     *(int32_t*)pPage = 0;
   } else {
     int32_t* curId = taosArrayGetLast(p->pPageList);
@@ -612,6 +615,11 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf
       // add a new page for current group
       int32_t pageId = 0;
       pPage = getNewBufPage(pInfo->pBuf, &pageId);
+      if (pPage == NULL) {
+        qError("failed to get new buffer, code:%s", tstrerror(terrno));
+        return NULL;
+      }
+
       taosArrayPush(p->pPageList, &pageId);
       memset(pPage, 0, getBufPageSize(pInfo->pBuf));
     }
diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c
index 87b44b2d13..f696a02e6e 100644
--- a/source/util/src/tpagedbuf.c
+++ b/source/util/src/tpagedbuf.c
@@ -222,7 +222,6 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
   char* p = doFlushPageToDisk(pBuf, pg);
   setPageNotInBuf(pg);
   pg->dirty = false;
-
   return p;
 }
 
@@ -286,32 +285,21 @@ static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) {
 }
 
 static char* evacOneDataPage(SDiskbasedBuf* pBuf) {
-  char*      bufPage = NULL;
   SListNode* pn = getEldestUnrefedPage(pBuf);
-  terrno = 0;
-
-  // all pages are referenced by user, try to allocate new space
-  if (pn == NULL) {
-    int32_t prev = pBuf->inMemPages;
-
-    // increase by 50% of previous mem pages
-    pBuf->inMemPages = (int32_t)(pBuf->inMemPages * 1.5f);
-
-    //    qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pBuf, prev,
-    //          pBuf->inMemPages, pBuf->pageSize);
-  } else {
-    tdListPopNode(pBuf->lruList, pn);
-
-    SPageInfo* d = *(SPageInfo**)pn->data;
-    ASSERTS(d->pn == pn, "d->pn not equal pn");
-
-    d->pn = NULL;
-    taosMemoryFreeClear(pn);
-
-    bufPage = flushPageToDisk(pBuf, d);
+  if (pn == NULL) {  // no available buffer pages now, return.
+    return NULL;
   }
 
-  return bufPage;
+  terrno = 0;
+  tdListPopNode(pBuf->lruList, pn);
+
+  SPageInfo* d = *(SPageInfo**)pn->data;
+  ASSERTS(d->pn == pn, "d->pn not equal pn");
+
+  d->pn = NULL;
+  taosMemoryFreeClear(pn);
+
+  return flushPageToDisk(pBuf, d);
 }
 
 static void lruListPushFront(SList* pList, SPageInfo* pi) {
@@ -338,7 +326,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem
 
   SDiskbasedBuf* pPBuf = *pBuf;
   if (pPBuf == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
+    goto _error;
   }
 
   pPBuf->pageSize = pagesize;
@@ -362,24 +350,50 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem
   pPBuf->pIdList = taosArrayInit(4, POINTER_BYTES);
 
   pPBuf->assistBuf = taosMemoryMalloc(pPBuf->pageSize + 2);  // EXTRA BYTES
-  pPBuf->all = taosHashInit(10, fn, true, false);
-  pPBuf->prefix = (char*) dir;
+  if (pPBuf->assistBuf == NULL) {
+    goto _error;
+  }
 
+  pPBuf->all = taosHashInit(10, fn, true, false);
+  if (pPBuf->all == NULL) {
+    goto _error;
+  }
+
+  pPBuf->prefix = (char*) dir;
   pPBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
 
   //  qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId,
-  //  pPBuf->pageSize,
-  //         pPBuf->inMemPages, pPBuf->path);
+  //  pPBuf->pageSize, pPBuf->inMemPages, pPBuf->path);
 
   return TSDB_CODE_SUCCESS;
+  _error:
+  destroyDiskbasedBuf(pPBuf);
+  return TSDB_CODE_OUT_OF_MEMORY;
+}
+
+static char* doExtractPage(SDiskbasedBuf* pBuf) {
+  char* availablePage = NULL;
+  if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) {
+    availablePage = evacOneDataPage(pBuf);
+    if (availablePage == NULL) {
+      uWarn("no available buf pages, current:%d, max:%d", listNEles(pBuf->lruList), pBuf->inMemPages)
+    }
+  } else {
+    availablePage = taosMemoryCalloc(1, getAllocPageSize(pBuf->pageSize));  // add extract bytes in case of zipped buffer increased.
+    if (availablePage == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+
+  return availablePage;
 }
 
 void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) {
   pBuf->statis.getPages += 1;
 
-  char* availablePage = NULL;
-  if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) {
-    availablePage = evacOneDataPage(pBuf);
+  char* availablePage = doExtractPage(pBuf);
+  if (availablePage == NULL) {
+    return NULL;
   }
 
   SPageInfo* pi = NULL;
@@ -402,16 +416,8 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) {
   }
 
   // add to LRU list
-  ASSERT(listNEles(pBuf->lruList) < pBuf->inMemPages && pBuf->inMemPages > 0);
   lruListPushFront(pBuf->lruList, pi);
-
-  // allocate buf
-  if (availablePage == NULL) {
-    pi->pData =
-        taosMemoryCalloc(1, getAllocPageSize(pBuf->pageSize));  // add extract bytes in case of zipped buffer increased.
-  } else {
-    pi->pData = availablePage;
-  }
+  pi->pData = availablePage;
 
   ((void**)pi->pData)[0] = pi;
 #ifdef BUF_PAGE_DEBUG
@@ -447,18 +453,9 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
     ASSERT((*pi)->pData == NULL && (*pi)->pn == NULL &&
            (((*pi)->length >= 0 && (*pi)->offset >= 0) || ((*pi)->length == -1 && (*pi)->offset == -1)));
 
-    char* availablePage = NULL;
-    if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) {
-      availablePage = evacOneDataPage(pBuf);
-      if (availablePage == NULL) {
-        return NULL;
-      }
-    }
-
-    if (availablePage == NULL) {
-      (*pi)->pData = taosMemoryCalloc(1, getAllocPageSize(pBuf->pageSize));
-    } else {
-      (*pi)->pData = availablePage;
+    (*pi)->pData = doExtractPage(pBuf);
+    if ((*pi)->pData == NULL) {
+      return NULL;
     }
 
     // set the ptr to the new SPageInfo
@@ -471,6 +468,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
     if ((*pi)->length > 0 && (*pi)->offset >= 0) {
       int32_t code = loadPageFromDisk(pBuf, *pi);
       if (code != 0) {
+        terrno = code;
         return NULL;
       }
     }

From 0abd58a95e6cdaf343dd92402eb429fc3e55937d Mon Sep 17 00:00:00 2001
From: Shuduo Sang <sangshuduo@gmail.com>
Date: Sun, 22 Jan 2023 20:33:39 +0800
Subject: [PATCH 36/40] feat: taosbenchmark specfying vgroups for main (#19660)

---
 cmake/taostools_CMakeLists.txt.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index 1e2247eedc..a69e1578c0 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 5c53cc8
+        GIT_TAG 7d24ed5
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

From c1ffbb5c1ccdd0143196c4cffa227a88745a55db Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 23 Jan 2023 01:11:04 +0800
Subject: [PATCH 37/40] refactor:do some internal refactor.

---
 source/libs/function/src/builtinsimpl.c |  4 --
 source/libs/function/src/tpercentile.c  | 13 +++---
 source/util/src/tpagedbuf.c             | 60 +++++++++++++++----------
 3 files changed, 43 insertions(+), 34 deletions(-)

diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 324011f238..e4081ddf0d 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -3061,14 +3061,12 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf,
     if (pHandle->currentPage == -1) {
       pPage = getNewBufPage(pHandle->pBuf, &pHandle->currentPage);
       if (pPage == NULL) {
-        terrno = TSDB_CODE_NO_AVAIL_DISK;
         return terrno;
       }
       pPage->num = sizeof(SFilePage);
     } else {
       pPage = getBufPage(pHandle->pBuf, pHandle->currentPage);
       if (pPage == NULL) {
-        terrno = TSDB_CODE_NO_AVAIL_DISK;
         return terrno;
       }
       if (pPage->num + length > getBufPageSize(pHandle->pBuf)) {
@@ -3076,7 +3074,6 @@ static int32_t doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf,
         releaseBufPage(pHandle->pBuf, pPage);
         pPage = getNewBufPage(pHandle->pBuf, &pHandle->currentPage);
         if (pPage == NULL) {
-          terrno = TSDB_CODE_NO_AVAIL_DISK;
           return terrno;
         }
         pPage->num = sizeof(SFilePage);
@@ -3123,7 +3120,6 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf
   if (pHandle->pBuf != NULL) {
     SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId);
     if (pPage == NULL) {
-      terrno = TSDB_CODE_NO_AVAIL_DISK;
       return terrno;
     }
     memcpy(pPage->data + pPos->offset, pBuf, length);
diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c
index 871b668cfd..6b85155486 100644
--- a/source/libs/function/src/tpercentile.c
+++ b/source/libs/function/src/tpercentile.c
@@ -43,8 +43,8 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx)
     if (pg == NULL) {
       return NULL;
     }
-    memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes));
 
+    memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes));
     offset += (int32_t)(pg->num * pMemBucket->bytes);
   }
 
@@ -109,7 +109,7 @@ int32_t findOnlyResult(tMemBucket *pMemBucket, double *result) {
       int32_t   *pageId = taosArrayGet(list, 0);
       SFilePage *pPage = getBufPage(pMemBucket->pBuffer, *pageId);
       if (pPage == NULL) {
-        return TSDB_CODE_NO_AVAIL_DISK;
+        return terrno;
       }
       ASSERT(pPage->num == 1);
 
@@ -276,7 +276,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval,
     return NULL;
   }
 
-  int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", tsTempDir);
+  int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 1024, "1", tsTempDir);
   if (ret != 0) {
     tMemBucketDestroy(pBucket);
     return NULL;
@@ -388,7 +388,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
 
       pSlot->info.data = getNewBufPage(pBucket->pBuffer, &pageId);
       if (pSlot->info.data == NULL) {
-        return TSDB_CODE_NO_AVAIL_DISK;
+        return terrno;
       }
       pSlot->info.pageId = pageId;
       taosArrayPush(pPageIdList, &pageId);
@@ -482,8 +482,9 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction
         // data in buffer and file are merged together to be processed.
         SFilePage *buffer = loadDataFromFilePage(pMemBucket, i);
         if (buffer == NULL) {
-          return TSDB_CODE_NO_AVAIL_DISK;
+          return terrno;
         }
+
         int32_t    currentIdx = count - num;
 
         char *thisVal = buffer->data + pMemBucket->bytes * currentIdx;
@@ -520,7 +521,7 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction
           int32_t *pageId = taosArrayGet(list, f);
           SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId);
           if (pg == NULL) {
-            return TSDB_CODE_NO_AVAIL_DISK;
+            return terrno;
           }
 
           int32_t code = tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num);
diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c
index f696a02e6e..99f555e20d 100644
--- a/source/util/src/tpagedbuf.c
+++ b/source/util/src/tpagedbuf.c
@@ -5,7 +5,7 @@
 #include "thash.h"
 #include "tlog.h"
 
-#define GET_DATA_PAYLOAD(_p)          ((char*)(_p)->pData + POINTER_BYTES)
+#define GET_PAYLOAD_DATA(_p)          ((char*)(_p)->pData + POINTER_BYTES)
 #define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
 
 typedef struct SPageDiskInfo {
@@ -89,7 +89,7 @@ static char* doDecompressData(void* data, int32_t srcSize, int32_t* dst, SDiskba
   return data;
 }
 
-static uint64_t allocatePositionInFile(SDiskbasedBuf* pBuf, size_t size) {
+static uint64_t allocateNewPositionInFile(SDiskbasedBuf* pBuf, size_t size) {
   if (pBuf->pFree == NULL) {
     return pBuf->nextPos;
   } else {
@@ -114,8 +114,6 @@ static uint64_t allocatePositionInFile(SDiskbasedBuf* pBuf, size_t size) {
 
 static void setPageNotInBuf(SPageInfo* pPageInfo) { pPageInfo->pData = NULL; }
 
-static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { return pageSize + POINTER_BYTES + sizeof(SFilePage); }
-
 /**
  *   +--------------------------+-------------------+--------------+
  *   | PTR to SPageInfo (8bytes)| Payload (PageSize)| 2 Extra Bytes|
@@ -124,13 +122,16 @@ static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { return pageSize
  * @param pg
  * @return
  */
-static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
+
+static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { return pageSize + POINTER_BYTES + sizeof(SFilePage); }
+
+static char* doFlushBufPage(SDiskbasedBuf* pBuf, SPageInfo* pg) {
   ASSERT(!pg->used && pg->pData != NULL);
 
   int32_t size = pBuf->pageSize;
   char*   t = NULL;
   if (pg->offset == -1 || pg->dirty) {
-    void* payload = GET_DATA_PAYLOAD(pg);
+    void* payload = GET_PAYLOAD_DATA(pg);
     t = doCompressData(payload, pBuf->pageSize, &size, pBuf);
     ASSERTS(size >= 0, "size is negative");
   }
@@ -140,7 +141,7 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
     if (pg->offset == -1) {
       ASSERTS(pg->dirty == true, "pg->dirty is false");
 
-      pg->offset = allocatePositionInFile(pBuf, size);
+      pg->offset = allocateNewPositionInFile(pBuf, size);
       pBuf->nextPos += size;
 
       int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET);
@@ -169,7 +170,7 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
         taosArrayPush(pBuf->pFree, &dinfo);
 
         // 2. allocate new position, and update the info
-        pg->offset = allocatePositionInFile(pBuf, size);
+        pg->offset = allocateNewPositionInFile(pBuf, size);
         pBuf->nextPos += size;
       }
 
@@ -208,9 +209,8 @@ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
   return pDataBuf;
 }
 
-static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
+static char* flushBufPage(SDiskbasedBuf* pBuf, SPageInfo* pg) {
   int32_t ret = TSDB_CODE_SUCCESS;
-  ASSERT(((int64_t)pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages);
 
   if (pBuf->pFile == NULL) {
     if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) {
@@ -219,7 +219,7 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
     }
   }
 
-  char* p = doFlushPageToDisk(pBuf, pg);
+  char* p = doFlushBufPage(pBuf, pg);
   setPageNotInBuf(pg);
   pg->dirty = false;
   return p;
@@ -233,7 +233,7 @@ static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
     return ret;
   }
 
-  void* pPage = (void*)GET_DATA_PAYLOAD(pg);
+  void* pPage = (void*)GET_PAYLOAD_DATA(pg);
   ret = (int32_t)taosReadFile(pBuf->pFile, pPage, pg->length);
   if (ret != pg->length) {
     ret = TAOS_SYSTEM_ERROR(errno);
@@ -252,6 +252,10 @@ static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t pageId) {
   pBuf->numOfPages += 1;
 
   SPageInfo* ppi = taosMemoryMalloc(sizeof(SPageInfo));
+  if (ppi == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
 
   ppi->pageId = pageId;
   ppi->pData = NULL;
@@ -274,17 +278,14 @@ static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) {
     ASSERT(pageInfo->pageId >= 0 && pageInfo->pn == pn);
 
     if (!pageInfo->used) {
-      //      printf("%d is chosen\n", pageInfo->pageId);
       break;
-    } else {
-      //      printf("page %d is used, dirty:%d\n", pageInfo->pageId, pageInfo->dirty);
     }
   }
 
   return pn;
 }
 
-static char* evacOneDataPage(SDiskbasedBuf* pBuf) {
+static char* evictBufPage(SDiskbasedBuf* pBuf) {
   SListNode* pn = getEldestUnrefedPage(pBuf);
   if (pn == NULL) {  // no available buffer pages now, return.
     return NULL;
@@ -299,7 +300,7 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) {
   d->pn = NULL;
   taosMemoryFreeClear(pn);
 
-  return flushPageToDisk(pBuf, d);
+  return flushBufPage(pBuf, d);
 }
 
 static void lruListPushFront(SList* pList, SPageInfo* pi) {
@@ -374,8 +375,9 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem
 static char* doExtractPage(SDiskbasedBuf* pBuf) {
   char* availablePage = NULL;
   if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) {
-    availablePage = evacOneDataPage(pBuf);
+    availablePage = evictBufPage(pBuf);
     if (availablePage == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
       uWarn("no available buf pages, current:%d, max:%d", listNEles(pBuf->lruList), pBuf->inMemPages)
     }
   } else {
@@ -409,6 +411,9 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) {
 
     // register page id info
     pi = registerPage(pBuf, *pageId);
+    if (pi == NULL) {
+      return NULL;
+    }
 
     // add to hash map
     taosHashPut(pBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
@@ -423,11 +428,15 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) {
 #ifdef BUF_PAGE_DEBUG
   uDebug("page_getNewBufPage , pi->pData:%p, pageId:%d, offset:%" PRId64, pi->pData, pi->pageId, pi->offset);
 #endif
-  return (void*)(GET_DATA_PAYLOAD(pi));
+
+  return (void*)(GET_PAYLOAD_DATA(pi));
 }
 
 void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
-  ASSERT(pBuf != NULL && id >= 0);
+  if (id < 0) {
+    return NULL;
+  }
+
   pBuf->statis.getPages += 1;
 
   SPageInfo** pi = taosHashGet(pBuf->all, &id, sizeof(int32_t));
@@ -437,7 +446,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
     // no need to update the LRU list if only one page exists
     if (pBuf->numOfPages == 1) {
       (*pi)->used = true;
-      return (void*)(GET_DATA_PAYLOAD(*pi));
+      return (void*)(GET_PAYLOAD_DATA(*pi));
     }
 
     SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data);
@@ -445,10 +454,11 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
 
     lruListMoveToFront(pBuf->lruList, (*pi));
     (*pi)->used = true;
+
 #ifdef BUF_PAGE_DEBUG
     uDebug("page_getBufPage1 pageId:%d, offset:%" PRId64, (*pi)->pageId, (*pi)->offset);
 #endif
-    return (void*)(GET_DATA_PAYLOAD(*pi));
+    return (void*)(GET_PAYLOAD_DATA(*pi));
   } else {  // not in memory
     ASSERT((*pi)->pData == NULL && (*pi)->pn == NULL &&
            (((*pi)->length >= 0 && (*pi)->offset >= 0) || ((*pi)->length == -1 && (*pi)->offset == -1)));
@@ -475,14 +485,15 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
 #ifdef BUF_PAGE_DEBUG
     uDebug("page_getBufPage2 pageId:%d, offset:%" PRId64, (*pi)->pageId, (*pi)->offset);
 #endif
-    return (void*)(GET_DATA_PAYLOAD(*pi));
+    return (void*)(GET_PAYLOAD_DATA(*pi));
   }
 }
 
 void releaseBufPage(SDiskbasedBuf* pBuf, void* page) {
-  if (ASSERTS(pBuf != NULL && page != NULL, "pBuf or page is NULL")) {
+  if (page == NULL) {
     return;
   }
+
   SPageInfo* ppi = getPageInfoFromPayload(page);
   releaseBufPageInfo(pBuf, ppi);
 }
@@ -491,6 +502,7 @@ void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) {
 #ifdef BUF_PAGE_DEBUG
   uDebug("page_releaseBufPageInfo pageId:%d, used:%d, offset:%" PRId64, pi->pageId, pi->used, pi->offset);
 #endif
+
   if (ASSERTS(pi->pData != NULL, "pi->pData is NULL")) {
     return;
   }

From 670eec4db5a572f949252121fd1b05acf7b4a83f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 24 Jan 2023 22:54:09 +0800
Subject: [PATCH 38/40] refactor: remove assert

---
 source/util/src/tarray.c    | 42 ++++++++++++++++++-------------------
 source/util/src/tpagedbuf.c | 27 ++++++++++++++++++------
 2 files changed, 42 insertions(+), 27 deletions(-)

diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index 4bd8294423..237edfdeb2 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -20,7 +20,10 @@
 // todo refactor API
 
 SArray* taosArrayInit(size_t size, size_t elemSize) {
-  assert(elemSize > 0);
+  if (elemSize == 0) {
+    terrno = TSDB_CODE_INVALID_PARA;
+    return NULL;
+  }
 
   if (size < TARRAY_MIN_SIZE) {
     size = TARRAY_MIN_SIZE;
@@ -96,8 +99,6 @@ void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles) {
 }
 
 void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)) {
-  assert(pArray);
-
   size_t size = pArray->size;
   if (size <= 1) {
     return;
@@ -136,8 +137,6 @@ void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)
 }
 
 void taosArrayRemoveDuplicateP(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)) {
-  assert(pArray);
-
   size_t size = pArray->size;
   if (size <= 1) {
     return;
@@ -195,11 +194,10 @@ void* taosArrayReserve(SArray* pArray, int32_t num) {
 }
 
 void* taosArrayPop(SArray* pArray) {
-  assert(pArray != NULL);
-
   if (pArray->size == 0) {
     return NULL;
   }
+
   pArray->size -= 1;
   return TARRAY_GET_ELEM(pArray, pArray->size);
 }
@@ -208,16 +206,21 @@ void* taosArrayGet(const SArray* pArray, size_t index) {
   if (NULL == pArray) {
     return NULL;
   }
-  assert(index < pArray->size);
+
+  if (index >= pArray->size) {
+    uError("index is out of range, current:%"PRIzu" max:%d", index, pArray->capacity);
+    return NULL;
+  }
+
   return TARRAY_GET_ELEM(pArray, index);
 }
 
 void* taosArrayGetP(const SArray* pArray, size_t index) {
-  assert(index < pArray->size);
-
-  void* d = TARRAY_GET_ELEM(pArray, index);
-
-  return *(void**)d;
+  void** p = taosArrayGet(pArray, index);
+  if (p == NULL) {
+    return NULL;
+  }
+  return *p;
 }
 
 void* taosArrayGetLast(const SArray* pArray) { return TARRAY_GET_ELEM(pArray, pArray->size - 1); }
@@ -296,9 +299,12 @@ void taosArrayRemove(SArray* pArray, size_t index) {
 }
 
 SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) {
-  assert(src != NULL && elemSize > 0);
-  SArray* pDst = taosArrayInit(size, elemSize);
+  if (elemSize <= 0) {
+    terrno = TSDB_CODE_INVALID_PARA;
+    return NULL;
+  }
 
+  SArray* pDst = taosArrayInit(size, elemSize);
   memcpy(pDst->pData, src, elemSize * size);
   pDst->size = size;
 
@@ -306,8 +312,6 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) {
 }
 
 SArray* taosArrayDup(const SArray* pSrc, __array_item_dup_fn_t fn) {
-  assert(pSrc != NULL);
-
   if (pSrc->size == 0) {  // empty array list
     return taosArrayInit(8, pSrc->elemSize);
   }
@@ -399,14 +403,10 @@ void taosArrayDestroyEx(SArray* pArray, FDelete fp) {
 }
 
 void taosArraySort(SArray* pArray, __compar_fn_t compar) {
-  ASSERT(pArray != NULL && compar != NULL);
   taosSort(pArray->pData, pArray->size, pArray->elemSize, compar);
 }
 
 void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int32_t flags) {
-  assert(pArray != NULL && comparFn != NULL);
-  assert(key != NULL);
-
   return taosbsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags);
 }
 
diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c
index 99f555e20d..d4103ab578 100644
--- a/source/util/src/tpagedbuf.c
+++ b/source/util/src/tpagedbuf.c
@@ -126,7 +126,11 @@ static void setPageNotInBuf(SPageInfo* pPageInfo) { pPageInfo->pData = NULL; }
 static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { return pageSize + POINTER_BYTES + sizeof(SFilePage); }
 
 static char* doFlushBufPage(SDiskbasedBuf* pBuf, SPageInfo* pg) {
-  ASSERT(!pg->used && pg->pData != NULL);
+  if (pg->pData == NULL || pg->used) {
+    uError("invalid params in paged buffer process when flushing buf to disk, %s", pBuf->id);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return NULL;
+  }
 
   int32_t size = pBuf->pageSize;
   char*   t = NULL;
@@ -333,7 +337,6 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem
   pPBuf->pageSize = pagesize;
   pPBuf->numOfPages = 0;  // all pages are in buffer in the first place
   pPBuf->totalBufSize = 0;
-  pPBuf->inMemPages = inMemBufSize / pagesize;  // maximum allowed pages, it is a soft limit.
   pPBuf->allocateId = -1;
   pPBuf->pFile = NULL;
   pPBuf->id = strdup(id);
@@ -342,13 +345,22 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem
   pPBuf->freePgList = tdListNew(POINTER_BYTES);
 
   // at least more than 2 pages must be in memory
-  ASSERT(inMemBufSize >= pagesize * 2);
+  if (inMemBufSize < pagesize * 2) {
+    inMemBufSize = pagesize * 2;
+  }
 
+  pPBuf->inMemPages = inMemBufSize / pagesize;  // maximum allowed pages, it is a soft limit.
   pPBuf->lruList = tdListNew(POINTER_BYTES);
+  if (pPBuf->lruList == NULL) {
+    goto _error;
+  }
 
   // init id hash table
   _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT);
   pPBuf->pIdList = taosArrayInit(4, POINTER_BYTES);
+  if (pPBuf->pIdList == NULL) {
+    goto _error;
+  }
 
   pPBuf->assistBuf = taosMemoryMalloc(pPBuf->pageSize + 2);  // EXTRA BYTES
   if (pPBuf->assistBuf == NULL) {
@@ -503,7 +515,12 @@ void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) {
   uDebug("page_releaseBufPageInfo pageId:%d, used:%d, offset:%" PRId64, pi->pageId, pi->used, pi->offset);
 #endif
 
-  if (ASSERTS(pi->pData != NULL, "pi->pData is NULL")) {
+  if (pi == NULL) {
+    return;
+  }
+
+  if (pi->pData == NULL) {
+    uError("pi->pData (page data) is null");
     return;
   }
 
@@ -514,7 +531,6 @@ void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) {
 size_t getTotalBufSize(const SDiskbasedBuf* pBuf) { return (size_t)pBuf->totalBufSize; }
 
 SArray* getDataBufPagesIdList(SDiskbasedBuf* pBuf) {
-  ASSERT(pBuf != NULL);
   return pBuf->pIdList;
 }
 
@@ -592,7 +608,6 @@ SPageInfo* getLastPageInfo(SArray* pList) {
 }
 
 int32_t getPageId(const SPageInfo* pPgInfo) {
-  ASSERT(pPgInfo != NULL);
   return pPgInfo->pageId;
 }
 

From fc80c3d3735a526417a7b81baad5c499ce78b048 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 24 Jan 2023 23:56:29 +0800
Subject: [PATCH 39/40] refactor:do some internal refactor.

---
 source/util/src/tpagedbuf.c | 61 +++++++++++++++++++++++++------------
 1 file changed, 41 insertions(+), 20 deletions(-)

diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c
index d4103ab578..7c60862c56 100644
--- a/source/util/src/tpagedbuf.c
+++ b/source/util/src/tpagedbuf.c
@@ -6,6 +6,9 @@
 #include "tlog.h"
 
 #define GET_PAYLOAD_DATA(_p)          ((char*)(_p)->pData + POINTER_BYTES)
+#define BUF_PAGE_IN_MEM(_p)           ((_p)->pData != NULL)
+#define CLEAR_BUF_PAGE_IN_MEM_FLAG(_p) ((_p)->pData = NULL)
+#define HAS_DATA_IN_DISK(_p)          ((_p)->offset >= 0)
 #define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
 
 typedef struct SPageDiskInfo {
@@ -14,7 +17,7 @@ typedef struct SPageDiskInfo {
 } SPageDiskInfo, SFreeListItem;
 
 struct SPageInfo {
-  SListNode* pn;  // point to list node struct
+  SListNode* pn;         // point to list node struct. it is NULL when the page is evicted from the in-memory buffer
   void*      pData;
   int64_t    offset;
   int32_t    pageId;
@@ -112,8 +115,6 @@ static uint64_t allocateNewPositionInFile(SDiskbasedBuf* pBuf, size_t size) {
   }
 }
 
-static void setPageNotInBuf(SPageInfo* pPageInfo) { pPageInfo->pData = NULL; }
-
 /**
  *   +--------------------------+-------------------+--------------+
  *   | PTR to SPageInfo (8bytes)| Payload (PageSize)| 2 Extra Bytes|
@@ -134,17 +135,18 @@ static char* doFlushBufPage(SDiskbasedBuf* pBuf, SPageInfo* pg) {
 
   int32_t size = pBuf->pageSize;
   char*   t = NULL;
-  if (pg->offset == -1 || pg->dirty) {
+  if ((!HAS_DATA_IN_DISK(pg)) || pg->dirty) {
     void* payload = GET_PAYLOAD_DATA(pg);
     t = doCompressData(payload, pBuf->pageSize, &size, pBuf);
-    ASSERTS(size >= 0, "size is negative");
+    if (size < 0) {
+      uError("failed to compress data when flushing data to disk, %s", pBuf->id);
+      return NULL;
+    }
   }
 
   // this page is flushed to disk for the first time
   if (pg->dirty) {
-    if (pg->offset == -1) {
-      ASSERTS(pg->dirty == true, "pg->dirty is false");
-
+    if (!HAS_DATA_IN_DISK(pg)) {
       pg->offset = allocateNewPositionInFile(pBuf, size);
       pBuf->nextPos += size;
 
@@ -160,6 +162,7 @@ static char* doFlushBufPage(SDiskbasedBuf* pBuf, SPageInfo* pg) {
         return NULL;
       }
 
+      // extend the file size
       if (pBuf->fileSize < pg->offset + size) {
         pBuf->fileSize = pg->offset + size;
       }
@@ -202,13 +205,13 @@ static char* doFlushBufPage(SDiskbasedBuf* pBuf, SPageInfo* pg) {
     size = pg->length;
   }
 
-  ASSERT(size > 0 || (pg->offset == -1 && pg->length == -1));
-
   char* pDataBuf = pg->pData;
   memset(pDataBuf, 0, getAllocPageSize(pBuf->pageSize));
+
 #ifdef BUF_PAGE_DEBUG
   uDebug("page_flush %p, pageId:%d, offset:%d", pDataBuf, pg->pageId, pg->offset);
 #endif
+
   pg->length = size;  // on disk size
   return pDataBuf;
 }
@@ -224,13 +227,19 @@ static char* flushBufPage(SDiskbasedBuf* pBuf, SPageInfo* pg) {
   }
 
   char* p = doFlushBufPage(pBuf, pg);
-  setPageNotInBuf(pg);
+  CLEAR_BUF_PAGE_IN_MEM_FLAG(pg);
+
   pg->dirty = false;
   return p;
 }
 
 // load file block data in disk
 static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
+  if (pg->offset < 0 || pg->length <= 0) {
+    uError("failed to load buf page from disk, offset:%"PRId64", length:%d, %s", pg->offset, pg->length, pBuf->id);
+    return TSDB_CODE_INVALID_PARA;
+  }
+
   int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET);
   if (ret == -1) {
     ret = TAOS_SYSTEM_ERROR(errno);
@@ -252,7 +261,7 @@ static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) {
   return 0;
 }
 
-static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t pageId) {
+static SPageInfo* registerNewPageInfo(SDiskbasedBuf* pBuf, int32_t pageId) {
   pBuf->numOfPages += 1;
 
   SPageInfo* ppi = taosMemoryMalloc(sizeof(SPageInfo));
@@ -279,7 +288,9 @@ static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) {
   SListNode* pn = NULL;
   while ((pn = tdListNext(&iter)) != NULL) {
     SPageInfo* pageInfo = *(SPageInfo**)pn->data;
-    ASSERT(pageInfo->pageId >= 0 && pageInfo->pn == pn);
+
+    SPageInfo* p = *(SPageInfo**)(pageInfo->pData);
+    ASSERT(pageInfo->pageId >= 0 && pageInfo->pn == pn && p == pageInfo);
 
     if (!pageInfo->used) {
       break;
@@ -299,7 +310,6 @@ static char* evictBufPage(SDiskbasedBuf* pBuf) {
   tdListPopNode(pBuf->lruList, pn);
 
   SPageInfo* d = *(SPageInfo**)pn->data;
-  ASSERTS(d->pn == pn, "d->pn not equal pn");
 
   d->pn = NULL;
   taosMemoryFreeClear(pn);
@@ -422,7 +432,7 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) {
     *pageId = (++pBuf->allocateId);
 
     // register page id info
-    pi = registerPage(pBuf, *pageId);
+    pi = registerNewPageInfo(pBuf, *pageId);
     if (pi == NULL) {
       return NULL;
     }
@@ -446,15 +456,21 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) {
 
 void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
   if (id < 0) {
+    terrno = TSDB_CODE_INVALID_PARA;
+    uError("invalid page id:%d, %s", id, pBuf->id);
     return NULL;
   }
 
   pBuf->statis.getPages += 1;
 
   SPageInfo** pi = taosHashGet(pBuf->all, &id, sizeof(int32_t));
-  ASSERT(pi != NULL && *pi != NULL);
+  if (pi == NULL || *pi == NULL) {
+    uError("failed to locate the buffer page:%d, %s", id, pBuf->id);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return NULL;
+  }
 
-  if ((*pi)->pData != NULL) {  // it is in memory
+  if (BUF_PAGE_IN_MEM(*pi)) {  // it is in memory
     // no need to update the LRU list if only one page exists
     if (pBuf->numOfPages == 1) {
       (*pi)->used = true;
@@ -462,7 +478,10 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
     }
 
     SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data);
-    ASSERT(*pInfo == *pi);
+    if (*pInfo != *pi) {
+      uError("inconsistently data in paged buffer, pInfo:%p, pi:%p, %s", *pInfo, *pi, pBuf->id);
+      return NULL;
+    }
 
     lruListMoveToFront(pBuf->lruList, (*pi));
     (*pi)->used = true;
@@ -472,10 +491,12 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
 #endif
     return (void*)(GET_PAYLOAD_DATA(*pi));
   } else {  // not in memory
-    ASSERT((*pi)->pData == NULL && (*pi)->pn == NULL &&
+    ASSERT((!BUF_PAGE_IN_MEM(*pi)) && (*pi)->pn == NULL &&
            (((*pi)->length >= 0 && (*pi)->offset >= 0) || ((*pi)->length == -1 && (*pi)->offset == -1)));
 
     (*pi)->pData = doExtractPage(pBuf);
+
+    // failed to evict buffer page, return with error code.
     if ((*pi)->pData == NULL) {
       return NULL;
     }
@@ -487,7 +508,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
     (*pi)->used = true;
 
     // some data has been flushed to disk, and needs to be loaded into buffer again.
-    if ((*pi)->length > 0 && (*pi)->offset >= 0) {
+    if (HAS_DATA_IN_DISK(*pi)) {
       int32_t code = loadPageFromDisk(pBuf, *pi);
       if (code != 0) {
         terrno = code;

From ab1f87d19f183bed690edb1c57bc0774dab076d9 Mon Sep 17 00:00:00 2001
From: freemine <freemine@yeah.net>
Date: Wed, 25 Jan 2023 15:32:09 +0800
Subject: [PATCH 40/40] typo correction (#19667)

---
 source/util/src/tcache.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c
index 7d1686ef80..761da6986b 100644
--- a/source/util/src/tcache.c
+++ b/source/util/src/tcache.c
@@ -921,7 +921,7 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1)
 void taosStopCacheRefreshWorker(void) {
   stopRefreshWorker = true;
   TdThreadOnce tmp = PTHREAD_ONCE_INIT;
-  if (memcmp(&cacheRefreshWorker, &tmp, sizeof(TdThreadOnce)) != 0) taosThreadJoin(cacheRefreshWorker, NULL);
+  if (memcmp(&cacheThreadInit, &tmp, sizeof(TdThreadOnce)) != 0) taosThreadJoin(cacheRefreshWorker, NULL);
   taosArrayDestroy(pCacheArrayList);
 }