From 797af5373ddca04177a64da4c78ebcab517e3b7c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 29 Nov 2022 10:59:40 +0800
Subject: [PATCH 01/32] enh(query): improve the multi-way merge performance.

---
 source/libs/executor/src/exchangeoperator.c |  7 +---
 source/libs/executor/src/sortoperator.c     |  5 ++-
 source/libs/executor/src/tsort.c            | 46 +++++++++++----------
 3 files changed, 30 insertions(+), 28 deletions(-)

diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c
index b2ddff45a4..001d668749 100644
--- a/source/libs/executor/src/exchangeoperator.c
+++ b/source/libs/executor/src/exchangeoperator.c
@@ -209,7 +209,7 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
   }
 }
 
-static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) {
+static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) {
   SExchangeInfo* pExchangeInfo = pOperator->info;
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
 
@@ -318,7 +318,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
   pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock);
 
   pOperator->fpSet =
-      createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL);
+      createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, NULL);
   return pOperator;
 
 _error:
@@ -581,13 +581,10 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) {
 
   pOperator->status = OP_RES_TO_RETURN;
   pOperator->cost.openCost = taosGetTimestampUs() - startTs;
-
-  tsem_wait(&pExchangeInfo->ready);
   if (isTaskKilled(pTaskInfo)) {
     longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
   }
 
-  tsem_post(&pExchangeInfo->ready);
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index ec754f31b0..2f504cee3d 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -559,7 +559,7 @@ typedef struct SMultiwayMergeOperatorInfo {
   STupleHandle*  prefetchedTuple;
 } SMultiwayMergeOperatorInfo;
 
-int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
+int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) {
   SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
   SExecTaskInfo*              pTaskInfo = pOperator->pTaskInfo;
 
@@ -580,6 +580,7 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
     SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
     ps->param = pOperator->pDownstream[i];
     ps->onlyRef = true;
+
     tsortAddSource(pInfo->pSortHandle, ps);
   }
 
@@ -792,7 +793,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
   pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1);  // one additional is reserved for merged result.
 
   setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo);
-  pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL,
+  pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL,
                                          destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo);
 
   code = appendDownstream(pOperator, downStreams, numStreams);
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 3f91142708..4db76b82f5 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -34,14 +34,12 @@ struct SSortHandle {
   int32_t        pageSize;
   int32_t        numOfPages;
   SDiskbasedBuf* pBuf;
-
-  SArray* pSortInfo;
-  SArray* pOrderedSource;
-
-  int32_t  loops;
-  uint64_t sortElapsed;
-  int64_t  startTs;
-  uint64_t totalElapsed;
+  SArray*        pSortInfo;
+  SArray*        pOrderedSource;
+  int32_t        loops;
+  uint64_t       sortElapsed;
+  int64_t        startTs;
+  uint64_t       totalElapsed;
 
   int32_t           sourceId;
   SSDataBlock*      pDataBlock;
@@ -99,9 +97,9 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page
 }
 
 static int32_t sortComparCleanup(SMsortComparParam* cmpParam) {
+  // NOTICE: pSource may be, if it is SORT_MULTISOURCE_MERGE
   for (int32_t i = 0; i < cmpParam->numOfSources; ++i) {
-    SSortSource* pSource =
-        cmpParam->pSources[i];  // NOTICE: pSource may be SGenericSource *, if it is SORT_MULTISOURCE_MERGE
+    SSortSource* pSource = cmpParam->pSources[i];
     blockDataDestroy(pSource->src.pBlock);
     taosMemoryFreeClear(pSource);
   }
@@ -228,15 +226,15 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
   return doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId, pPageIdList);
 }
 
-static void setCurrentSourceIsDone(SSortSource* pSource, SSortHandle* pHandle) {
+static void setCurrentSourceDone(SSortSource* pSource, SSortHandle* pHandle) {
   pSource->src.rowIndex = -1;
   ++pHandle->numOfCompletedSources;
 }
 
-static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex,
+static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32_t startIndex, int32_t endIndex,
                               SSortHandle* pHandle) {
-  cmpParam->pSources = taosArrayGet(pSources, startIndex);
-  cmpParam->numOfSources = (endIndex - startIndex + 1);
+  pParam->pSources = taosArrayGet(pSources, startIndex);
+  pParam->numOfSources = (endIndex - startIndex + 1);
 
   int32_t code = 0;
 
@@ -244,7 +242,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
   if (pHandle->pBuf == NULL) {
     if (!osTempSpaceAvailable()) {
       code = TSDB_CODE_NO_AVAIL_DISK;
-      qError("Sort compare init failed since %s", terrstr(code));
+      qError("Sort compare init failed since %s, %s", terrstr(code), pHandle->idStr);
       return code;
     }
 
@@ -257,12 +255,12 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
   }
 
   if (pHandle->type == SORT_SINGLESOURCE_SORT) {
-    for (int32_t i = 0; i < cmpParam->numOfSources; ++i) {
-      SSortSource* pSource = cmpParam->pSources[i];
+    for (int32_t i = 0; i < pParam->numOfSources; ++i) {
+      SSortSource* pSource = pParam->pSources[i];
 
       // set current source is done
       if (taosArrayGetSize(pSource->pageIdList) == 0) {
-        setCurrentSourceIsDone(pSource, pHandle);
+        setCurrentSourceDone(pSource, pHandle);
         continue;
       }
 
@@ -277,15 +275,21 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
       releaseBufPage(pHandle->pBuf, pPage);
     }
   } else {
-    for (int32_t i = 0; i < cmpParam->numOfSources; ++i) {
-      SSortSource* pSource = cmpParam->pSources[i];
+    qDebug("start init for the multiway merge sort, %s", pHandle->idStr);
+    int64_t st = taosGetTimestampUs();
+
+    for (int32_t i = 0; i < pParam->numOfSources; ++i) {
+      SSortSource* pSource = pParam->pSources[i];
       pSource->src.pBlock = pHandle->fetchfp(pSource->param);
 
       // set current source is done
       if (pSource->src.pBlock == NULL) {
-        setCurrentSourceIsDone(pSource, pHandle);
+        setCurrentSourceDone(pSource, pHandle);
       }
     }
+
+    int64_t et = taosGetTimestampUs();
+    qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr);
   }
 
   return code;

From a1585f16f10076cffc0d2c82737b4e42e5c3fedb Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 29 Nov 2022 11:06:20 +0800
Subject: [PATCH 02/32] ehn(query): improve the merge sort performance.

---
 source/libs/executor/src/sortoperator.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 2f504cee3d..0287af62d2 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -577,8 +577,13 @@ int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) {
   tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort);
 
   for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
+    SOperatorInfo* pDownstream = pOperator->pDownstream[i];
+    if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) {
+      pDownstream->fpSet._openFn(pDownstream);
+    }
+
     SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
-    ps->param = pOperator->pDownstream[i];
+    ps->param = pDownstream;
     ps->onlyRef = true;
 
     tsortAddSource(pInfo->pSortHandle, ps);

From dac930352b7764c3f0b36b9703a8251ad1d31784 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 29 Nov 2022 12:22:04 +0800
Subject: [PATCH 03/32] enh(query): optimize the perf in compare.

---
 include/common/ttypes.h            |  2 -
 source/libs/scalar/src/sclvector.c | 62 +++++++++++++++++++-----------
 2 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/include/common/ttypes.h b/include/common/ttypes.h
index 761ffd0f1c..6350057c1f 100644
--- a/include/common/ttypes.h
+++ b/include/common/ttypes.h
@@ -278,11 +278,9 @@ typedef struct {
 #define IS_VALID_TINYINT(_t)   ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
 #define IS_VALID_SMALLINT(_t)  ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
 #define IS_VALID_INT(_t)       ((_t) >= INT32_MIN && (_t) <= INT32_MAX)
-#define IS_VALID_BIGINT(_t)    ((_t) >= INT64_MIN && (_t) <= INT64_MAX)
 #define IS_VALID_UTINYINT(_t)  ((_t) >= 0 && (_t) <= UINT8_MAX)
 #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX)
 #define IS_VALID_UINT(_t)      ((_t) >= 0 && (_t) <= UINT32_MAX)
-#define IS_VALID_UBIGINT(_t)   ((_t) >= 0 && (_t) <= UINT64_MAX)
 #define IS_VALID_FLOAT(_t)     ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)    ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index c4ff5b2b01..98391d95f4 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -1543,9 +1543,44 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut,
 int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t startIndex,
                             int32_t numOfRows, int32_t step, __compar_fn_t fp, int32_t optr) {
   int32_t num = 0;
-  bool *  pRes = (bool *)pOut->columnData->pData;
+  bool   *pRes = (bool *)pOut->columnData->pData;
 
-  if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_JSON || GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_JSON) {
+  if (IS_MATHABLE_TYPE(GET_PARAM_TYPE(pLeft)) && IS_MATHABLE_TYPE(GET_PARAM_TYPE(pRight))) {
+    if (!(pLeft->columnData->hasNull || pRight->columnData->hasNull)) {
+      for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) {
+        int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i;
+        int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i;
+
+        char *pLeftData = colDataGetData(pLeft->columnData, leftIndex);
+        char *pRightData = colDataGetData(pRight->columnData, rightIndex);
+
+        pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData);
+        if (pRes[i]) {
+          ++num;
+        }
+      }
+    } else {
+      for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) {
+        int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i;
+        int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i;
+
+        if (colDataIsNull_f(pLeft->columnData->nullbitmap, leftIndex) ||
+            colDataIsNull_f(pRight->columnData->nullbitmap, rightIndex)) {
+          pRes[i] = false;
+          continue;
+        }
+
+        char *pLeftData = colDataGetData(pLeft->columnData, leftIndex);
+        char *pRightData = colDataGetData(pRight->columnData, rightIndex);
+
+        pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData);
+        if (pRes[i]) {
+          ++num;
+        }
+      }
+    }
+  } else {
+    //  if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_JSON || GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_JSON) {
     for (int32_t i = startIndex; i < numOfRows && i >= startIndex; i += step) {
       int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i;
       int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i;
@@ -1556,8 +1591,8 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa
         continue;
       }
 
-      char *  pLeftData = colDataGetData(pLeft->columnData, leftIndex);
-      char *  pRightData = colDataGetData(pRight->columnData, rightIndex);
+      char   *pLeftData = colDataGetData(pLeft->columnData, leftIndex);
+      char   *pRightData = colDataGetData(pRight->columnData, rightIndex);
       int64_t leftOut = 0;
       int64_t rightOut = 0;
       bool    freeLeft = false;
@@ -1592,25 +1627,6 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa
         taosMemoryFreeClear(pRightData);
       }
     }
-  } else {
-    for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) {
-      int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i;
-      int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i;
-
-      if (colDataIsNull_s(pLeft->columnData, leftIndex) ||
-          colDataIsNull_s(pRight->columnData, rightIndex)) {
-        pRes[i] = false;
-        continue;
-      }
-
-      char *pLeftData = colDataGetData(pLeft->columnData, leftIndex);
-      char *pRightData = colDataGetData(pRight->columnData, rightIndex);
-
-      pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData);
-      if (pRes[i]) {
-        ++num;
-      }
-    }
   }
 
   return num;

From 445c4f28f0d09f3f38813f5c26f016c10350f265 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 29 Nov 2022 23:34:22 +0800
Subject: [PATCH 04/32] enh(query): optimize the perf.

---
 source/common/src/tdatablock.c          | 5 ++++-
 source/dnode/vnode/src/tsdb/tsdbRead.c  | 2 +-
 source/libs/executor/src/sortoperator.c | 3 +--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index a0f795a729..20eda52aac 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1139,6 +1139,9 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
 void blockDataCleanup(SSDataBlock* pDataBlock) {
   SDataBlockInfo* pInfo = &pDataBlock->info;
 
+  int32_t existedRows = pInfo->rows;
+  ASSERT(existedRows <= pDataBlock->info.capacity);
+
   pInfo->rows = 0;
   pInfo->id.uid = 0;
   pInfo->id.groupId = 0;
@@ -1152,7 +1155,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) {
   size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    colInfoDataCleanup(p, pInfo->capacity);
+    colInfoDataCleanup(p, existedRows);
   }
 }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 4ba311212a..660d189451 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -3589,7 +3589,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
 
   SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i);
   if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-    colDataAppendInt64(pColData, outputRowIndex, &pBlockData->aTSKEY[rowIndex]);
+    ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex];
     i += 1;
   }
 
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 0287af62d2..b355b85861 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -720,7 +720,6 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) {
   }
 
   qDebug("start to merge final sorted rows, %s", GET_TASKID(pTaskInfo));
-
   SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->matchInfo.pList, pOperator);
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
@@ -787,7 +786,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
   SPhysiNode*  pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
   SSDataBlock* pInputBlock = createDataBlockFromDescNode(pChildNode->pOutputDataBlockDesc);
 
-  initResultSizeInfo(&pOperator->resultInfo, 4096);
+  initResultSizeInfo(&pOperator->resultInfo, 1024);
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
 
   pInfo->groupSort = pMergePhyNode->groupSort;

From a8703c7fc22d661d1fbd7e196ddeb475b4ffbfd7 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 00:11:21 +0800
Subject: [PATCH 05/32] enh(query): optimize the perf.

---
 source/libs/executor/src/timewindowoperator.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index a6a477a9e3..dae2dafe9e 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1286,7 +1286,6 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
     return NULL;
   }
 
-  blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
   while (1) {
     doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
     doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL);
@@ -1737,7 +1736,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
   pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId;
 
   size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
-  initResultSizeInfo(&pOperator->resultInfo, 4096);
+  initResultSizeInfo(&pOperator->resultInfo, 1024);
+  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
 
   int32_t    num = 0;
   SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);

From 3f0c93a62308112a1ba2a5093277fb8c342ff509 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 14:53:25 +0800
Subject: [PATCH 06/32] enh(query): optimize query perf.

---
 source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index 294a4bd3e4..c149e52fed 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -40,7 +40,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd
   }
   pFD->szPage = szPage;
   pFD->pgno = 0;
-  pFD->pBuf = taosMemoryCalloc(1, szPage);
+  pFD->pBuf = taosMemoryMalloc(szPage);
   if (pFD->pBuf == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     taosCloseFile(&pFD->pFD);

From ae816331441c1602d500a5f3d208da76e1fe323b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 14:55:27 +0800
Subject: [PATCH 07/32] enh(query): optimize query perf.

---
 source/common/src/tdatablock.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 20eda52aac..af45db053a 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1242,7 +1242,7 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
   size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, true);
+    code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, false);
     if (code) {
       return code;
     }

From 8dcbafb78066547b8f8f04c4772e2f7a559f1415 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 15:24:03 +0800
Subject: [PATCH 08/32] enh(query): avoiding unnecessary remove operation.

---
 source/util/src/tpagedbuf.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c
index e1a43ace47..b7dbdad053 100644
--- a/source/util/src/tpagedbuf.c
+++ b/source/util/src/tpagedbuf.c
@@ -507,7 +507,9 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
 
   dBufPrintStatis(pBuf);
 
+  bool needRemoveFile = false;
   if (pBuf->pFile != NULL) {
+    needRemoveFile = true;
     uDebug(
         "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page "
         "size:%.2f Kb, %s\n",
@@ -534,9 +536,14 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
     }
   }
 
-  if (taosRemoveFile(pBuf->path) < 0) {
-    uDebug("WARNING tPage remove file failed. path=%s", pBuf->path);
+  if (needRemoveFile) {
+    int32_t ret = taosRemoveFile(pBuf->path);
+    if (ret != 0) {  // print the error and discard this error info
+      int32_t code = TAOS_SYSTEM_ERROR(errno);
+      uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(code));
+    }
   }
+
   taosMemoryFreeClear(pBuf->path);
 
   size_t n = taosArrayGetSize(pBuf->pIdList);

From d66b7f3de07d6bb1149b5be41b5aab6ac0fa2e7e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 15:25:19 +0800
Subject: [PATCH 09/32] refactor: do some internal refactor.

---
 source/util/src/tpagedbuf.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c
index b7dbdad053..0f4c51060b 100644
--- a/source/util/src/tpagedbuf.c
+++ b/source/util/src/tpagedbuf.c
@@ -539,8 +539,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
   if (needRemoveFile) {
     int32_t ret = taosRemoveFile(pBuf->path);
     if (ret != 0) {  // print the error and discard this error info
-      int32_t code = TAOS_SYSTEM_ERROR(errno);
-      uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(code));
+      uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(errno));
     }
   }
 

From 4fef284cfe07d71afe4f088b0d7767f79f1ae636 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 21:26:27 +0800
Subject: [PATCH 10/32] refactor: do some internal refactor.

---
 include/libs/function/function.h         | 59 +++++++-----------------
 include/libs/function/tudf.h             | 26 +++++++++++
 source/dnode/mgmt/node_util/inc/dmUtil.h |  2 +-
 3 files changed, 43 insertions(+), 44 deletions(-)

diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index 240772bfc2..8fe9d98e57 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -137,22 +137,22 @@ typedef struct SqlFunctionCtx {
   int16_t              functionId;  // function id
   char                *pOutput;     // final result output buffer, point to sdata->data
   int32_t              numOfParams;
-  SFunctParam     *param;  // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
-  SColumnInfoData *pTsOutput;  // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
-  int32_t          offset;
-  struct SResultRowEntryInfo *resultInfo;
-  SSubsidiaryResInfo          subsidiaries;
-  SPoint1                     start;
-  SPoint1                     end;
-  SFuncExecFuncs              fpSet;
-  SScalarFuncExecFuncs        sfp;
-  struct SExprInfo           *pExpr;
-  struct SSDataBlock         *pSrcBlock;
-  struct SSDataBlock         *pDstBlock;  // used by indefinite rows function to set selectivity
-  SSerializeDataHandle        saveHandle;
-  bool                        isStream;
-
-  char udfName[TSDB_FUNC_NAME_LEN];
+  // input parameter, e.g., top(k, 20), the number of results of top query is kept in param
+  SFunctParam         *param;
+  // corresponding output buffer for timestamp of each result, e.g., diff/csum
+  SColumnInfoData     *pTsOutput;
+  int32_t              offset;
+  SResultRowEntryInfo *resultInfo;
+  SSubsidiaryResInfo   subsidiaries;
+  SPoint1              start;
+  SPoint1              end;
+  SFuncExecFuncs       fpSet;
+  SScalarFuncExecFuncs sfp;
+  struct SExprInfo    *pExpr;
+  struct SSDataBlock  *pSrcBlock;
+  struct SSDataBlock  *pDstBlock;  // used by indefinite rows function to set selectivity
+  SSerializeDataHandle saveHandle;
+  char                 udfName[TSDB_FUNC_NAME_LEN];
 } SqlFunctionCtx;
 
 typedef struct tExprNode {
@@ -182,7 +182,6 @@ struct SScalarParam {
 };
 
 void    cleanupResultRowEntry(struct SResultRowEntryInfo *pCell);
-//int32_t getNumOfResult(SqlFunctionCtx *pCtx, int32_t num, SSDataBlock *pResBlock);
 bool    isRowEntryCompleted(struct SResultRowEntryInfo *pEntry);
 bool    isRowEntryInitialized(struct SResultRowEntryInfo *pEntry);
 
@@ -194,32 +193,6 @@ typedef struct SPoint {
 int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2,
                                       int32_t inputType);
 
-///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-// udf api
-/**
- * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf
- * @return error code
- */
-int32_t udfcOpen();
-
-/**
- * destroy udfd proxy
- * @return error code
- */
-int32_t udfcClose();
-
-/**
- * start udfd that serves udf function invocation under dnode startDnodeId
- * @param startDnodeId
- * @return
- */
-int32_t udfStartUdfd(int32_t startDnodeId);
-/**
- * stop udfd
- * @return
- */
-int32_t udfStopUdfd();
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h
index 31cc53bb9f..b71d50d43c 100644
--- a/include/libs/function/tudf.h
+++ b/include/libs/function/tudf.h
@@ -85,6 +85,32 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols,
 
 int32_t cleanUpUdfs();
 
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// udf api
+/**
+ * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf
+ * @return error code
+ */
+int32_t udfcOpen();
+
+/**
+ * destroy udfd proxy
+ * @return error code
+ */
+int32_t udfcClose();
+
+/**
+ * start udfd that serves udf function invocation under dnode startDnodeId
+ * @param startDnodeId
+ * @return
+ */
+int32_t udfStartUdfd(int32_t startDnodeId);
+/**
+ * stop udfd
+ * @return
+ */
+int32_t udfStopUdfd();
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h
index 8719e988e7..2124b387ec 100644
--- a/source/dnode/mgmt/node_util/inc/dmUtil.h
+++ b/source/dnode/mgmt/node_util/inc/dmUtil.h
@@ -39,7 +39,7 @@
 #include "sync.h"
 #include "wal.h"
 
-#include "libs/function/function.h"
+#include "libs/function/tudf.h"
 #ifdef __cplusplus
 extern "C" {
 #endif

From 26ab0894a88f97699cbf9cb6a05bd666e0e618b6 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 21:56:02 +0800
Subject: [PATCH 11/32] fix(query): fix syntax error.

---
 source/libs/executor/src/executil.c           | 2 --
 source/libs/executor/src/timewindowoperator.c | 7 +------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index b135566caa..7c37d66eb6 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1536,8 +1536,6 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
     pCtx->start.key = INT64_MIN;
     pCtx->end.key = INT64_MIN;
     pCtx->numOfParams = pExpr->base.numOfParams;
-    pCtx->isStream = false;
-
     pCtx->param = pFunct->pParam;
     pCtx->saveHandle.currentPage = -1;
   }
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index dae2dafe9e..35548a6ade 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1717,7 +1717,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSuppor
 
 void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
   for (int32_t i = 0; i < numOfExpr; i++) {
-    pCtx[i].isStream = true;
+//    pCtx[i].isStream = true;
   }
 }
 
@@ -1783,11 +1783,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
     goto _error;
   }
 
-  if (isStream) {
-    ASSERT(num > 0);
-    initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
-  }
-
   initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
   pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, pInfo);
   if (pInfo->timeWindowInterpo) {

From cefe4be1abef27af09104d7a053ecc3478d93bfb Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 00:34:06 +0800
Subject: [PATCH 12/32] refactor: do some internal refactor.

---
 include/common/tdatablock.h                   |  1 +
 source/common/src/tdatablock.c                | 19 +++++++++
 .../dnode/vnode/src/tsdb/tsdbReaderWriter.c   |  2 +-
 source/libs/executor/src/executorimpl.c       |  2 +-
 source/libs/executor/src/scanoperator.c       |  2 +-
 source/libs/executor/src/timewindowoperator.c | 41 ++++++++++---------
 source/util/src/tarray.c                      |  2 +-
 source/util/src/thash.c                       |  4 +-
 8 files changed, 48 insertions(+), 25 deletions(-)

diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index db6ca65cf0..a5d3a0829d 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -234,6 +234,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
 
 int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload);
 int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
+int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows);
 
 void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
 void blockDataCleanup(SSDataBlock* pDataBlock);
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index af45db053a..f5e73d8f0e 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1239,6 +1239,25 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
     return TSDB_CODE_SUCCESS;
   }
 
+  size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
+    code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, true);
+    if (code) {
+      return code;
+    }
+  }
+
+  pDataBlock->info.capacity = numOfRows;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows) {
+  int32_t code = 0;
+  if (numOfRows == 0 || numOfRows <= pDataBlock->info.capacity) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index c149e52fed..294a4bd3e4 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -40,7 +40,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd
   }
   pFD->szPage = szPage;
   pFD->pgno = 0;
-  pFD->pBuf = taosMemoryMalloc(szPage);
+  pFD->pBuf = taosMemoryCalloc(1, szPage);
   if (pFD->pBuf == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     taosCloseFile(&pFD->pFD);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 300ba52934..3e28798a3a 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -1749,7 +1749,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n
   pAggSup->currentPageId = -1;
   pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput);
   pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t));
-  pAggSup->pResultRowHashTable = tSimpleHashInit(10, hashFn);
+  pAggSup->pResultRowHashTable = tSimpleHashInit(100, hashFn);
 
   if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 84b7678b9f..ca3734d91e 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -886,7 +886,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
 
   initResultSizeInfo(&pOperator->resultInfo, 4096);
   pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
-  blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
+  blockDataEnsureCapacityNoClear(pInfo->pResBlock, pOperator->resultInfo.capacity);
 
   code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
   if (code != TSDB_CODE_SUCCESS) {
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 35548a6ade..551d668763 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1658,24 +1658,6 @@ static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
 static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
   // the primary timestamp column
   bool needed = false;
-  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
-  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
-
-  {  // ts column
-    SColumn c = {0};
-    c.colId = 1;
-    c.slotId = pInfo->primaryTsIndex;
-    c.type = TSDB_DATA_TYPE_TIMESTAMP;
-    c.bytes = sizeof(int64_t);
-    taosArrayPush(pInfo->pInterpCols, &c);
-
-    SGroupKeys key = {0};
-    key.bytes = c.bytes;
-    key.type = c.type;
-    key.isNull = true;  // to denote no value is assigned yet
-    key.pData = taosMemoryCalloc(1, c.bytes);
-    taosArrayPush(pInfo->pPrevValues, &key);
-  }
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SExprInfo* pExpr = pCtx[i].pExpr;
@@ -1696,6 +1678,27 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
     }
   }
 
+  if (needed) {
+    pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
+    pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
+
+    {  // ts column
+      SColumn c = {0};
+      c.colId = 1;
+      c.slotId = pInfo->primaryTsIndex;
+      c.type = TSDB_DATA_TYPE_TIMESTAMP;
+      c.bytes = sizeof(int64_t);
+      taosArrayPush(pInfo->pInterpCols, &c);
+
+      SGroupKeys key;
+      key.bytes = c.bytes;
+      key.type = c.type;
+      key.isNull = true;  // to denote no value is assigned yet
+      key.pData = taosMemoryCalloc(1, c.bytes);
+      taosArrayPush(pInfo->pPrevValues, &key);
+    }
+  }
+
   return needed;
 }
 
@@ -1737,7 +1740,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
 
   size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
   initResultSizeInfo(&pOperator->resultInfo, 1024);
-  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+  blockDataEnsureCapacityNoClear(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
 
   int32_t    num = 0;
   SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index f53b1cfd7f..6a16c044f9 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -33,7 +33,7 @@ SArray* taosArrayInit(size_t size, size_t elemSize) {
   }
 
   pArray->size = 0;
-  pArray->pData = taosMemoryCalloc(size, elemSize);
+  pArray->pData = taosMemoryMalloc(size * elemSize);
   if (pArray->pData == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     taosMemoryFree(pArray);
diff --git a/source/util/src/thash.c b/source/util/src/thash.c
index a0411483ca..b07d485c87 100644
--- a/source/util/src/thash.c
+++ b/source/util/src/thash.c
@@ -244,7 +244,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
     capacity = 4;
   }
 
-  SHashObj *pHashObj = (SHashObj *)taosMemoryCalloc(1, sizeof(SHashObj));
+  SHashObj *pHashObj = (SHashObj *)taosMemoryMalloc(sizeof(SHashObj));
   if (pHashObj == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
@@ -264,7 +264,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
 
   ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0);
 
-  pHashObj->hashList = (SHashEntry **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *));
+  pHashObj->hashList = (SHashEntry **)taosMemoryMalloc(pHashObj->capacity * sizeof(void *));
   if (pHashObj->hashList == NULL) {
     taosMemoryFree(pHashObj);
     terrno = TSDB_CODE_OUT_OF_MEMORY;

From c996837fab936338a81491e9c4ae88412091f7ef Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 01:10:37 +0800
Subject: [PATCH 13/32] refactor: do some internal refactor.

---
 source/util/src/tarray.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index 6a16c044f9..f53b1cfd7f 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -33,7 +33,7 @@ SArray* taosArrayInit(size_t size, size_t elemSize) {
   }
 
   pArray->size = 0;
-  pArray->pData = taosMemoryMalloc(size * elemSize);
+  pArray->pData = taosMemoryCalloc(size, elemSize);
   if (pArray->pData == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     taosMemoryFree(pArray);

From f8a2ab8338cedbdb68d3f2c1fc4236c1d95fee57 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 17:24:26 +0800
Subject: [PATCH 14/32] refactor: do some internal refactor.

---
 include/common/tcommon.h                      |   5 +-
 include/common/tdatablock.h                   |   6 +-
 source/common/src/tdatablock.c                |  34 +++--
 source/dnode/vnode/inc/vnode.h                |   3 +-
 source/dnode/vnode/src/tsdb/tsdbRead.c        | 123 ++++++++----------
 source/libs/executor/src/executil.c           |  13 +-
 source/libs/executor/src/executor.c           |   4 +-
 source/libs/executor/src/executorimpl.c       |   2 -
 source/libs/executor/src/scanoperator.c       |  39 ++----
 source/libs/executor/src/sysscanoperator.c    |  85 ++++++------
 source/libs/executor/src/timewindowoperator.c |   6 +-
 source/libs/scalar/src/sclvector.c            |   2 +-
 source/util/src/thash.c                       |   5 +-
 13 files changed, 156 insertions(+), 171 deletions(-)

diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 77f1879b81..6ec3d5db10 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -249,10 +249,11 @@ typedef struct SColumnInfoData {
 
 typedef struct SQueryTableDataCond {
   uint64_t     suid;
-  int32_t      order;  // desc|asc order to iterate the data block
+  int32_t      order;    // desc|asc order to iterate the data block
   int32_t      numOfCols;
   SColumnInfo* colList;
-  int32_t      type;  // data block load type:
+  int32_t*     pSlotList; // the column output destation slot, and it may be null
+  int32_t      type;     // data block load type:
   STimeWindow  twindows;
   int64_t      startVersion;
   int64_t      endVersion;
diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index a5d3a0829d..4208d85648 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -41,7 +41,7 @@ typedef struct SBlockOrderInfo {
     BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
   } while (0)
 
-#define colDataSetNotNull_f(bm_, r_)                  \
+#define colDataClearNull_f(bm_, r_)                  \
   do {                                                \
     BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \
   } while (0)
@@ -151,9 +151,6 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui
     for (int32_t i = start; i < start + nRows; ++i) {
       colDataSetNull_f(pColumnInfoData->nullbitmap, i);
     }
-
-    int32_t bytes = pColumnInfoData->info.bytes;
-    memset(pColumnInfoData->pData + start * bytes, 0, nRows * bytes);
   }
 
   pColumnInfoData->hasNull = true;
@@ -238,6 +235,7 @@ int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRo
 
 void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
 void blockDataCleanup(SSDataBlock* pDataBlock);
+void blockDataEmpty(SSDataBlock* pDataBlock);
 
 size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
 
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index f5e73d8f0e..c300c3e6f0 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1137,17 +1137,15 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
 }
 
 void blockDataCleanup(SSDataBlock* pDataBlock) {
+  blockDataEmpty(pDataBlock);
   SDataBlockInfo* pInfo = &pDataBlock->info;
-
-  int32_t existedRows = pInfo->rows;
-  ASSERT(existedRows <= pDataBlock->info.capacity);
-
-  pInfo->rows = 0;
   pInfo->id.uid = 0;
   pInfo->id.groupId = 0;
-  pInfo->window.ekey = 0;
-  pInfo->window.skey = 0;
+}
 
+void blockDataEmpty(SSDataBlock* pDataBlock) {
+  SDataBlockInfo* pInfo = &pDataBlock->info;
+  ASSERT(pInfo->rows <= pDataBlock->info.capacity);
   if (pInfo->capacity == 0) {
     return;
   }
@@ -1155,8 +1153,12 @@ void blockDataCleanup(SSDataBlock* pDataBlock) {
   size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    colInfoDataCleanup(p, existedRows);
+    colInfoDataCleanup(p, pInfo->capacity);
   }
+
+  pInfo->rows = 0;
+  pInfo->window.ekey = 0;
+  pInfo->window.skey = 0;
 }
 
 // todo temporarily disable it
@@ -1643,6 +1645,8 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s
 static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
     pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, n, total);
+
+    // clear the offset value of the unused entries.
     memset(&pColInfoData->varmeta.offset[total - n], 0, n);
   } else {
     int32_t bytes = pColInfoData->info.bytes;
@@ -1657,7 +1661,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
   }
 
   if (pBlock->info.rows <= n) {
-    blockDataCleanup(pBlock);
+    blockDataEmpty(pBlock);
   } else {
     size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
     for (int32_t i = 0; i < numOfCols; ++i) {
@@ -1674,12 +1678,22 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
     pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
     memset(&pColInfoData->varmeta.offset[n], 0, total - n);
+  } else {  // reset the bitmap value
+    int32_t stopIndex = BitmapLen(n) * 8;
+    for(int32_t i = n + 1; i < stopIndex; ++i) {
+      colDataClearNull_f(pColInfoData->nullbitmap, i);
+    }
+
+    int32_t remain = BitmapLen(total) - BitmapLen(n);
+    if (remain > 0) {
+      memset(pColInfoData->nullbitmap, 0, remain);
+    }
   }
 }
 
 int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) {
   if (n == 0) {
-    blockDataCleanup(pBlock);
+    blockDataEmpty(pBlock);
     return TSDB_CODE_SUCCESS;
   }
 
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 7ef3207b4d..53b8eadb33 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -159,11 +159,10 @@ typedef struct STsdbReader STsdbReader;
 
 int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num);
 int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
-                       STsdbReader **ppReader, const char *idstr);
+                       SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr);
 
 void     tsdbReaderClose(STsdbReader *pReader);
 bool     tsdbNextDataBlock(STsdbReader *pReader);
-bool     tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid);
 void     tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow);
 int32_t  tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
 SArray  *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 660d189451..27d99bd42d 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -85,7 +85,8 @@ typedef struct SBlockLoadSuppInfo {
   SArray*          pColAgg;
   SColumnDataAgg   tsColAgg;
   SColumnDataAgg** plist;
-  int16_t*         colIds;  // column ids for loading file block data
+  int16_t*         colIds;    // column ids for loading file block data
+  int16_t*         slotIds;   // the ordinal index in the destination SSDataBlock
   int32_t          numOfCols;
   char**           buildBuf;  // build string tmp buffer, todo remove it later after all string format being updated.
   bool             smaValid;  // the sma on all queried columns are activated
@@ -214,25 +215,25 @@ static bool          hasDataInFileBlock(const SBlockData* pBlockData, const SFil
 
 static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
 
-static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SSDataBlock* pBlock) {
-  size_t numOfCols = blockDataGetNumOfCols(pBlock);
-
+static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) {
   pSupInfo->smaValid = true;
   pSupInfo->numOfCols = numOfCols;
   pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t));
   pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES);
-  if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL) {
+  pSupInfo->slotIds = taosMemoryMalloc(numOfCols * sizeof(int16_t));
+  if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL || pSupInfo->slotIds == NULL) {
     taosMemoryFree(pSupInfo->colIds);
     taosMemoryFree(pSupInfo->buildBuf);
+    taosMemoryFree(pSupInfo->slotIds);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
   for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
-    pSupInfo->colIds[i] = pCol->info.colId;
+    pSupInfo->colIds[i] = pCols[i].colId;
+    pSupInfo->slotIds[i] = pSlotIdList[i];
 
-    if (IS_VAR_DATA_TYPE(pCol->info.type)) {
-      pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes);
+    if (IS_VAR_DATA_TYPE(pCols[i].type)) {
+      pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes);
     }
   }
 
@@ -566,7 +567,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity)
 }
 
 static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, int32_t capacity,
-                                const char* idstr) {
+                                SSDataBlock* pResBlock, const char* idstr) {
   int32_t      code = 0;
   int8_t       level = 0;
   STsdbReader* pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader));
@@ -585,6 +586,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
   pReader->suid = pCond->suid;
   pReader->order = pCond->order;
   pReader->capacity = capacity;
+  pReader->pResBlock = pResBlock;
   pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL;
   pReader->verRange = getQueryVerRange(pVnode, pCond, level);
   pReader->type = pCond->type;
@@ -592,6 +594,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
   pReader->blockInfoBuf.numPerBucket = 1000;  // 1000 tables per bucket
   ASSERT(pCond->numOfCols > 0);
 
+  // todo refactor.
   limitOutputBufferSize(pCond, &pReader->capacity);
 
   // allocate buffer in order to load data blocks from file
@@ -611,13 +614,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
     goto _end;
   }
 
-  pReader->pResBlock = createResBlock(pCond, pReader->capacity);
-  if (pReader->pResBlock == NULL) {
-    code = terrno;
-    goto _end;
-  }
-
-  setColumnIdSlotList(&pReader->suppInfo, pReader->pResBlock);
+  setColumnIdSlotList(&pReader->suppInfo, pCond->colList, pCond->pSlotList, pCond->numOfCols);
 
   *ppReader = pReader;
   return code;
@@ -1041,17 +1038,16 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo
 }
 
 static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) {
-  SReaderStatus*  pStatus = &pReader->status;
-  SDataBlockIter* pBlockIter = &pStatus->blockIter;
+  SReaderStatus*      pStatus = &pReader->status;
+  SDataBlockIter*     pBlockIter = &pStatus->blockIter;
+  SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
+  SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
 
   SBlockData*         pBlockData = &pStatus->fileBlockData;
   SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter);
   SDataBlk*           pBlock = getCurrentBlock(pBlockIter);
   SSDataBlock*        pResBlock = pReader->pResBlock;
-  int32_t             numOfOutputCols = blockDataGetNumOfCols(pResBlock);
-
-  SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
-  SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
+  int32_t             numOfOutputCols = pSupInfo->numOfCols;
 
   SColVal cv = {0};
   int64_t st = taosGetTimestampUs();
@@ -1087,8 +1083,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
   int32_t i = 0;
   int32_t rowIndex = 0;
 
-  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i);
-  if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+  if (pSupInfo->colIds[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
     copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);
     i += 1;
   }
@@ -1097,12 +1093,13 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
   int32_t num = pBlockData->nColData;
   while (i < numOfOutputCols && colIndex < num) {
     rowIndex = 0;
-    pColData = taosArrayGet(pResBlock->pDataBlock, i);
 
     SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);
-    if (pData->cid < pColData->info.colId) {
+    if (pData->cid < pSupInfo->colIds[i]) {
       colIndex += 1;
-    } else if (pData->cid == pColData->info.colId) {
+    } else if (pData->cid == pSupInfo->colIds[i]) {
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+
       if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {
         colDataAppendNNULL(pColData, 0, dumpedRows);
       } else {
@@ -1119,6 +1116,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
       colIndex += 1;
       i += 1;
     } else {  // the specified column does not exist in file block, fill with null data
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
       colDataAppendNNULL(pColData, 0, dumpedRows);
       i += 1;
     }
@@ -1126,7 +1124,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
 
   // fill the mis-matched columns with null value
   while (i < numOfOutputCols) {
-    pColData = taosArrayGet(pResBlock->pDataBlock, i);
+    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
     colDataAppendNNULL(pColData, 0, dumpedRows);
     i += 1;
   }
@@ -3535,7 +3533,7 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
 
 int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow,
                              STableBlockScanInfo* pScanInfo) {
-  int32_t numOfRows = pBlock->info.rows;
+  int32_t outputRowIndex = pBlock->info.rows;
   int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
   int64_t uid = pScanInfo->uid;
 
@@ -3545,23 +3543,26 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
   SColVal colVal = {0};
   int32_t i = 0, j = 0;
 
-  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-  if (pColInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-    colDataAppend(pColInfoData, numOfRows, (const char*)&pTSRow->ts, false);
+  if (pSupInfo->colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+    ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts;
     i += 1;
   }
 
   while (i < numOfCols && j < pSchema->numOfCols) {
-    pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-    col_id_t colId = pColInfoData->info.colId;
+    col_id_t colId = pSupInfo->colIds[i];
 
     if (colId == pSchema->columns[j].colId) {
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+
       tTSRowGetVal(pTSRow, pSchema, j, &colVal);
-      doCopyColVal(pColInfoData, numOfRows, i, &colVal, pSupInfo);
+      doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo);
       i += 1;
       j += 1;
     } else if (colId < pSchema->columns[j].colId) {
-      colDataAppendNULL(pColInfoData, numOfRows);
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+
+      colDataAppendNULL(pColInfoData, outputRowIndex);
       i += 1;
     } else if (colId > pSchema->columns[j].colId) {
       j += 1;
@@ -3570,8 +3571,8 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
 
   // set null value since current column does not exist in the "pSchema"
   while (i < numOfCols) {
-    pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-    colDataAppendNULL(pColInfoData, numOfRows);
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+    colDataAppendNULL(pColInfoData, outputRowIndex);
     i += 1;
   }
 
@@ -3586,27 +3587,25 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   int32_t outputRowIndex = pResBlock->info.rows;
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
-
-  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i);
-  if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+  if (pReader->suppInfo.colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
     ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex];
     i += 1;
   }
 
   SColVal cv = {0};
   int32_t numOfInputCols = pBlockData->nColData;
-  int32_t numOfOutputCols = pResBlock->pDataBlock->size;
+  int32_t numOfOutputCols = pSupInfo->numOfCols;
 
   while (i < numOfOutputCols && j < numOfInputCols) {
-    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, i);
-    SColData*        pData = tBlockDataGetColDataByIdx(pBlockData, j);
-
-    if (pData->cid < pCol->info.colId) {
+    SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j);
+    if (pData->cid < pSupInfo->colIds[i]) {
       j += 1;
       continue;
     }
 
-    if (pData->cid == pCol->info.colId) {
+    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+    if (pData->cid == pSupInfo->colIds[i]) {
       tColDataGetValue(pData, rowIndex, &cv);
       doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
       j += 1;
@@ -3619,7 +3618,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   }
 
   while (i < numOfOutputCols) {
-    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i);
+    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
     colDataAppendNULL(pCol, outputRowIndex);
     i += 1;
   }
@@ -3718,14 +3717,14 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
 
 // ====================================== EXPOSED APIs ======================================
 int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
-                       STsdbReader** ppReader, const char* idstr) {
+                       SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) {
   STimeWindow window = pCond->twindows;
   if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) {
     pCond->twindows.skey += 1;
     pCond->twindows.ekey -= 1;
   }
 
-  int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, 4096, idstr);
+  int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, pResBlock->info.capacity, pResBlock, idstr);
   if (code != TSDB_CODE_SUCCESS) {
     goto _err;
   }
@@ -3751,7 +3750,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
     }
 
     // here we only need one more row, so the capacity is set to be ONE.
-    code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, idstr);
+    code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, pResBlock, idstr);
     if (code != TSDB_CODE_SUCCESS) {
       goto _err;
     }
@@ -3765,7 +3764,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
     }
     pCond->order = order;
 
-    code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, idstr);
+    code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, pResBlock, idstr);
     if (code != TSDB_CODE_SUCCESS) {
       goto _err;
     }
@@ -3837,10 +3836,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
   tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr);
   return code;
 
-_err:
+  _err:
   tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr);
   return code;
-}
+  }
 
 void tsdbReaderClose(STsdbReader* pReader) {
   if (pReader == NULL) {
@@ -3874,7 +3873,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
   taosMemoryFree(pSupInfo->colIds);
 
   taosArrayDestroy(pSupInfo->pColAgg);
-  for (int32_t i = 0; i < blockDataGetNumOfCols(pReader->pResBlock); ++i) {
+  for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) {
     if (pSupInfo->buildBuf[i] != NULL) {
       taosMemoryFreeClear(pSupInfo->buildBuf[i]);
     }
@@ -3891,8 +3890,6 @@ void tsdbReaderClose(STsdbReader* pReader) {
     clearBlockScanInfoBuf(&pReader->blockInfoBuf);
   }
 
-  blockDataDestroy(pReader->pResBlock);
-
   if (pReader->pFileReader != NULL) {
     tsdbDataFReaderClose(&pReader->pFileReader);
   }
@@ -4007,16 +4004,6 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
   return false;
 }
 
-bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) {
-  STableBlockScanInfo* pBlockScanInfo =
-      *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid));
-  if (pBlockScanInfo == NULL) {  // no data block for the table of given uid
-    return false;
-  }
-
-  return true;
-}
-
 static void setBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow) {
   ASSERT(pReader != NULL);
   *rows = pReader->pResBlock->info.rows;
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 7c37d66eb6..3f62191ca1 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1241,6 +1241,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
     }
   }
 
+  // set the output flag for each column in SColMatchInfo, according to the
   *numOfOutputCols = 0;
   int32_t num = LIST_LENGTH(pOutputNodeList->pSlots);
   for (int32_t i = 0; i < num; ++i) {
@@ -1599,20 +1600,22 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) {
 int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) {
   pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
   pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols);
+
   pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo));
-  if (pCond->colList == NULL) {
+  pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)*pCond->numOfCols);
+  if (pCond->colList == NULL || pCond->pSlotList == NULL) {
     terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    taosMemoryFreeClear(pCond->colList);
+    taosMemoryFreeClear(pCond->pSlotList);
     return terrno;
   }
 
-  // pCond->twindow = pTableScanNode->scanRange;
   // TODO: get it from stable scan node
   pCond->twindows = pTableScanNode->scanRange;
   pCond->suid = pTableScanNode->scan.suid;
   pCond->type = TIMEWINDOW_RANGE_CONTAINED;
   pCond->startVersion = -1;
   pCond->endVersion = -1;
-  //  pCond->type = pTableScanNode->scanFlag;
 
   int32_t j = 0;
   for (int32_t i = 0; i < pCond->numOfCols; ++i) {
@@ -1625,6 +1628,8 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
     pCond->colList[j].type = pColNode->node.resType.type;
     pCond->colList[j].bytes = pColNode->node.resType.bytes;
     pCond->colList[j].colId = pColNode->colId;
+
+    pCond->pSlotList[j] = pNode->slotId;
     j += 1;
   }
 
@@ -1962,7 +1967,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
 
 int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle,
                                 STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond,
-                                struct SExecTaskInfo* pTaskInfo) {
+                                SExecTaskInfo* pTaskInfo) {
   int64_t     st = taosGetTimestampUs();
   const char* idStr = GET_TASKID(pTaskInfo);
 
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 34bd9cf8ca..d675ac05d5 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1064,7 +1064,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
         int32_t        num = tableListGetSize(pTaskInfo->pTableInfoList);
 
         if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num,
-                           &pTableScanInfo->base.dataReader, NULL) < 0 ||
+                           pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 ||
             pTableScanInfo->base.dataReader == NULL) {
           ASSERT(0);
         }
@@ -1116,7 +1116,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
     int32_t        size = tableListGetSize(pTaskInfo->pTableInfoList);
     ASSERT(size == 1);
 
-    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->dataReader, NULL);
+    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->pRes, &pInfo->dataReader, NULL);
 
     cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
     strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 3e28798a3a..aa2682c490 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -1190,8 +1190,6 @@ void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) {
 //       T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
 //     }
 //
-//     tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo);
-//
 //     if (pQueryAttr->limit.offset > blockInfo.rows) {
 //       pQueryAttr->limit.offset -= blockInfo.rows;
 //       pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index ca3734d91e..1729c9f1e8 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -285,7 +285,7 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
   if (pLimit->offset > 0 && pLimitInfo->remainOffset > 0) {
     if (pLimitInfo->remainOffset >= pBlock->info.rows) {
       pLimitInfo->remainOffset -= pBlock->info.rows;
-      pBlock->info.rows = 0;
+      blockDataEmpty(pBlock);
       qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset, id);
     } else {
       blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset);
@@ -390,7 +390,6 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
     return terrno;
   }
 
-  relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
   doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
 
   // restore the previous value
@@ -638,16 +637,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
       continue;
     }
 
-    blockDataCleanup(pBlock);
-    SDataBlockInfo* pBInfo = &pBlock->info;
-
-    int32_t rows = 0;
-    tsdbRetrieveDataBlockInfo(pTableScanInfo->base.dataReader, &rows, &pBInfo->id.uid, &pBInfo->window);
-
-    blockDataEnsureCapacity(pBlock, rows);  // todo remove it latter
-    pBInfo->rows = rows;
-
-    ASSERT(pBInfo->id.uid != 0);
+    ASSERT(pBlock->info.id.uid != 0);
     pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
 
     uint32_t status = 0;
@@ -778,7 +768,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
       ASSERT(pInfo->base.dataReader == NULL);
 
       int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num,
-                                    (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo));
+                                    pInfo->pResBlock, (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo));
       if (code != TSDB_CODE_SUCCESS) {
         T_LONG_JMP(pTaskInfo->env, code);
       }
@@ -879,14 +869,14 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
   pInfo->base.scanFlag = MAIN_SCAN;
   pInfo->base.pdInfo.interval = extractIntervalInfo(pTableScanNode);
   pInfo->base.readHandle = *readHandle;
+  pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired;
+
   pInfo->sample.sampleRatio = pTableScanNode->ratio;
   pInfo->sample.seed = taosGetTimestampSec();
 
-  pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired;
-
   initResultSizeInfo(&pOperator->resultInfo, 4096);
   pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
-  blockDataEnsureCapacityNoClear(pInfo->pResBlock, pOperator->resultInfo.capacity);
+  blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
 
   code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
   if (code != TSDB_CODE_SUCCESS) {
@@ -993,10 +983,8 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
   SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo;
 
   SSDataBlock* pBlock = pTableScanInfo->pResBlock;
-  blockDataCleanup(pBlock);
-
   STsdbReader* pReader = NULL;
-  int32_t      code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader,
+  int32_t      code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, (STsdbReader**)&pReader,
                                      GET_TASKID(pTaskInfo));
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
@@ -2301,7 +2289,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
     if (pHandle->initTableReader) {
       pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
       pTSInfo->base.dataReader = NULL;
-      code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, &pTSInfo->base.dataReader, NULL);
+      code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, pTSInfo->pResBlock, &pTSInfo->base.dataReader, NULL);
       if (code != 0) {
         terrno = code;
         destroyTableScanOperatorInfo(pTableScanOp);
@@ -2531,11 +2519,10 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
   blockDataCleanup(pBlock);
 
   int64_t st = taosGetTimestampUs();
-
-  void*        p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex);
+  void*   p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex);
   SReadHandle* pHandle = &pInfo->base.readHandle;
 
-  int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->base.dataReader, GET_TASKID(pTaskInfo));
+  int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &pInfo->base.dataReader, GET_TASKID(pTaskInfo));
   if (code != 0) {
     T_LONG_JMP(pTaskInfo->env, code);
   }
@@ -2553,12 +2540,6 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
     }
 
     blockDataCleanup(pBlock);
-
-    int32_t rows = 0;
-    tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.id.uid, &pBlock->info.window);
-    blockDataEnsureCapacity(pBlock, rows);
-    pBlock->info.rows = rows;
-
     if (pQueryCond->order == TSDB_ORDER_ASC) {
       pQueryCond->twindows.skey = pBlock->info.window.ekey + 1;
     } else {
diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c
index 3d35326749..7ac6cee2af 100644
--- a/source/libs/executor/src/sysscanoperator.c
+++ b/source/libs/executor/src/sysscanoperator.c
@@ -1899,53 +1899,52 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC
 
 SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode,
                                                SExecTaskInfo* pTaskInfo) {
-    SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo));
-    SOperatorInfo*  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
-    if (pInfo == NULL || pOperator == NULL) {
-        pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _error;
-    }
+  SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo));
+  SOperatorInfo*  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  if (pInfo == NULL || pOperator == NULL) {
+    pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _error;
+  }
 
-    {
-        SQueryTableDataCond cond = {0};
+  pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc);
+  blockDataEnsureCapacity(pInfo->pResBlock, 1);
 
-        int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond);
-        if (code != TSDB_CODE_SUCCESS) {
-            goto _error;
-        }
-
-        STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
-        size_t          num = tableListGetSize(pTableListInfo);
-        void*           pList = tableListGetInfo(pTableListInfo, 0);
-
-        code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str);
-        cleanupQueryTableDataCond(&cond);
-        if (code != 0) {
-            goto _error;
-        }
-    }
-
-    pInfo->readHandle = *readHandle;
-    pInfo->uid = pBlockScanNode->suid;
-
-    pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc);
-    blockDataEnsureCapacity(pInfo->pResBlock, 1);
-
-    int32_t    numOfCols = 0;
-    SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols);
-    int32_t    code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
+  {
+    SQueryTableDataCond cond = {0};
+    int32_t             code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond);
     if (code != TSDB_CODE_SUCCESS) {
-        goto _error;
+      goto _error;
     }
 
-    setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false,
-                    OP_NOT_OPENED, pInfo, pTaskInfo);
-    pOperator->fpSet =
-            createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL);
-    return pOperator;
+    STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
+    size_t          num = tableListGetSize(pTableListInfo);
+    void*           pList = tableListGetInfo(pTableListInfo, 0);
 
-    _error:
-    taosMemoryFreeClear(pInfo);
-    taosMemoryFreeClear(pOperator);
-    return NULL;
+    code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str);
+    cleanupQueryTableDataCond(&cond);
+    if (code != 0) {
+      goto _error;
+    }
+  }
+
+  pInfo->readHandle = *readHandle;
+  pInfo->uid = pBlockScanNode->suid;
+
+  int32_t    numOfCols = 0;
+  SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols);
+  int32_t    code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false,
+                  OP_NOT_OPENED, pInfo, pTaskInfo);
+  pOperator->fpSet =
+      createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL);
+  return pOperator;
+
+_error:
+  taosMemoryFreeClear(pInfo);
+  taosMemoryFreeClear(pOperator);
+  return NULL;
 }
\ No newline at end of file
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 551d668763..5f077c98d7 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1739,8 +1739,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
   pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId;
 
   size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
-  initResultSizeInfo(&pOperator->resultInfo, 1024);
-  blockDataEnsureCapacityNoClear(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+  initResultSizeInfo(&pOperator->resultInfo, 512);
+  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
 
   int32_t    num = 0;
   SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
@@ -4319,7 +4319,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
   iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock;
 
   size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
-  initResultSizeInfo(&pOperator->resultInfo, 4096);
+  initResultSizeInfo(&pOperator->resultInfo, 512);
 
   int32_t    num = 0;
   SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index 98391d95f4..6074c72424 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -1782,7 +1782,7 @@ void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut,
     if (colDataIsNull_s(pOut->columnData, i)) {
       int8_t v = 0;
       colDataAppendInt8(pOut->columnData, i, &v);
-      colDataSetNotNull_f(pOut->columnData->nullbitmap, i);
+      colDataClearNull_f(pOut->columnData->nullbitmap, i);
     }
   }
   pOut->columnData->hasNull = false;
diff --git a/source/util/src/thash.c b/source/util/src/thash.c
index b07d485c87..e9548613aa 100644
--- a/source/util/src/thash.c
+++ b/source/util/src/thash.c
@@ -279,7 +279,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
     return NULL;
   }
 
-  void *p = taosMemoryCalloc(pHashObj->capacity, sizeof(SHashEntry));
+  void *p = taosMemoryMalloc(pHashObj->capacity * sizeof(SHashEntry));
   if (p == NULL) {
     taosArrayDestroy(pHashObj->pMemBlock);
     taosMemoryFree(pHashObj->hashList);
@@ -290,6 +290,9 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
 
   for (int32_t i = 0; i < pHashObj->capacity; ++i) {
     pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry));
+    pHashObj->hashList[i]->num = 0;
+    pHashObj->hashList[i]->latch = 0;
+    pHashObj->hashList[i]->next = NULL;
   }
 
   taosArrayPush(pHashObj->pMemBlock, &p);

From 9f55ca7f80fa3c8ff83fbc471c74353264fa82e3 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 17:45:33 +0800
Subject: [PATCH 15/32] test:update the test cases.

---
 tests/script/tsim/parser/groupby-basic.sim | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tests/script/tsim/parser/groupby-basic.sim b/tests/script/tsim/parser/groupby-basic.sim
index 4c70399e4b..2e436945e0 100644
--- a/tests/script/tsim/parser/groupby-basic.sim
+++ b/tests/script/tsim/parser/groupby-basic.sim
@@ -78,8 +78,8 @@ $ts2 = $tb2 . .ts
 
 print ===============================groupby_operation
 print
-print ==== select count(*), c1 from group_tb0 group by c1
-sql select count(*), c1 from group_tb0 group by c1
+print ==== select count(*), c1 from group_tb0 group by c1 order by c1
+sql select count(*), c1 from group_tb0 group by c1 order by c1
 print rows: $rows
 print $data00 $data01 $data02 $data03
 print $data10 $data11 $data12 $data13
@@ -98,18 +98,18 @@ endi
 if $data90 != 10 then
   return -1
 endi
-if $data01 != 7 then
+if $data01 != 0 then
   return -1
 endi
-if $data11 != 6 then
+if $data11 != 1 then
   return -1
 endi
-if $data91 != 3 then
+if $data91 != 9 then
   return -1
 endi
 
-print ==== select first(ts),c1 from group_tb0 group by c1;
-sql select first(ts),c1 from group_tb0 group by c1;
+print ==== select first(ts),c1 from group_tb0 group by c1 order by c1;
+sql select first(ts),c1 from group_tb0 group by c1 order by c1;
 print rows: $rows
 print $data00 $data01 $data02 $data03
 print $data10 $data11 $data12 $data13
@@ -120,16 +120,16 @@ if $row != 10 then
   return -1
 endi
 
-if $data00 != @22-01-01 00:00:00.007@ then
+if $data00 != @22-01-01 00:00:00.000@ then
   return -1
 endi
-if $data01 != 7 then
+if $data01 != 0 then
   return -1
 endi
-if $data90 != @22-01-01 00:00:00.003@ then
+if $data90 != @22-01-01 00:00:00.009@ then
   return -1
 endi
-if $data91 != 3 then
+if $data91 != 9 then
   return -1
 endi
 

From 8c62d5f9fa027c28382ec84641fdd37101cc38d2 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 18:34:26 +0800
Subject: [PATCH 16/32] fix(query): fix bug in tsdbread.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c        |  9 +++--
 source/libs/executor/src/scanoperator.c       | 13 ++++----
 source/libs/executor/src/timewindowoperator.c | 33 +++++++++++--------
 3 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 27d99bd42d..7c39e378cb 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -1621,7 +1621,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
   int64_t st = taosGetTimestampUs();
   int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader);
 
-  blockDataUpdateTsWindow(pBlock, 0);
+  blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotIds[0]);
   pBlock->info.id.uid = pBlockScanInfo->uid;
 
   setComposedBlockFlag(pReader, true);
@@ -2493,7 +2493,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
 
 _end:
   pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
-  blockDataUpdateTsWindow(pResBlock, 0);
+  blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotIds[0]);
 
   setComposedBlockFlag(pReader, true);
   double el = (taosGetTimestampUs() - st) / 1000.0;
@@ -3534,7 +3534,6 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
 int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow,
                              STableBlockScanInfo* pScanInfo) {
   int32_t outputRowIndex = pBlock->info.rows;
-  int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
   int64_t uid = pScanInfo->uid;
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
@@ -3549,7 +3548,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
     i += 1;
   }
 
-  while (i < numOfCols && j < pSchema->numOfCols) {
+  while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) {
     col_id_t colId = pSupInfo->colIds[i];
 
     if (colId == pSchema->columns[j].colId) {
@@ -3570,7 +3569,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
   }
 
   // set null value since current column does not exist in the "pSchema"
-  while (i < numOfCols) {
+  while (i < pSupInfo->numOfCols) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
     colDataAppendNULL(pColInfoData, outputRowIndex);
     i += 1;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 1729c9f1e8..9fc3575491 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -2513,10 +2513,8 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
   SExecTaskInfo*                  pTaskInfo = pOperator->pTaskInfo;
   int32_t                         readIdx = source->readerIdx;
   SSDataBlock*                    pBlock = source->inputBlock;
-  STableMergeScanInfo*            pTableScanInfo = pOperator->info;
 
-  SQueryTableDataCond* pQueryCond = taosArrayGet(pTableScanInfo->queryConds, readIdx);
-  blockDataCleanup(pBlock);
+  SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx);
 
   int64_t st = taosGetTimestampUs();
   void*   p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex);
@@ -2534,12 +2532,11 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
     }
 
     // process this data block based on the probabilities
-    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
+    bool processThisBlock = processBlockWithProbability(&pInfo->sample);
     if (!processThisBlock) {
       continue;
     }
 
-    blockDataCleanup(pBlock);
     if (pQueryCond->order == TSDB_ORDER_ASC) {
       pQueryCond->twindows.skey = pBlock->info.window.ekey + 1;
     } else {
@@ -2547,7 +2544,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
     }
 
     uint32_t status = 0;
-    loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status);
+    loadDataBlock(pOperator, &pInfo->base, pBlock, &status);
     //    code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status);
     if (code != TSDB_CODE_SUCCESS) {
       T_LONG_JMP(pTaskInfo->env, code);
@@ -2561,7 +2558,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
     pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
 
     pOperator->resultInfo.totalRows += pBlock->info.rows;
-    pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
+    pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
 
     tsdbReaderClose(pInfo->base.dataReader);
     pInfo->base.dataReader = NULL;
@@ -2641,6 +2638,8 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
     param.readerIdx = i;
     param.pOperator = pOperator;
     param.inputBlock = createOneDataBlock(pInfo->pResBlock, false);
+    blockDataEnsureCapacity(param.inputBlock, pOperator->resultInfo.capacity);
+
     taosArrayPush(pInfo->sortSourceParams, &param);
 
     SQueryTableDataCond cond;
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 5f077c98d7..52f06ffab4 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1659,22 +1659,11 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
   // the primary timestamp column
   bool needed = false;
 
-  for (int32_t i = 0; i < numOfCols; ++i) {
+  for(int32_t i = 0; i < numOfCols; ++i) {
     SExprInfo* pExpr = pCtx[i].pExpr;
-
     if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
-      SFunctParam* pParam = &pExpr->base.pParam[0];
-
-      SColumn c = *pParam->pCol;
-      taosArrayPush(pInfo->pInterpCols, &c);
       needed = true;
-
-      SGroupKeys key = {0};
-      key.bytes = c.bytes;
-      key.type = c.type;
-      key.isNull = false;
-      key.pData = taosMemoryCalloc(1, c.bytes);
-      taosArrayPush(pInfo->pPrevValues, &key);
+      break;
     }
   }
 
@@ -1699,6 +1688,24 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
     }
   }
 
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SExprInfo* pExpr = pCtx[i].pExpr;
+
+    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
+      SFunctParam* pParam = &pExpr->base.pParam[0];
+
+      SColumn c = *pParam->pCol;
+      taosArrayPush(pInfo->pInterpCols, &c);
+
+      SGroupKeys key = {0};
+      key.bytes = c.bytes;
+      key.type = c.type;
+      key.isNull = false;
+      key.pData = taosMemoryCalloc(1, c.bytes);
+      taosArrayPush(pInfo->pPrevValues, &key);
+    }
+  }
+
   return needed;
 }
 

From 740daefdc54a49a0451566a28db77b6cee9098f8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 18:39:15 +0800
Subject: [PATCH 17/32] fix(query): fix a typo.

---
 source/libs/executor/src/scanoperator.c | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 9fc3575491..87aacfdd34 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -996,15 +996,13 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
   if (hasBlock) {
     SDataBlockInfo* pBInfo = &pBlock->info;
 
-    int32_t rows = 0;
-    tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window);
+//    int32_t rows = 0;
+//    tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window);
 
     SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL);
-    blockDataEnsureCapacity(pBlock, rows);
-    pBlock->info.rows = rows;
 
-    relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true);
-    doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, rows);
+//    relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true);
+    doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
 
     pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->id.uid);
   }

From ee5d0a384f8df9f342e984a3be9de6c504aa2038 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 1 Dec 2022 19:05:33 +0800
Subject: [PATCH 18/32] fix: refactor table scan count

---
 source/libs/executor/inc/executorimpl.h |  12 +-
 source/libs/executor/src/scanoperator.c | 228 ++++++++++++++----------
 2 files changed, 141 insertions(+), 99 deletions(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index 3896a8a419..9d797f8520 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -489,8 +489,8 @@ typedef struct STableCountScanSupp {
 
   bool groupByDbName;
   bool groupByStbName;
-  char    dbName[TSDB_DB_NAME_LEN];
-  char    stbName[TSDB_TABLE_NAME_LEN];
+  char    dbNameFilter[TSDB_DB_NAME_LEN];
+  char    stbNameFilter[TSDB_TABLE_NAME_LEN];
 
 } STableCountScanSupp;
 
@@ -498,16 +498,10 @@ typedef struct STableCountScanOperatorInfo {
   SReadHandle  readHandle;
   SSDataBlock* pRes;
 
-  SName tableName;
-
-  SNodeList* groupTags;
-  SNodeList* scanCols;
-  SNodeList* pseudoCols;
-
   STableCountScanSupp supp;
 
   int32_t   currGrpIdx;
-  SArray*   stbUidList; // when group by db_name and stable_name
+  SArray*   stbUidList; // when group by db_name and/or stable_name
 } STableCountScanOperatorInfo;
 
 typedef struct SOptrBasicInfo {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 3dd027ad04..07ef30f8c5 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -486,10 +486,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
     code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid);
     if (code != TSDB_CODE_SUCCESS) {
       if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
-        qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid,
-              tstrerror(terrno), idStr);
+        qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s",
+              pBlock->info.id.uid, tstrerror(terrno), idStr);
       } else {
-        qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr);
+        qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno),
+               idStr);
       }
       metaReaderClear(&mr);
       return terrno;
@@ -1435,8 +1436,8 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
                                        NULL);
       if (closedWin && pInfo->partitionSup.needCalc) {
         gpId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId);
-        appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid, &gpId,
-                                         NULL);
+        appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid,
+                                         &gpId, NULL);
       }
     }
   }
@@ -2461,7 +2462,8 @@ static void destroyTagScanOperatorInfo(void* param) {
   taosMemoryFreeClear(param);
 }
 
-SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
+SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
+                                         SExecTaskInfo* pTaskInfo) {
   STagScanInfo*  pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
   SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL) {
@@ -2917,6 +2919,21 @@ _error:
 // TableCountScanOperator
 static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator);
 static void         destoryTableCountScanOperator(void* param);
+static void         buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
+                                                   SSDataBlock* pRes, char* dbName, tb_uid_t stbUid);
+static void         buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
+                                                   SSDataBlock* pRes, char* dbName);
+static void         buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+                                              STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName);
+static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+                                        STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName);
+static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+                                           STableCountScanSupp* pSupp, SSDataBlock* pRes);
+static void         buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+                                                STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum,
+                                                size_t perfdbTableNum);
+static void         buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes,
+                                               size_t infodbTableNum, size_t perfdbTableNum);
 static const char*  GROUP_TAG_DB_NAME = "db_name";
 static const char*  GROUP_TAG_STABLE_NAME = "stable_name";
 
@@ -2978,8 +2995,8 @@ int32_t tblCountScanGetInputs(SNodeList* groupTags, SName* tableName, STableCoun
       }
     }
   } else {
-    strncpy(supp->dbName, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN);
-    strncpy(supp->stbName, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN);
+    strncpy(supp->dbNameFilter, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN);
+    strncpy(supp->stbNameFilter, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN);
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -3061,7 +3078,7 @@ void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char*
   if (pSupp->stbNameSlotId != -1) {
     SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId);
     if (strlen(stbName) != 0) {
-      char             varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
+      char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
       strncpy(varDataVal(varStbName), stbName, strlen(stbName));
       varDataSetLen(varStbName, strlen(stbName));
       colDataAppend(colInfoData, 0, varStbName, false);
@@ -3087,33 +3104,43 @@ static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountSc
   getPerfDbMeta(NULL, &perfdbTableNum);
 
   if (pSupp->groupByDbName) {
-    if (pInfo->currGrpIdx == 0) {
-      uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB));
-      pRes->info.id.groupId = groupId;
-      fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes);
-    } else if (pInfo->currGrpIdx == 1) {
-      uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB));
-      pRes->info.id.groupId = groupId;
-      fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes);
-    } else {
-      setOperatorCompleted(pOperator);
-      return NULL;
-    }
-    pInfo->currGrpIdx++;
+    buildSysDbGroupedTableCount(pOperator, pInfo, pSupp, pRes, infodbTableNum, perfdbTableNum);
     return (pRes->info.rows > 0) ? pRes : NULL;
   } else {
-    if (strcmp(pSupp->dbName, TSDB_INFORMATION_SCHEMA_DB) == 0) {
-      fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes);
-    } else if (strcmp(pSupp->dbName, TSDB_PERFORMANCE_SCHEMA_DB) == 0) {
-      fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes);
-    } else if (strlen(pSupp->dbName) == 0) {
-      fillTableCountScanDataBlock(pSupp, "", "", infodbTableNum + perfdbTableNum, pRes);
-    }
-    setOperatorCompleted(pOperator);
+    buildSysDbFilterTableCount(pOperator, pSupp, pRes, infodbTableNum, perfdbTableNum);
     return (pRes->info.rows > 0) ? pRes : NULL;
   }
 }
 
+static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes,
+                                       size_t infodbTableNum, size_t perfdbTableNum) {
+  if (strcmp(pSupp->dbNameFilter, TSDB_INFORMATION_SCHEMA_DB) == 0) {
+    fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes);
+  } else if (strcmp(pSupp->dbNameFilter, TSDB_PERFORMANCE_SCHEMA_DB) == 0) {
+    fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes);
+  } else if (strlen(pSupp->dbNameFilter) == 0) {
+    fillTableCountScanDataBlock(pSupp, "", "", infodbTableNum + perfdbTableNum, pRes);
+  }
+  setOperatorCompleted(pOperator);
+}
+
+static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+                                        STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum,
+                                        size_t perfdbTableNum) {
+  if (pInfo->currGrpIdx == 0) {
+    uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB));
+    pRes->info.id.groupId = groupId;
+    fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes);
+  } else if (pInfo->currGrpIdx == 1) {
+    uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB));
+    pRes->info.id.groupId = groupId;
+    fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes);
+  } else {
+    setOperatorCompleted(pOperator);
+  }
+  pInfo->currGrpIdx++;
+}
+
 static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) {
   SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
   STableCountScanOperatorInfo* pInfo = pOperator->info;
@@ -3128,89 +3155,110 @@ static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) {
     return buildSysDbTableCount(pOperator, pInfo);
   }
 
+  return buildVnodeDbTableCount(pOperator, pInfo, pSupp, pRes);
+}
+
+static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+                                           STableCountScanSupp* pSupp, SSDataBlock* pRes) {
   const char* db = NULL;
   int32_t     vgId = 0;
   char        dbName[TSDB_DB_NAME_LEN] = {0};
 
-  {
-    // get dbname
-    vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
-    SName            sn = {0};
-    tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB);
-    tNameGetDbName(&sn, dbName);
-  }
+  // get dbname
+  vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId);
+  SName sn = {0};
+  tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB);
+  tNameGetDbName(&sn, dbName);
+
   if (pSupp->groupByDbName) {
-    if (pSupp->groupByStbName) {
-      if (pInfo->stbUidList == NULL) {
-        pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t));
-        if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) {
-          qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr());
-        }
+    buildVnodeGroupedTableCount(pOperator, pInfo, pSupp, pRes, vgId, dbName);
+  } else {
+    buildVnodeFilteredTbCount(pOperator, pInfo, pSupp, pRes, dbName);
+  }
+  return pRes->info.rows > 0 ? pRes : NULL;
+}
+
+static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+                                        STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) {
+  if (pSupp->groupByStbName) {
+    if (pInfo->stbUidList == NULL) {
+      pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t));
+      if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) {
+        qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr());
       }
-      if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) {
-        tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx);
+    }
+    if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) {
+      tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx);
+      buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid);
 
-        char stbName[TSDB_TABLE_NAME_LEN] = {0};
-        metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName);
+      pInfo->currGrpIdx++;
+    } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) {
+      buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName);
 
-        char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
-        snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName);
-        uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
-        pRes->info.id.groupId = groupId;
-
-        SMetaStbStats stats = {0};
-        metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats);
-        int64_t ctbNum = stats.ctbNum;
-
-        fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes);
-        
-        pInfo->currGrpIdx++;
-      } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) {
-        char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
-        snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, "");
-        uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
-        pRes->info.id.groupId = groupId;
-        int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta);
-        fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes);
-
-        pInfo->currGrpIdx++;
-      } else {
-        setOperatorCompleted(pOperator);
-        return NULL;
-      }
+      pInfo->currGrpIdx++;
     } else {
-      uint64_t groupId = calcGroupId(dbName, strlen(dbName));
-      pRes->info.id.groupId = groupId;
-      int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta);
-      fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes);
       setOperatorCompleted(pOperator);
     }
   } else {
-    if (strlen(pSupp->dbName) != 0) {
-      if (strlen(pSupp->stbName) != 0) {
-        tb_uid_t      uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbName);
-        SMetaStbStats stats = {0};
-        metaGetStbStats(pInfo->readHandle.meta, uid, &stats);
-        int64_t ctbNum = stats.ctbNum;
-        fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbName, ctbNum, pRes);
-      } else {
-        int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta);
-        fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes);
-      }
+    uint64_t groupId = calcGroupId(dbName, strlen(dbName));
+    pRes->info.id.groupId = groupId;
+    int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta);
+    fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes);
+    setOperatorCompleted(pOperator);
+  }
+}
+
+static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
+                                      STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) {
+  if (strlen(pSupp->dbNameFilter) != 0) {
+    if (strlen(pSupp->stbNameFilter) != 0) {
+      tb_uid_t      uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter);
+      SMetaStbStats stats = {0};
+      metaGetStbStats(pInfo->readHandle.meta, uid, &stats);
+      int64_t ctbNum = stats.ctbNum;
+      fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes);
     } else {
       int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta);
       fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes);
     }
-    setOperatorCompleted(pOperator);
+  } else {
+    int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta);
+    fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes);
   }
-  return pRes->info.rows > 0 ? pRes : NULL;
+  setOperatorCompleted(pOperator);
+}
+
+static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
+                                           SSDataBlock* pRes, char* dbName) {
+  char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
+  snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, "");
+  uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
+  pRes->info.id.groupId = groupId;
+  int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta);
+  fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes);
+}
+
+static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
+                                           SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) {
+  char stbName[TSDB_TABLE_NAME_LEN] = {0};
+  metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName);
+
+  char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
+  snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName);
+  uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
+  pRes->info.id.groupId = groupId;
+
+  SMetaStbStats stats = {0};
+  metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats);
+  int64_t ctbNum = stats.ctbNum;
+
+  fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes);
 }
 
 static void destoryTableCountScanOperator(void* param) {
   STableCountScanOperatorInfo* pTableCountScanInfo = param;
   blockDataDestroy(pTableCountScanInfo->pRes);
 
-  nodesDestroyList(pTableCountScanInfo->groupTags);
   taosArrayDestroy(pTableCountScanInfo->stbUidList);
   taosMemoryFreeClear(param);
 }

From efcc3b06bb3af6791d670922c290ec5de297d3e9 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 20:01:09 +0800
Subject: [PATCH 19/32] fix(query): fix memory leak.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 1 +
 source/libs/executor/src/executil.c    | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 7c39e378cb..f86540e3e2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -3870,6 +3870,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
 
   taosMemoryFreeClear(pSupInfo->plist);
   taosMemoryFree(pSupInfo->colIds);
+  taosMemoryFree(pSupInfo->slotIds);
 
   taosArrayDestroy(pSupInfo->pColAgg);
   for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) {
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 3f62191ca1..6c219f8bd4 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1637,7 +1637,10 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
   return TSDB_CODE_SUCCESS;
 }
 
-void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFreeClear(pCond->colList); }
+void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) {
+  taosMemoryFreeClear(pCond->colList);
+  taosMemoryFreeClear(pCond->pSlotList);
+}
 
 int32_t convertFillType(int32_t mode) {
   int32_t type = TSDB_FILL_NONE;

From 1750b5cc92332fd68d551bdfc83fcec7a9ca03f1 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 1 Dec 2022 20:34:06 +0800
Subject: [PATCH 20/32] remove invalid table

---
 source/libs/executor/src/executil.c | 18 +++++++++---------
 source/util/src/tarray.c            |  4 ++--
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 8a85885d98..d5bd2c6010 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -438,13 +438,12 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
       goto end;
     }
   }
+  removeInvalidTable(uidList, tags);
 
   int32_t rows = taosArrayGetSize(uidList);
   if (rows == 0) {
     goto end;
   }
-  //  int64_t stt1 = taosGetTimestampUs();
-  //  qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt);
 
   code = blockDataEnsureCapacity(pResBlock, rows);
   if (code != TSDB_CODE_SUCCESS) {
@@ -452,7 +451,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
     goto end;
   }
 
-  //  int64_t st = taosGetTimestampUs();
   for (int32_t i = 0; i < rows; i++) {
     int64_t* uid = taosArrayGet(uidList, i);
     for (int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++) {
@@ -467,7 +465,9 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
 #endif
       } else {
         void* tag = taosHashGet(tags, uid, sizeof(int64_t));
-        ASSERT(tag);
+        if (tag == NULL) {
+          continue;
+        }
         STagVal tagVal = {0};
         tagVal.cid = pColInfo->info.colId;
         const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal);
@@ -923,14 +923,14 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
       return -1;
     }
 
-    SArray* pTbList = getTableNameList(pList);
-    int32_t numOfTables = taosArrayGetSize(pTbList);
-    SHashObj *uHash = NULL;
+    SArray*   pTbList = getTableNameList(pList);
+    int32_t   numOfTables = taosArrayGetSize(pTbList);
+    SHashObj* uHash = NULL;
     size_t    listlen = taosArrayGetSize(list);  // len > 0 means there already have uids
     if (listlen > 0) {
       uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
       for (int i = 0; i < listlen; i++) {
-        int64_t *uid = taosArrayGet(list, i);
+        int64_t* uid = taosArrayGet(list, i);
         taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i));
       }
     }
@@ -2011,4 +2011,4 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag) {
   char* pBuf = NULL;
   qDebug("%s", dumpBlockData(pBlock, flag, &pBuf));
   taosMemoryFree(pBuf);
-}
\ No newline at end of file
+}
diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index ae25786375..4bd8294423 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -319,9 +319,9 @@ SArray* taosArrayDup(const SArray* pSrc, __array_item_dup_fn_t fn) {
   } else {
     ASSERT(pSrc->elemSize == sizeof(void*));
 
-    for(int32_t i = 0; i < pSrc->size; ++i) {
+    for (int32_t i = 0; i < pSrc->size; ++i) {
       void* p = fn(taosArrayGetP(pSrc, i));
-      memcpy(((char*)dst->pData )+ i * dst->elemSize, &p, dst->elemSize);
+      memcpy(((char*)dst->pData) + i * dst->elemSize, &p, dst->elemSize);
     }
   }
 

From 24614d03bd2ef21b454d8b9c558fb29977418960 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 21:35:01 +0800
Subject: [PATCH 21/32] fix(query): set correct sma retrieve procedure.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c  | 42 ++++++++++++++-----------
 source/libs/executor/src/scanoperator.c |  2 ++
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index f86540e3e2..24e9181ae6 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -84,7 +84,7 @@ typedef struct SIOCostSummary {
 typedef struct SBlockLoadSuppInfo {
   SArray*          pColAgg;
   SColumnDataAgg   tsColAgg;
-  SColumnDataAgg** plist;
+  SColumnDataAgg** plist;     // todo remove this
   int16_t*         colIds;    // column ids for loading file block data
   int16_t*         slotIds;   // the ordinal index in the destination SSDataBlock
   int32_t          numOfCols;
@@ -4041,12 +4041,11 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
   }
 
   SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter);
-
-  SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
-  //  int64_t   stime = taosGetTimestampUs();
-
   SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
 
+  ASSERT(pReader->pResBlock->info.id.uid == pFBlock->uid);
+
+  SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
   if (tDataBlkHasSma(pBlock)) {
     code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg);
     if (code != TSDB_CODE_SUCCESS) {
@@ -4071,7 +4070,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
   pSup->plist[0] = pTsAgg;
 
   // update the number of NULL data rows
-  size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock);
+  size_t numOfCols = pSup->numOfCols;
 
   int32_t i = 0, j = 0;
   size_t  size = taosArrayGetSize(pSup->pColAgg);
@@ -4095,43 +4094,48 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
   }
 #else
 
+  SSDataBlock* pResBlock = pReader->pResBlock;
+  if (pResBlock->pBlockAgg == NULL) {
+    size_t num = taosArrayGetSize(pResBlock->pDataBlock);
+    pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg));
+  }
+
   // fill the all null data column
-  SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg));
+//  SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg));
 
   while (j < numOfCols && i < size) {
     SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
     if (pAgg->colId == pSup->colIds[j]) {
-      taosArrayPush(pNewAggList, pAgg);
+      pResBlock->pBlockAgg[pSup->slotIds[j]] = pAgg;
       i += 1;
       j += 1;
     } else if (pAgg->colId < pSup->colIds[j]) {
       i += 1;
     } else if (pSup->colIds[j] < pAgg->colId) {
       if (pSup->colIds[j] == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        taosArrayPush(pNewAggList, &pSup->tsColAgg);
+        pResBlock->pBlockAgg[pSup->slotIds[j]] = &pSup->tsColAgg;
       } else {
       // all date in this block are null
         SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow};
-        taosArrayPush(pNewAggList, &nullColAgg);
+        taosArrayPush(pSup->pColAgg, &nullColAgg);
+
+        pResBlock->pBlockAgg[pSup->slotIds[j]] = taosArrayGetLast(pSup->pColAgg);
       }
       j += 1;
     }
   }
 
-  taosArrayClear(pSup->pColAgg);
-  taosArrayAddAll(pSup->pColAgg, pNewAggList);
+//  taosArrayClear(pSup->pColAgg);
 
-  size_t num = taosArrayGetSize(pSup->pColAgg);
-  for(int32_t k = 0; k < num; ++k) {
-    pSup->plist[k] = taosArrayGet(pSup->pColAgg, k);
-  }
-
-  taosArrayDestroy(pNewAggList);
+//  size_t num = taosArrayGetSize(pSup->pColAgg);
+//  for(int32_t k = 0; k < num; ++k) {
+//    pSup->plist[k] = taosArrayGet(pSup->pColAgg, k);
+//  }
 
 #endif
 
   pReader->cost.smaDataLoad += 1;
-  *pBlockStatis = pSup->plist;
+//  *pBlockStatis = pSup->plist;
 
   tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr);
   return code;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 87aacfdd34..b0104c6025 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -236,6 +236,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
     return false;
   }
 
+#if 0
   //  if (allColumnsHaveAgg == true) {
   int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
 
@@ -256,6 +257,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
 
     pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
   }
+#endif
 
   return true;
 }

From 1faf616f1d089ce592c8dcd56326d7140c75aea0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 22:17:53 +0800
Subject: [PATCH 22/32] fix(query): fix null ptr access in evaluation block
 distribution.

---
 source/libs/executor/src/sysscanoperator.c | 37 ++++++++++++----------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c
index 7ac6cee2af..96f21babe6 100644
--- a/source/libs/executor/src/sysscanoperator.c
+++ b/source/libs/executor/src/sysscanoperator.c
@@ -1874,27 +1874,30 @@ static void destroyBlockDistScanOperatorInfo(void* param) {
 }
 
 static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) {
-    memset(pCond, 0, sizeof(SQueryTableDataCond));
+  memset(pCond, 0, sizeof(SQueryTableDataCond));
 
-    pCond->order = TSDB_ORDER_ASC;
-    pCond->numOfCols = 1;
-    pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo));
-    if (pCond->colList == NULL) {
-        terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
-        return terrno;
-    }
+  pCond->order = TSDB_ORDER_ASC;
+  pCond->numOfCols = 1;
+  pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo));
+  pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t));
+  if (pCond->colList == NULL || pCond->pSlotList == NULL) {
+    terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    return terrno;
+  }
 
-    pCond->colList->colId = 1;
-    pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP;
-    pCond->colList->bytes = sizeof(TSKEY);
+  pCond->colList->colId = 1;
+  pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP;
+  pCond->colList->bytes = sizeof(TSKEY);
 
-    pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
-    pCond->suid = uid;
-    pCond->type = TIMEWINDOW_RANGE_CONTAINED;
-    pCond->startVersion = -1;
-    pCond->endVersion = -1;
+  pCond->pSlotList[0] = 0;
 
-    return TSDB_CODE_SUCCESS;
+  pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
+  pCond->suid = uid;
+  pCond->type = TIMEWINDOW_RANGE_CONTAINED;
+  pCond->startVersion = -1;
+  pCond->endVersion = -1;
+
+  return TSDB_CODE_SUCCESS;
 }
 
 SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode,

From 83782fb27bf679b44353511e0eb70393e4193902 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 2 Dec 2022 10:07:48 +0800
Subject: [PATCH 23/32] fix(query): fix invalid tag fill and clear nullbitmp

---
 include/common/tdatablock.h                |  6 ++---
 source/common/src/tdatablock.c             |  8 +++----
 source/libs/executor/src/tfill.c           | 28 +++++++++++++++++-----
 tests/script/tsim/parser/lastrow_query.sim |  4 ++--
 tests/script/tsim/valgrind/checkUdf.sim    |  3 ++-
 5 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 4208d85648..e1d3b01611 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -41,9 +41,9 @@ typedef struct SBlockOrderInfo {
     BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
   } while (0)
 
-#define colDataClearNull_f(bm_, r_)                  \
-  do {                                                \
-    BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \
+#define colDataClearNull_f(bm_, r_)                             \
+  do {                                                          \
+    BMCharPos(bm_, r_) &= ((char)(~(1u << (7u - BitPos(r_))))); \
   } while (0)
 
 #define colDataIsNull_var(pColumnInfoData, row)  (pColumnInfoData->varmeta.offset[row] == -1)
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index c300c3e6f0..e9be89a68b 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1679,15 +1679,15 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_
     pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
     memset(&pColInfoData->varmeta.offset[n], 0, total - n);
   } else {  // reset the bitmap value
-    int32_t stopIndex = BitmapLen(n) * 8;
-    for(int32_t i = n + 1; i < stopIndex; ++i) {
+    /*int32_t stopIndex = BitmapLen(n) * 8;
+    for(int32_t i = n; i < stopIndex; ++i) {
       colDataClearNull_f(pColInfoData->nullbitmap, i);
     }
 
     int32_t remain = BitmapLen(total) - BitmapLen(n);
     if (remain > 0) {
-      memset(pColInfoData->nullbitmap, 0, remain);
-    }
+      memset(pColInfoData->nullbitmap+BitmapLen(n), 0, remain);
+    }*/
   }
 }
 
diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c
index 42b1d058b5..4c8967744e 100644
--- a/source/libs/executor/src/tfill.c
+++ b/source/libs/executor/src/tfill.c
@@ -60,8 +60,18 @@ static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowInd
     if (pCol->notFillCol) {
       bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstColInfo, rowIndex);
       if (!filled) {
-        SArray*     p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal;
-        SGroupKeys* pKey = taosArrayGet(p, i);
+        SRowVal* p = NULL;
+        if (FILL_IS_ASC_FILL(pFillInfo)) {
+          if (pFillInfo->prev.key != 0) {
+            p = &pFillInfo->prev;  // prev has been set value
+          } else { // otherwise, use the value in the next row
+            p = &pFillInfo->next;
+          }
+        } else {
+          p = &pFillInfo->next;
+        }
+
+        SGroupKeys* pKey = taosArrayGet(p->pRowVal, i);
         doSetVal(pDstColInfo, rowIndex, pKey);
       }
     } else {
@@ -261,7 +271,10 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) {
 
 static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull);
 
-static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray* pRow) {
+static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal) {
+  SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, pFillInfo->srcTsSlotId);
+  pRowVal->key = ((int64_t*)pTsCol->pData)[rowIndex];
+
   for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
     int32_t type = pFillInfo->pFillCol[i].pExpr->pExpr->nodeType;
     if (type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) {
@@ -272,7 +285,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray
       bool  isNull = colDataIsNull_s(pSrcCol, rowIndex);
       char* p = colDataGetData(pSrcCol, rowIndex);
 
-      saveColData(pRow, i, p, isNull);
+      saveColData(pRowVal->pRowVal, i, p, isNull);
     } else {
       ASSERT(0);
     }
@@ -296,7 +309,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t
 
     // set the next value for interpolation
     if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) {
-      copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next.pRowVal);
+      copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, &pFillInfo->next);
     }
 
     if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) &&
@@ -318,7 +331,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t
 
       if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) {
         int32_t nextRowIndex = pFillInfo->index + 1;
-        copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, pFillInfo->next.pRowVal);
+        copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next);
       }
 
       // copy rows to dst buffer
@@ -334,6 +347,9 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t
         if (!colDataIsNull_s(pSrc, pFillInfo->index)) {
           colDataAppend(pDst, index, src, false);
           saveColData(pFillInfo->prev.pRowVal, i, src, false);
+          if (pFillInfo->srcTsSlotId == dstSlotId) {
+            pFillInfo->prev.key = *(int64_t*)src;
+          }
         } else {  // the value is null
           if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
             colDataAppend(pDst, index, (const char*)&pFillInfo->currentKey, false);
diff --git a/tests/script/tsim/parser/lastrow_query.sim b/tests/script/tsim/parser/lastrow_query.sim
index 5f557fd7bd..9ca9d407b8 100644
--- a/tests/script/tsim/parser/lastrow_query.sim
+++ b/tests/script/tsim/parser/lastrow_query.sim
@@ -70,10 +70,10 @@ sql select _wstart, t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00
 if $row != 2 then
   return -1
 endi
-if $data01 != NULL then
+if $data01 != 8 then
   return -1
 endi
-if $data02 != NULL then
+if $data02 != 8 then
   return -1
 endi
 if  $data03 != NULL then
diff --git a/tests/script/tsim/valgrind/checkUdf.sim b/tests/script/tsim/valgrind/checkUdf.sim
index 594d649714..caf316bd86 100644
--- a/tests/script/tsim/valgrind/checkUdf.sim
+++ b/tests/script/tsim/valgrind/checkUdf.sim
@@ -121,12 +121,13 @@ if $data01 != 152.420471066 then
   return -1
 endi
 
-sql select udf2(f2) from udf.t2 group by 1-udf1(f1);
+sql select udf2(f2) from udf.t2 group by 1-udf1(f1) order by 1-udf1(f1)
 print $rows , $data00 , $data10
 if $rows != 2 then
   return -1
 endi
 if $data00 != 2.000000000 then
+  print expect 2.000000000 , actual: $data00
   return -1
 endi
 if $data10 != 12.083045974 then

From 42a512c39fa8a456496d1b154dc5be22b1bbb1bf Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 2 Dec 2022 15:52:32 +0800
Subject: [PATCH 24/32] refactor: do some internal refactor.

---
 source/dnode/vnode/inc/vnode.h          |  22 +--
 source/dnode/vnode/src/tsdb/tsdbRead.c  | 176 +++++++++++-------------
 source/libs/executor/src/executor.c     |  18 ++-
 source/libs/executor/src/scanoperator.c |  37 ++---
 tests/system-test/2-query/unique.py     |   6 +-
 5 files changed, 116 insertions(+), 143 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 53b8eadb33..64c15c6c0e 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -161,17 +161,17 @@ int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t n
 int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
                        SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr);
 
-void     tsdbReaderClose(STsdbReader *pReader);
-bool     tsdbNextDataBlock(STsdbReader *pReader);
-void     tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow);
-int32_t  tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
-SArray  *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
-int32_t  tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
-int32_t  tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
-int64_t  tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
-void    *tsdbGetIdx(SMeta *pMeta);
-void    *tsdbGetIvtIdx(SMeta *pMeta);
-uint64_t getReaderMaxVersion(STsdbReader *pReader);
+void         tsdbReaderClose(STsdbReader *pReader);
+bool         tsdbNextDataBlock(STsdbReader *pReader);
+void         tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow);
+int32_t      tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockSMA, bool *allHave);
+SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
+int32_t      tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
+int32_t      tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
+int64_t      tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
+void        *tsdbGetIdx(SMeta *pMeta);
+void        *tsdbGetIvtIdx(SMeta *pMeta);
+uint64_t     getReaderMaxVersion(STsdbReader *pReader);
 
 int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
                                 uint64_t suid, void **pReader);
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 24e9181ae6..859ae026a2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -81,12 +81,15 @@ typedef struct SIOCostSummary {
   double  createScanInfoList;
 } SIOCostSummary;
 
+typedef struct SColInfo {
+  int16_t colId;
+  int16_t slotId;
+} SColInfo;
+
 typedef struct SBlockLoadSuppInfo {
   SArray*          pColAgg;
   SColumnDataAgg   tsColAgg;
-  SColumnDataAgg** plist;     // todo remove this
-  int16_t*         colIds;    // column ids for loading file block data
-  int16_t*         slotIds;   // the ordinal index in the destination SSDataBlock
+  SColInfo*        colInfo;    // column ids for loading file block data
   int32_t          numOfCols;
   char**           buildBuf;  // build string tmp buffer, todo remove it later after all string format being updated.
   bool             smaValid;  // the sma on all queried columns are activated
@@ -159,6 +162,7 @@ struct STsdbReader {
   STsdb*             pTsdb;
   uint64_t           suid;
   int16_t            order;
+  bool               freeBlock;
   STimeWindow        window;  // the primary query time window that applies to all queries
   SSDataBlock*       pResBlock;
   int32_t            capacity;
@@ -218,22 +222,21 @@ static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWi
 static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) {
   pSupInfo->smaValid = true;
   pSupInfo->numOfCols = numOfCols;
-  pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t));
-  pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES);
-  pSupInfo->slotIds = taosMemoryMalloc(numOfCols * sizeof(int16_t));
-  if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL || pSupInfo->slotIds == NULL) {
-    taosMemoryFree(pSupInfo->colIds);
-    taosMemoryFree(pSupInfo->buildBuf);
-    taosMemoryFree(pSupInfo->slotIds);
+  pSupInfo->colInfo = taosMemoryMalloc(numOfCols * (sizeof(SColInfo) + POINTER_BYTES));
+  if (pSupInfo->colInfo == NULL) {
+    taosMemoryFree(pSupInfo->colInfo);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  pSupInfo->buildBuf = (char**)((char*)pSupInfo->colInfo + (sizeof(SColInfo) * numOfCols));
   for (int32_t i = 0; i < numOfCols; ++i) {
-    pSupInfo->colIds[i] = pCols[i].colId;
-    pSupInfo->slotIds[i] = pSlotIdList[i];
+    pSupInfo->colInfo[i].colId = pCols[i].colId;
+    pSupInfo->colInfo[i].slotId = pSlotIdList[i];
 
     if (IS_VAR_DATA_TYPE(pCols[i].type)) {
       pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes);
+    } else {
+      pSupInfo->buildBuf[i] = NULL;
     }
   }
 
@@ -245,7 +248,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
 
   while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) {
     STColumn* pTCol = &pSchema->columns[i];
-    if (pTCol->colId == pSupInfo->colIds[j]) {
+    if (pTCol->colId == pSupInfo->colInfo[j].colId) {
       if (!IS_BSMA_ON(pTCol)) {
         pSupInfo->smaValid = false;
         return;
@@ -253,7 +256,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
 
       i += 1;
       j += 1;
-    } else if (pTCol->colId < pSupInfo->colIds[j]) {
+    } else if (pTCol->colId < pSupInfo->colInfo[j].colId) {
       // do nothing
       i += 1;
     } else {
@@ -455,7 +458,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
   if (pLReader->pInfo == NULL) {
     // here we ignore the first column, which is always be the primary timestamp column
     pLReader->pInfo =
-        tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1);
+        tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1);
     if (pLReader->pInfo == NULL) {
       tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr);
       return terrno;
@@ -594,14 +597,22 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
   pReader->blockInfoBuf.numPerBucket = 1000;  // 1000 tables per bucket
   ASSERT(pCond->numOfCols > 0);
 
+  if (pReader->pResBlock == NULL) {
+    pReader->freeBlock = true;
+    pReader->pResBlock = createResBlock(pCond, pReader->capacity);
+    if (pReader->pResBlock == NULL) {
+      code = terrno;
+      goto _end;
+    }
+  }
+
   // todo refactor.
   limitOutputBufferSize(pCond, &pReader->capacity);
 
   // allocate buffer in order to load data blocks from file
   SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
   pSup->pColAgg = taosArrayInit(pCond->numOfCols, sizeof(SColumnDataAgg));
-  pSup->plist = taosMemoryCalloc(pCond->numOfCols, POINTER_BYTES);
-  if (pSup->pColAgg == NULL || pSup->plist == NULL) {
+  if (pSup->pColAgg == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _end;
   }
@@ -1083,8 +1094,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
   int32_t i = 0;
   int32_t rowIndex = 0;
 
-  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
-  if (pSupInfo->colIds[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
+  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+  if (pSupInfo->colInfo[i].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
     copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);
     i += 1;
   }
@@ -1095,10 +1106,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
     rowIndex = 0;
 
     SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);
-    if (pData->cid < pSupInfo->colIds[i]) {
+    if (pData->cid < pSupInfo->colInfo[i].colId) {
       colIndex += 1;
-    } else if (pData->cid == pSupInfo->colIds[i]) {
-      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+    } else if (pData->cid == pSupInfo->colInfo[i].colId) {
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
 
       if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {
         colDataAppendNNULL(pColData, 0, dumpedRows);
@@ -1116,7 +1127,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
       colIndex += 1;
       i += 1;
     } else {  // the specified column does not exist in file block, fill with null data
-      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
       colDataAppendNNULL(pColData, 0, dumpedRows);
       i += 1;
     }
@@ -1124,7 +1135,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
 
   // fill the mis-matched columns with null value
   while (i < numOfOutputCols) {
-    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
     colDataAppendNNULL(pColData, 0, dumpedRows);
     i += 1;
   }
@@ -1162,7 +1173,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
   tBlockDataReset(pBlockData);
   TABLEID tid = {.suid = pReader->suid, .uid = uid};
   int32_t code =
-      tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1);
+      tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -1621,7 +1632,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
   int64_t st = taosGetTimestampUs();
   int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader);
 
-  blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotIds[0]);
+  blockDataUpdateTsWindow(pBlock, pReader->suppInfo.colInfo[0].slotId);
   pBlock->info.id.uid = pBlockScanInfo->uid;
 
   setComposedBlockFlag(pReader, true);
@@ -2493,7 +2504,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
 
 _end:
   pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
-  blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotIds[0]);
+  blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.colInfo[0].slotId);
 
   setComposedBlockFlag(pReader, true);
   double el = (taosGetTimestampUs() - st) / 1000.0;
@@ -3542,24 +3553,24 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
   SColVal colVal = {0};
   int32_t i = 0, j = 0;
 
-  if (pSupInfo->colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
-    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+  if (pSupInfo->colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
     ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts;
     i += 1;
   }
 
   while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) {
-    col_id_t colId = pSupInfo->colIds[i];
+    col_id_t colId = pSupInfo->colInfo[i].colId;
 
     if (colId == pSchema->columns[j].colId) {
-      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
 
       tTSRowGetVal(pTSRow, pSchema, j, &colVal);
       doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo);
       i += 1;
       j += 1;
     } else if (colId < pSchema->columns[j].colId) {
-      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
 
       colDataAppendNULL(pColInfoData, outputRowIndex);
       i += 1;
@@ -3570,7 +3581,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
 
   // set null value since current column does not exist in the "pSchema"
   while (i < pSupInfo->numOfCols) {
-    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
     colDataAppendNULL(pColInfoData, outputRowIndex);
     i += 1;
   }
@@ -3586,8 +3597,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   int32_t outputRowIndex = pResBlock->info.rows;
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
-  if (pReader->suppInfo.colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
-    SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+  if (pReader->suppInfo.colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
     ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex];
     i += 1;
   }
@@ -3598,13 +3609,13 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
 
   while (i < numOfOutputCols && j < numOfInputCols) {
     SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j);
-    if (pData->cid < pSupInfo->colIds[i]) {
+    if (pData->cid < pSupInfo->colInfo[i].colId) {
       j += 1;
       continue;
     }
 
-    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
-    if (pData->cid == pSupInfo->colIds[i]) {
+    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+    if (pData->cid == pSupInfo->colInfo[i].colId) {
       tColDataGetValue(pData, rowIndex, &cv);
       doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
       j += 1;
@@ -3617,7 +3628,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   }
 
   while (i < numOfOutputCols) {
-    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
     colDataAppendNULL(pCol, outputRowIndex);
     i += 1;
   }
@@ -3723,7 +3734,14 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
     pCond->twindows.ekey -= 1;
   }
 
-  int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, pResBlock->info.capacity, pResBlock, idstr);
+  int32_t capacity = 0;
+  if (pResBlock == NULL) {
+    capacity = 4096;
+  } else {
+    capacity = pResBlock->info.capacity;
+  }
+
+  int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr);
   if (code != TSDB_CODE_SUCCESS) {
     goto _err;
   }
@@ -3868,10 +3886,6 @@ void tsdbReaderClose(STsdbReader* pReader) {
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
 
-  taosMemoryFreeClear(pSupInfo->plist);
-  taosMemoryFree(pSupInfo->colIds);
-  taosMemoryFree(pSupInfo->slotIds);
-
   taosArrayDestroy(pSupInfo->pColAgg);
   for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) {
     if (pSupInfo->buildBuf[i] != NULL) {
@@ -3879,9 +3893,12 @@ void tsdbReaderClose(STsdbReader* pReader) {
     }
   }
 
-  taosMemoryFree(pSupInfo->buildBuf);
-  tBlockDataDestroy(&pReader->status.fileBlockData, true);
+  if (pReader->freeBlock) {
+    pReader->pResBlock = blockDataDestroy(pReader->pResBlock);
+  }
 
+  taosMemoryFree(pSupInfo->colInfo);
+  tBlockDataDestroy(&pReader->status.fileBlockData, true);
   cleanupDataBlockIterator(&pReader->status.blockIter);
 
   size_t numOfTables = taosHashGetSize(pReader->status.pTableMap);
@@ -4025,18 +4042,18 @@ void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64
   }
 }
 
-int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) {
+int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg ***pBlockSMA, bool* allHave) {
   int32_t code = 0;
   *allHave = false;
 
   if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {
-    *pBlockStatis = NULL;
+    *pBlockSMA = NULL;
     return TSDB_CODE_SUCCESS;
   }
 
   // there is no statistics data for composed block
   if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) {
-    *pBlockStatis = NULL;
+    *pBlockSMA = NULL;
     return TSDB_CODE_SUCCESS;
   }
 
@@ -4054,7 +4071,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
       return code;
     }
   } else {
-    *pBlockStatis = NULL;
+    *pBlockSMA = NULL;
     return TSDB_CODE_SUCCESS;
   }
 
@@ -4067,32 +4084,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
   pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
   pTsAgg->min = pReader->pResBlock->info.window.skey;
   pTsAgg->max = pReader->pResBlock->info.window.ekey;
-  pSup->plist[0] = pTsAgg;
 
   // update the number of NULL data rows
   size_t numOfCols = pSup->numOfCols;
 
   int32_t i = 0, j = 0;
   size_t  size = taosArrayGetSize(pSup->pColAgg);
-#if 0
-  while (j < numOfCols && i < size) {
-    SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
-    if (pAgg->colId == pSup->colIds[j]) {
-      if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) {
-        pSup->plist[j] = pAgg;
-      } else {
-        *allHave = false;
-        break;
-      }
-      i += 1;
-      j += 1;
-    } else if (pAgg->colId < pSup->colIds[j]) {
-      i += 1;
-    } else if (pSup->colIds[j] < pAgg->colId) {
-      j += 1;
-    }
-  }
-#else
 
   SSDataBlock* pResBlock = pReader->pResBlock;
   if (pResBlock->pBlockAgg == NULL) {
@@ -4100,52 +4097,40 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
     pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg));
   }
 
-  // fill the all null data column
-//  SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg));
-
   while (j < numOfCols && i < size) {
     SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
-    if (pAgg->colId == pSup->colIds[j]) {
-      pResBlock->pBlockAgg[pSup->slotIds[j]] = pAgg;
+    if (pAgg->colId == pSup->colInfo[j].colId) {
+      pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = pAgg;
       i += 1;
       j += 1;
-    } else if (pAgg->colId < pSup->colIds[j]) {
+    } else if (pAgg->colId < pSup->colInfo[j].colId) {
       i += 1;
-    } else if (pSup->colIds[j] < pAgg->colId) {
-      if (pSup->colIds[j] == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        pResBlock->pBlockAgg[pSup->slotIds[j]] = &pSup->tsColAgg;
+    } else if (pSup->colInfo[j].colId < pAgg->colId) {
+      if (pSup->colInfo[j].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+        pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = &pSup->tsColAgg;
       } else {
       // all date in this block are null
-        SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow};
+        SColumnDataAgg nullColAgg = {.colId = pSup->colInfo[j].colId, .numOfNull = pBlock->nRow};
         taosArrayPush(pSup->pColAgg, &nullColAgg);
 
-        pResBlock->pBlockAgg[pSup->slotIds[j]] = taosArrayGetLast(pSup->pColAgg);
+        pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = taosArrayGetLast(pSup->pColAgg);
       }
       j += 1;
     }
   }
 
-//  taosArrayClear(pSup->pColAgg);
-
-//  size_t num = taosArrayGetSize(pSup->pColAgg);
-//  for(int32_t k = 0; k < num; ++k) {
-//    pSup->plist[k] = taosArrayGet(pSup->pColAgg, k);
-//  }
-
-#endif
-
+  *pBlockSMA = pResBlock->pBlockAgg;
   pReader->cost.smaDataLoad += 1;
-//  *pBlockStatis = pSup->plist;
 
   tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr);
   return code;
 }
 
-static SArray* doRetrieveDataBlock(STsdbReader* pReader) {
+static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) {
   SReaderStatus* pStatus = &pReader->status;
 
   if (pStatus->composedDataBlock) {
-    return pReader->pResBlock->pDataBlock;
+    return pReader->pResBlock;
   }
 
   SFileDataBlockInfo*  pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter);
@@ -4166,10 +4151,10 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) {
   }
 
   copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
-  return pReader->pResBlock->pDataBlock;
+  return pReader->pResBlock;
 }
 
-SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
+SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
   if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {
     if (pReader->step == EXTERNAL_ROWS_PREV) {
       return doRetrieveDataBlock(pReader->innerReader[0]);
@@ -4196,7 +4181,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
 
   // allocate buffer in order to load data blocks from file
   memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg));
-  memset(pReader->suppInfo.plist, 0, POINTER_BYTES);
 
   pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID;
   tsdbDataFReaderClose(&pReader->pFileReader);
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index d675ac05d5..df11add919 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -957,21 +957,27 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s
   pCond->order = TSDB_ORDER_ASC;
   pCond->numOfCols = pMtInfo->schema->nCols;
   pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo));
-  if (pCond->colList == NULL) {
+  pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t) * pCond->numOfCols);
+  if (pCond->colList == NULL || pCond->pSlotList == NULL) {
+    taosMemoryFreeClear(pCond->colList);
+    taosMemoryFreeClear(pCond->pSlotList);
     terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
     return terrno;
   }
 
-  pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
+  pCond->twindows = TSWINDOW_INITIALIZER;
   pCond->suid = pMtInfo->suid;
   pCond->type = TIMEWINDOW_RANGE_CONTAINED;
   pCond->startVersion = -1;
   pCond->endVersion = sContext->snapVersion;
 
   for (int32_t i = 0; i < pCond->numOfCols; ++i) {
-    pCond->colList[i].type = pMtInfo->schema->pSchema[i].type;
-    pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes;
-    pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId;
+    SColumnInfo* pColInfo = &pCond->colList[i];
+    pColInfo->type = pMtInfo->schema->pSchema[i].type;
+    pColInfo->bytes = pMtInfo->schema->pSchema[i].bytes;
+    pColInfo->colId = pMtInfo->schema->pSchema[i].colId;
+
+    pCond->pSlotList[i] = i;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -1116,7 +1122,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
     int32_t        size = tableListGetSize(pTaskInfo->pTableInfoList);
     ASSERT(size == 1);
 
-    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->pRes, &pInfo->dataReader, NULL);
+    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL);
 
     cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
     strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index b0104c6025..f09b7dae58 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -224,10 +224,8 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA
 }
 
 static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
-  bool             allColumnsHaveAgg = true;
-  SColumnDataAgg** pColAgg = NULL;
-
-  int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg);
+  bool    allColumnsHaveAgg = true;
+  int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pBlock->pBlockAgg, &allColumnsHaveAgg);
   if (code != TSDB_CODE_SUCCESS) {
     T_LONG_JMP(pTaskInfo->env, code);
   }
@@ -387,11 +385,12 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
   pCost->totalCheckedRows += pBlock->info.rows;
   pCost->loadBlocks += 1;
 
-  SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
-  if (pCols == NULL) {
+  SSDataBlock* p = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
+  if (p == NULL) {
     return terrno;
   }
 
+  ASSERT(p == pBlock);
   doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
 
   // restore the previous value
@@ -994,19 +993,10 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
     return NULL;
   }
 
-  bool hasBlock = tsdbNextDataBlock(pReader);
-  if (hasBlock) {
-    SDataBlockInfo* pBInfo = &pBlock->info;
-
-//    int32_t rows = 0;
-//    tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window);
-
-    SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL);
-
-//    relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true);
+  if (tsdbNextDataBlock(pReader)) {
+    /*SSDataBlock* p = */tsdbRetrieveDataBlock(pReader, NULL);
     doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
-
-    pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->id.uid);
+    pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
   }
 
   tsdbReaderClose(pReader);
@@ -2030,20 +2020,13 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
 
   qDebug("tmqsnap doRawScan called");
   if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    SSDataBlock* pBlock = &pInfo->pRes;
-
     if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) {
       if (isTaskKilled(pTaskInfo)) {
         longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
       }
 
-      int32_t rows = 0;
-      tsdbRetrieveDataBlockInfo(pInfo->dataReader, &rows, &pBlock->info.id.uid, &pBlock->info.window);
-      pBlock->info.rows = rows;
-
-      SArray* pCols = tsdbRetrieveDataBlock(pInfo->dataReader, NULL);
-      pBlock->pDataBlock = pCols;
-      if (pCols == NULL) {
+      SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL);
+      if (pBlock == NULL) {
         longjmp(pTaskInfo->env, terrno);
       }
 
diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py
index 2b0336d2d7..6af9b130ef 100644
--- a/tests/system-test/2-query/unique.py
+++ b/tests/system-test/2-query/unique.py
@@ -429,10 +429,10 @@ class TDTestCase:
         tdSql.checkRows(2)
 
         # nest query
-        tdSql.query(f"select  unique(c1) from (select _rowts , t1 ,c1 , tbname from  {dbname}.stb1 ) ")
+        tdSql.query(f"select  unique(c1) v from (select _rowts , t1 ,c1 , tbname from  {dbname}.stb1 ) order by v")
         tdSql.checkRows(11)
-        tdSql.checkData(0,0,6)
-        tdSql.checkData(10,0,3)
+        tdSql.checkData(1,0,0)
+        tdSql.checkData(10,0,9)
         tdSql.query(f"select  unique(t1) from (select _rowts , t1 , tbname from  {dbname}.stb1 )")
         tdSql.checkRows(2)
         tdSql.checkData(0,0,4)

From 867f7abeea3099dcf3ea61fb64a8dabb9c53f049 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 2 Dec 2022 17:44:39 +0800
Subject: [PATCH 25/32] test: add asan case

---
 tests/parallel_test/cases.task | 44 +++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index e514b069a8..13bfe4c981 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -623,12 +623,12 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py
-,,,system-test,python3 ./test.py -f 2-query/stablity.py
-,,,system-test,python3 ./test.py -f 2-query/stablity_1.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py
@@ -788,12 +788,12 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 2
-,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 2
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 2
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 2
@@ -881,12 +881,12 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 3
-,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 3
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 3
@@ -974,10 +974,10 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4
-,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4
+,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 4
 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 4

From 27ed740eb56317a5bcb5dc4328fecca8179a51b4 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 2 Dec 2022 17:52:49 +0800
Subject: [PATCH 26/32] refactor: do some internal refactor.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 89 +++++++++++++-------------
 1 file changed, 43 insertions(+), 46 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 859ae026a2..ac003783ae 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -81,15 +81,11 @@ typedef struct SIOCostSummary {
   double  createScanInfoList;
 } SIOCostSummary;
 
-typedef struct SColInfo {
-  int16_t colId;
-  int16_t slotId;
-} SColInfo;
-
 typedef struct SBlockLoadSuppInfo {
   SArray*          pColAgg;
   SColumnDataAgg   tsColAgg;
-  SColInfo*        colInfo;    // column ids for loading file block data
+  int16_t*         colId;
+  int16_t*         slotId;
   int32_t          numOfCols;
   char**           buildBuf;  // build string tmp buffer, todo remove it later after all string format being updated.
   bool             smaValid;  // the sma on all queried columns are activated
@@ -222,16 +218,17 @@ static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWi
 static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) {
   pSupInfo->smaValid = true;
   pSupInfo->numOfCols = numOfCols;
-  pSupInfo->colInfo = taosMemoryMalloc(numOfCols * (sizeof(SColInfo) + POINTER_BYTES));
-  if (pSupInfo->colInfo == NULL) {
-    taosMemoryFree(pSupInfo->colInfo);
+  pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t)*2 + POINTER_BYTES));
+  if (pSupInfo->colId == NULL) {
+    taosMemoryFree(pSupInfo->colId);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  pSupInfo->buildBuf = (char**)((char*)pSupInfo->colInfo + (sizeof(SColInfo) * numOfCols));
+  pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols));
+  pSupInfo->buildBuf = (char**) ((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols));
   for (int32_t i = 0; i < numOfCols; ++i) {
-    pSupInfo->colInfo[i].colId = pCols[i].colId;
-    pSupInfo->colInfo[i].slotId = pSlotIdList[i];
+    pSupInfo->colId[i] = pCols[i].colId;
+    pSupInfo->slotId[i] = pSlotIdList[i];
 
     if (IS_VAR_DATA_TYPE(pCols[i].type)) {
       pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes);
@@ -248,7 +245,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
 
   while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) {
     STColumn* pTCol = &pSchema->columns[i];
-    if (pTCol->colId == pSupInfo->colInfo[j].colId) {
+    if (pTCol->colId == pSupInfo->colId[j]) {
       if (!IS_BSMA_ON(pTCol)) {
         pSupInfo->smaValid = false;
         return;
@@ -256,7 +253,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
 
       i += 1;
       j += 1;
-    } else if (pTCol->colId < pSupInfo->colInfo[j].colId) {
+    } else if (pTCol->colId < pSupInfo->colId[j]) {
       // do nothing
       i += 1;
     } else {
@@ -458,7 +455,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
   if (pLReader->pInfo == NULL) {
     // here we ignore the first column, which is always be the primary timestamp column
     pLReader->pInfo =
-        tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1);
+        tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1);
     if (pLReader->pInfo == NULL) {
       tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr);
       return terrno;
@@ -1094,8 +1091,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
   int32_t i = 0;
   int32_t rowIndex = 0;
 
-  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
-  if (pSupInfo->colInfo[i].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
+  if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
     copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);
     i += 1;
   }
@@ -1106,10 +1103,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
     rowIndex = 0;
 
     SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);
-    if (pData->cid < pSupInfo->colInfo[i].colId) {
+    if (pData->cid < pSupInfo->colId[i]) {
       colIndex += 1;
-    } else if (pData->cid == pSupInfo->colInfo[i].colId) {
-      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+    } else if (pData->cid == pSupInfo->colId[i]) {
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
 
       if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {
         colDataAppendNNULL(pColData, 0, dumpedRows);
@@ -1127,7 +1124,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
       colIndex += 1;
       i += 1;
     } else {  // the specified column does not exist in file block, fill with null data
-      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
       colDataAppendNNULL(pColData, 0, dumpedRows);
       i += 1;
     }
@@ -1135,7 +1132,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
 
   // fill the mis-matched columns with null value
   while (i < numOfOutputCols) {
-    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
     colDataAppendNNULL(pColData, 0, dumpedRows);
     i += 1;
   }
@@ -1173,7 +1170,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
   tBlockDataReset(pBlockData);
   TABLEID tid = {.suid = pReader->suid, .uid = uid};
   int32_t code =
-      tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1);
+      tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -1632,7 +1629,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
   int64_t st = taosGetTimestampUs();
   int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader);
 
-  blockDataUpdateTsWindow(pBlock, pReader->suppInfo.colInfo[0].slotId);
+  blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotId[0]);
   pBlock->info.id.uid = pBlockScanInfo->uid;
 
   setComposedBlockFlag(pReader, true);
@@ -2504,7 +2501,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
 
 _end:
   pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
-  blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.colInfo[0].slotId);
+  blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]);
 
   setComposedBlockFlag(pReader, true);
   double el = (taosGetTimestampUs() - st) / 1000.0;
@@ -3553,24 +3550,24 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
   SColVal colVal = {0};
   int32_t i = 0, j = 0;
 
-  if (pSupInfo->colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) {
-    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+  if (pSupInfo->colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
     ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts;
     i += 1;
   }
 
   while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) {
-    col_id_t colId = pSupInfo->colInfo[i].colId;
+    col_id_t colId = pSupInfo->colId[i];
 
     if (colId == pSchema->columns[j].colId) {
-      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
 
       tTSRowGetVal(pTSRow, pSchema, j, &colVal);
       doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo);
       i += 1;
       j += 1;
     } else if (colId < pSchema->columns[j].colId) {
-      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
 
       colDataAppendNULL(pColInfoData, outputRowIndex);
       i += 1;
@@ -3581,7 +3578,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
 
   // set null value since current column does not exist in the "pSchema"
   while (i < pSupInfo->numOfCols) {
-    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
     colDataAppendNULL(pColInfoData, outputRowIndex);
     i += 1;
   }
@@ -3597,8 +3594,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   int32_t outputRowIndex = pResBlock->info.rows;
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
-  if (pReader->suppInfo.colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) {
-    SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+  if (pReader->suppInfo.colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
     ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex];
     i += 1;
   }
@@ -3609,13 +3606,13 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
 
   while (i < numOfOutputCols && j < numOfInputCols) {
     SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j);
-    if (pData->cid < pSupInfo->colInfo[i].colId) {
+    if (pData->cid < pSupInfo->colId[i]) {
       j += 1;
       continue;
     }
 
-    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
-    if (pData->cid == pSupInfo->colInfo[i].colId) {
+    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]);
+    if (pData->cid == pSupInfo->colId[i]) {
       tColDataGetValue(pData, rowIndex, &cv);
       doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
       j += 1;
@@ -3628,7 +3625,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   }
 
   while (i < numOfOutputCols) {
-    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
     colDataAppendNULL(pCol, outputRowIndex);
     i += 1;
   }
@@ -3897,7 +3894,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
     pReader->pResBlock = blockDataDestroy(pReader->pResBlock);
   }
 
-  taosMemoryFree(pSupInfo->colInfo);
+  taosMemoryFree(pSupInfo->colId);
   tBlockDataDestroy(&pReader->status.fileBlockData, true);
   cleanupDataBlockIterator(&pReader->status.blockIter);
 
@@ -4099,21 +4096,21 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg ***pBlockS
 
   while (j < numOfCols && i < size) {
     SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
-    if (pAgg->colId == pSup->colInfo[j].colId) {
-      pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = pAgg;
+    if (pAgg->colId == pSup->colId[j]) {
+      pResBlock->pBlockAgg[pSup->slotId[j]] = pAgg;
       i += 1;
       j += 1;
-    } else if (pAgg->colId < pSup->colInfo[j].colId) {
+    } else if (pAgg->colId < pSup->colId[j]) {
       i += 1;
-    } else if (pSup->colInfo[j].colId < pAgg->colId) {
-      if (pSup->colInfo[j].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = &pSup->tsColAgg;
+    } else if (pSup->colId[j] < pAgg->colId) {
+      if (pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID) {
+        pResBlock->pBlockAgg[pSup->slotId[j]] = &pSup->tsColAgg;
       } else {
       // all date in this block are null
-        SColumnDataAgg nullColAgg = {.colId = pSup->colInfo[j].colId, .numOfNull = pBlock->nRow};
+        SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = pBlock->nRow};
         taosArrayPush(pSup->pColAgg, &nullColAgg);
 
-        pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = taosArrayGetLast(pSup->pColAgg);
+        pResBlock->pBlockAgg[pSup->slotId[j]] = taosArrayGetLast(pSup->pColAgg);
       }
       j += 1;
     }

From 55fb892e70bd6631196f1078aea31b27f25f7abc Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 2 Dec 2022 15:50:53 +0800
Subject: [PATCH 27/32] fix(tsdb/cache): use lru erase to invalidate cache
 entries

---
 source/dnode/vnode/src/tsdb/tsdbCache.c | 125 +++++++-----------------
 1 file changed, 35 insertions(+), 90 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index d71eb33951..7309ea3407 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -228,23 +228,23 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST
             invalidate = true;
 
             break;
-          }
-        } else {
-          SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
-          if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
-            SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
-            taosMemoryFree(pLastCol->colVal.value.pData);
+          } else {  // new inserting key is greater than cached, update cached entry
+            SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
+            if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
+              SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
+              taosMemoryFree(pLastCol->colVal.value.pData);
 
-            lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
-            if (lastCol.colVal.value.pData == NULL) {
-              terrno = TSDB_CODE_OUT_OF_MEMORY;
-              code = TSDB_CODE_OUT_OF_MEMORY;
-              goto _invalidate;
+              lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
+              if (lastCol.colVal.value.pData == NULL) {
+                terrno = TSDB_CODE_OUT_OF_MEMORY;
+                code = TSDB_CODE_OUT_OF_MEMORY;
+                goto _invalidate;
+              }
+              memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData);
             }
-            memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData);
-          }
 
-          taosArraySet(pLast, iCol, &lastCol);
+            taosArraySet(pLast, iCol, &lastCol);
+          }
         }
       }
     }
@@ -253,65 +253,10 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST
     taosMemoryFreeClear(pTSchema);
 
     taosLRUCacheRelease(pCache, h, invalidate);
-    /*
-    cacheRow = (STSRow *)taosLRUCacheValue(pCache, h);
-    if (row->ts >= cacheRow->ts) {
-      if (row->ts == cacheRow->ts) {
-        STSRow    *mergedRow = NULL;
-        SRowMerger merger = {0};
-        STSchema  *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1);
-
-        tRowMergerInit(&merger, &tsdbRowFromTSRow(0, cacheRow), pTSchema);
-
-        tRowMerge(&merger, &tsdbRowFromTSRow(1, row));
-
-        tRowMergerGetRow(&merger, &mergedRow);
-        tRowMergerClear(&merger);
-
-        taosMemoryFreeClear(pTSchema);
-
-        row = mergedRow;
-        dup = false;
-      }
-
-      if (TD_ROW_LEN(row) <= TD_ROW_LEN(cacheRow)) {
-        tdRowCpy(cacheRow, row);
-        if (!dup) {
-          taosMemoryFree(row);
-        }
-
-        taosLRUCacheRelease(pCache, h, false);
-      } else {
-        taosLRUCacheRelease(pCache, h, true);
-        // tsdbCacheDeleteLastrow(pCache, uid, TSKEY_MAX);
-        if (dup) {
-          cacheRow = tdRowDup(row);
-        } else {
-          cacheRow = row;
-        }
-        _taos_lru_deleter_t deleter = deleteTableCacheLastrow;
-        LRUStatus status = taosLRUCacheInsert(pCache, key, keyLen, cacheRow, TD_ROW_LEN(cacheRow), deleter, NULL,
-                                              TAOS_LRU_PRIORITY_LOW);
-        if (status != TAOS_LRU_STATUS_OK) {
-          code = -1;
-        }
-        // tsdbCacheInsertLastrow(pCache, uid, row, dup);
-      }
-    }*/
-  } /*else {
-    if (dup) {
-      cacheRow = tdRowDup(row);
-    } else {
-      cacheRow = row;
+    if (invalidate) {
+      taosLRUCacheErase(pCache, key, keyLen);
     }
-
-    _taos_lru_deleter_t deleter = deleteTableCacheLastrow;
-    LRUStatus           status =
-        taosLRUCacheInsert(pCache, key, keyLen, cacheRow, TD_ROW_LEN(cacheRow), deleter, NULL, TAOS_LRU_PRIORITY_LOW);
-    if (status != TAOS_LRU_STATUS_OK) {
-      code = -1;
-    }
-    }*/
+  }
 
   return code;
 }
@@ -349,28 +294,28 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb
 
         SColVal colVal = {0};
         tTSRowGetVal(row, pTSchema, iCol, &colVal);
-        if (!COL_VAL_IS_VALUE(&colVal)) {
+        if (COL_VAL_IS_VALUE(&colVal)) {
           if (keyTs == tTsVal1->ts && COL_VAL_IS_VALUE(tColVal)) {
             invalidate = true;
 
             break;
-          }
-        } else {
-          SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
-          if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
-            SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
-            taosMemoryFree(pLastCol->colVal.value.pData);
+          } else {
+            SLastCol lastCol = {.ts = keyTs, .colVal = colVal};
+            if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) {
+              SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol);
+              taosMemoryFree(pLastCol->colVal.value.pData);
 
-            lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
-            if (lastCol.colVal.value.pData == NULL) {
-              terrno = TSDB_CODE_OUT_OF_MEMORY;
-              code = TSDB_CODE_OUT_OF_MEMORY;
-              goto _invalidate;
+              lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData);
+              if (lastCol.colVal.value.pData == NULL) {
+                terrno = TSDB_CODE_OUT_OF_MEMORY;
+                code = TSDB_CODE_OUT_OF_MEMORY;
+                goto _invalidate;
+              }
+              memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData);
             }
-            memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData);
-          }
 
-          taosArraySet(pLast, iCol, &lastCol);
+            taosArraySet(pLast, iCol, &lastCol);
+          }
         }
       }
     }
@@ -379,9 +324,9 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb
     taosMemoryFreeClear(pTSchema);
 
     taosLRUCacheRelease(pCache, h, invalidate);
-
-    // clear last cache anyway, lazy load when get last lookup
-    // taosLRUCacheRelease(pCache, h, true);
+    if (invalidate) {
+      taosLRUCacheErase(pCache, key, keyLen);
+    }
   }
 
   return code;

From 61cd7539475df28316708933cc46417a381699ab Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 2 Dec 2022 18:59:03 +0800
Subject: [PATCH 28/32] enh: remove sync batch propose in vnode

---
 source/dnode/vnode/src/vnd/vnodeSync.c | 74 ++++++++++++++++++++++----
 1 file changed, 65 insertions(+), 9 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index d01fdabac8..ee95e5b7d1 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -16,7 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "vnd.h"
 
-#define BATCH_DISABLE 1
+#define BATCH_ENABLE 0
 
 static inline bool vnodeIsMsgBlock(tmsg_t type) {
   return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) ||
@@ -87,7 +87,7 @@ static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
 }
 
 static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) {
-  if (code == TSDB_CODE_SYN_NOT_LEADER) {
+  if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) {
     vnodeRedirectRpcMsg(pVnode, pMsg);
   } else {
     const STraceId *trace = &pMsg->info.traceId;
@@ -99,15 +99,12 @@ static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code)
   }
 }
 
+#if BATCH_ENABLE
+
 static void inline vnodeProposeBatchMsg(SVnode *pVnode, SRpcMsg **pMsgArr, bool *pIsWeakArr, int32_t *arrSize) {
   if (*arrSize <= 0) return;
 
-#if BATCH_DISABLE
-  int32_t code = syncPropose(pVnode->sync, pMsgArr[0], pIsWeakArr[0]);
-#else
   int32_t code = syncProposeBatch(pVnode->sync, pMsgArr, pIsWeakArr, *arrSize);
-#endif
-
   if (code > 0) {
     for (int32_t i = 0; i < *arrSize; ++i) {
       vnodeHandleWriteMsg(pVnode, pMsgArr[i]);
@@ -177,7 +174,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
       continue;
     }
 
-    if (isBlock || BATCH_DISABLE) {
+    if (isBlock) {
       vnodeProposeBatchMsg(pVnode, pMsgArr, pIsWeakArr, &arrayPos);
     }
 
@@ -185,7 +182,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
     pIsWeakArr[arrayPos] = isWeak;
     arrayPos++;
 
-    if (isBlock || msg == numOfMsgs - 1 || BATCH_DISABLE) {
+    if (isBlock || msg == numOfMsgs - 1) {
       vnodeProposeBatchMsg(pVnode, pMsgArr, pIsWeakArr, &arrayPos);
     }
   }
@@ -194,6 +191,65 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
   taosMemoryFree(pIsWeakArr);
 }
 
+#else
+
+static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) {
+  int32_t code = syncPropose(pVnode->sync, pMsg, isWeak);
+  if (code > 0) {
+    vnodeHandleWriteMsg(pVnode, pMsg);
+  } else if (code == 0) {
+    vnodeWaitBlockMsg(pVnode, pMsg);
+  } else {
+    if (terrno != 0) code = terrno;
+    vnodeHandleProposeError(pVnode, pMsg, code);
+  }
+
+  return code;
+}
+
+void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
+  SVnode  *pVnode = pInfo->ahandle;
+  int32_t  vgId = pVnode->config.vgId;
+  int32_t  code = 0;
+  SRpcMsg *pMsg = NULL;
+  vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs);
+
+  for (int32_t msg = 0; msg < numOfMsgs; msg++) {
+    if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
+    bool isWeak = vnodeIsMsgWeak(pMsg->msgType);
+
+    const STraceId *trace = &pMsg->info.traceId;
+    vGTrace("vgId:%d, msg:%p get from vnode-write queue, weak:%d block:%d msg:%d:%d, handle:%p", vgId, pMsg, isWeak,
+            vnodeIsMsgBlock(pMsg->msgType), msg, numOfMsgs, pMsg->info.handle);
+
+    if (!pVnode->restored) {
+      vGError("vgId:%d, msg:%p failed to process since restore not finished", vgId, pMsg);
+      vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING);
+      rpcFreeCont(pMsg->pCont);
+      taosFreeQitem(pMsg);
+      continue;
+    }
+
+    code = vnodePreProcessWriteMsg(pVnode, pMsg);
+    if (code != 0) {
+      vGError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, terrstr());
+      if (terrno != 0) code = terrno;
+      vnodeHandleProposeError(pVnode, pMsg, code);
+      rpcFreeCont(pMsg->pCont);
+      taosFreeQitem(pMsg);
+      continue;
+    }
+
+    code = vnodeProposeMsg(pVnode, pMsg, isWeak);
+
+    vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code);
+    rpcFreeCont(pMsg->pCont);
+    taosFreeQitem(pMsg);
+  }
+}
+
+#endif
+
 void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnode  *pVnode = pInfo->ahandle;
   int32_t  vgId = pVnode->config.vgId;

From 524de026a553a83500d288b27f3a397561d0bd51 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 2 Dec 2022 22:24:35 +0800
Subject: [PATCH 29/32] enh: remove TSDB_CODE_RPC_REDIRECT

---
 include/common/tmsgcb.h                       |  3 ---
 include/libs/qcom/query.h                     | 18 ++++++++---------
 include/util/taoserror.h                      |  4 ++--
 source/client/src/clientMain.c                |  7 ++-----
 source/dnode/mgmt/mgmt_mnode/src/mmWorker.c   |  2 +-
 source/dnode/mgmt/node_mgmt/src/dmTransport.c | 20 +------------------
 source/dnode/mnode/impl/src/mndTrans.c        |  6 +++---
 source/dnode/vnode/src/vnd/vnodeSync.c        | 12 ++++++++++-
 source/libs/function/src/udfd.c               |  7 ++++---
 source/libs/qworker/src/qwDbg.c               |  6 +++---
 source/libs/transport/src/tmsgcb.c            |  2 --
 source/libs/transport/src/transCli.c          |  2 +-
 source/util/src/terror.c                      |  4 +---
 13 files changed, 38 insertions(+), 55 deletions(-)

diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h
index b5b997dac0..32d00bb422 100644
--- a/include/common/tmsgcb.h
+++ b/include/common/tmsgcb.h
@@ -43,7 +43,6 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg);
 typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
 typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg);
 typedef void (*SendRspFp)(SRpcMsg* pMsg);
-typedef void (*SendRedirectRspFp)(SRpcMsg* pMsg, const SEpSet* pNewEpSet);
 typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg);
 typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type);
 typedef void (*ReportStartup)(const char* name, const char* desc);
@@ -55,7 +54,6 @@ typedef struct {
   GetQueueSizeFp          qsizeFp;
   SendReqFp               sendReqFp;
   SendRspFp               sendRspFp;
-  SendRedirectRspFp       sendRedirectRspFp;
   RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
   ReleaseHandleFp         releaseHandleFp;
   ReportStartup           reportStartupFp;
@@ -66,7 +64,6 @@ int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg);
 int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype);
 int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg);
 void    tmsgSendRsp(SRpcMsg* pMsg);
-void    tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet);
 void    tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg);
 void    tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type);
 void    tmsgReportStartup(const char* name, const char* desc);
diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index e2e9da6624..cd61c29559 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -265,21 +265,21 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false)  // used later
 
 #define NEED_REDIRECT_ERROR(_code)                                                      \
-  ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL ||     \
-   (_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
+  ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL ||  \
+   (_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ||   \
    SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \
-   (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_RPC_BROKEN_LINK ||        \
-   (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING)
+   (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING ||      \
+   (_code) == TSDB_CODE_APP_IS_STOPPING)
 
 #define NEED_CLIENT_RM_TBLMETA_REQ(_type)                                                                  \
   ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
    (_type) == TDMT_MND_DROP_STB)
 
-#define NEED_SCHEDULER_REDIRECT_ERROR(_code)                                              \
-  ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_MNODE_NOT_FOUND ||           \
-   SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \
-   SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_SYN_RESTORING ||       \
-   (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING)
+#define NEED_SCHEDULER_REDIRECT_ERROR(_code)                                            \
+  ((_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ||   \
+   SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \
+   (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING ||      \
+   (_code) == TSDB_CODE_APP_IS_STOPPING)
 
 #define REQUEST_TOTAL_EXEC_TIMES 2
 
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 114cc45a59..d9e9f5a4fd 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -42,8 +42,8 @@ int32_t* taosGetErrno();
 // rpc
 // #define TSDB_CODE_RPC_ACTION_IN_PROGRESS     TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x
 // #define TSDB_CODE_RPC_AUTH_REQUIRED          TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x
-#define TSDB_CODE_RPC_AUTH_FAILURE              TAOS_DEF_ERROR_CODE(0, 0x0003)
-#define TSDB_CODE_RPC_REDIRECT                  TAOS_DEF_ERROR_CODE(0, 0x0004)
+// #define TSDB_CODE_RPC_AUTH_FAILURE           TAOS_DEF_ERROR_CODE(0, 0x0003) //2.x
+// #define TSDB_CODE_RPC_REDIRECT               TAOS_DEF_ERROR_CODE(0, 0x0004)
 // #define TSDB_CODE_RPC_NOT_READY              TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x
 // #define TSDB_CODE_RPC_ALREADY_PROCESSED      TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x
 // #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 976d1dd1b0..8ee774ceee 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -146,7 +146,6 @@ void taos_close(TAOS *taos) {
 
 int taos_errno(TAOS_RES *res) {
   if (res == NULL || TD_RES_TMQ_META(res)) {
-    if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY;
     return terrno;
   }
 
@@ -154,12 +153,11 @@ int taos_errno(TAOS_RES *res) {
     return 0;
   }
 
-  return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code;
+  return ((SRequestObj *)res)->code;
 }
 
 const char *taos_errstr(TAOS_RES *res) {
   if (res == NULL || TD_RES_TMQ_META(res)) {
-    if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY;
     return (const char *)tstrerror(terrno);
   }
 
@@ -171,8 +169,7 @@ const char *taos_errstr(TAOS_RES *res) {
   if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
     return pRequest->msgBuf;
   } else {
-    return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY)
-                                                    : (const char *)tstrerror(pRequest->code);
+    return (const char *)tstrerror(pRequest->code);
   }
 }
 
diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
index 212de0bfb4..857fbcbce5 100644
--- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
+++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
@@ -61,7 +61,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
     pMsg->info.rsp = NULL;
   }
 
-  if (code == TSDB_CODE_RPC_REDIRECT) {
+  if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) {
     mndPostProcessQueryMsg(pMsg);
   }
 
diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index 6e7d869eed..a3430d0bde 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -33,23 +33,6 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) {
   }
 }
 
-static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) {
-  pMsg->info.hasEpSet = 1;
-  SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info, .msgType = pMsg->msgType};
-  int32_t contLen = tSerializeSEpSet(NULL, 0, pNewEpSet);
-
-  rsp.pCont = rpcMallocCont(contLen);
-  if (rsp.pCont == NULL) {
-    pMsg->code = TSDB_CODE_OUT_OF_MEMORY;
-  } else {
-    tSerializeSEpSet(rsp.pCont, contLen, pNewEpSet);
-    rsp.contLen = contLen;
-  }
-  dmSendRsp(&rsp);
-  rpcFreeCont(pMsg->pCont);
-  pMsg->pCont = NULL;
-}
-
 int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
   const STraceId *trace = &pMsg->info.traceId;
 
@@ -243,7 +226,7 @@ static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { rpcRegisterBrokenLin
 static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcReleaseHandle(pHandle, type); }
 
 static bool rpcRfp(int32_t code, tmsg_t msgType) {
-  if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_MNODE_NOT_FOUND ||
+  if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_MNODE_NOT_FOUND ||
       code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_RPC_BROKEN_LINK ||
       code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) {
     if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH ||
@@ -334,7 +317,6 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) {
       .clientRpc = pDnode->trans.clientRpc,
       .sendReqFp = dmSendReq,
       .sendRspFp = dmSendRsp,
-      .sendRedirectRspFp = dmSendRedirectRsp,
       .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg,
       .releaseHandleFp = dmReleaseHandle,
       .reportStartupFp = dmReportStartup,
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 38bfe7f6cb..4b11884050 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -945,7 +945,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
         code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL;
       }
       if (i != 0 && code == 0) {
-        code = TSDB_CODE_RPC_REDIRECT;
+        code = TSDB_CODE_MNODE_NOT_FOUND;
       }
       mInfo("trans:%d, client:%d send rsp, code:0x%x stage:%s app:%p", pTrans->id, i, code, mndTransStr(pTrans->stage),
             pInfo->ahandle);
@@ -1042,8 +1042,8 @@ static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pA
   pAction->rawWritten = 0;
   pAction->msgSent = 0;
   pAction->msgReceived = 0;
-  if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR ||
-      pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) {
+  if (pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR ||
+      pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) {
     pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps;
     mInfo("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage),
           pAction->id, pAction->epSet.inUse);
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index ee95e5b7d1..ed612ff6a6 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -67,7 +67,17 @@ void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   pMsg->info.hasEpSet = 1;
 
   SRpcMsg rsp = {.code = TSDB_CODE_SYN_NOT_LEADER, .info = pMsg->info, .msgType = pMsg->msgType + 1};
-  tmsgSendRedirectRsp(&rsp, &newEpSet);
+  int32_t contLen = tSerializeSEpSet(NULL, 0, &newEpSet);
+
+  rsp.pCont = rpcMallocCont(contLen);
+  if (rsp.pCont == NULL) {
+    pMsg->code = TSDB_CODE_OUT_OF_MEMORY;
+  } else {
+    tSerializeSEpSet(rsp.pCont, contLen, &newEpSet);
+    rsp.contLen = contLen;
+  }
+
+  tmsgSendRsp(&rsp);
 }
 
 static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index ec54f8ff2d..9d249e0831 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -598,9 +598,10 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
   return 0;
 }
 static bool udfdRpcRfp(int32_t code, tmsg_t msgType) {
-  if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK ||
-      code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND|| code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) {
-    if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) {
+  if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_SYN_NOT_LEADER ||
+      code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING ||
+      code == TSDB_CODE_APP_IS_STOPPING) {
+      if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) {
       return false;
     } 
     return true;
diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c
index 4c4a41df82..a122508f98 100644
--- a/source/libs/qworker/src/qwDbg.c
+++ b/source/libs/qworker/src/qwDbg.c
@@ -214,20 +214,20 @@ void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) {
       epSet.eps[2].port = 7300;
 
       ctx->phase = QW_PHASE_POST_QUERY;
-      qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, &epSet);
+      qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, &epSet);
       *rsped = true;
       return;
     }
 
     if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) {
       QW_SET_PHASE(ctx, QW_PHASE_POST_QUERY);
-      qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL);
+      qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL);
       *rsped = true;
       return;
     }
 
     if ((TDMT_SCH_FETCH == qwMsg->msgType) && (0 == taosRand() % 9)) {
-      qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL);
+      qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL);
       *rsped = true;
       return;
     }
diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c
index 559715084c..95bc532994 100644
--- a/source/libs/transport/src/tmsgcb.c
+++ b/source/libs/transport/src/tmsgcb.c
@@ -54,8 +54,6 @@ void tmsgSendRsp(SRpcMsg* pMsg) {
 #endif
 }
 
-void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { (*defaultMsgCb.sendRedirectRspFp)(pMsg, pNewEpSet); }
-
 void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { (*defaultMsgCb.registerBrokenLinkArgFp)(pMsg); }
 
 void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { (*defaultMsgCb.releaseHandleFp)(pHandle, type); }
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index a9afbd7ba8..d588e2db1c 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -1519,7 +1519,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
     transFreeMsg(pResp->pCont);
     transUnrefCliHandle(pConn);
   } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR ||
-             code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_VND_STOPPED) {
+             code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED) {
     tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen);
     noDelay = cliResetEpset(pCtx, pResp, true);
     transFreeMsg(pResp->pCont);
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 4082e8985a..2110c0b3aa 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -46,8 +46,6 @@ STaosError errors[] = {
 #endif
 
 // rpc
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE,             "Authentication failure")
-TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT,                 "Database not ready, need retry")
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL,          "Unable to establish connection")
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR,               "Unable to resolve FQDN")
 TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE,          "Port already in use")
@@ -320,7 +318,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_ALREADY_DEPLOYED,       "Snode already deploye
 TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_DEPLOYED,           "Snode not deployed")
 
 // vnode
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID,        "Invalid Vgroup ID")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID,        "Vnode moved to another dnode or was deleted")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH,            "Database write operation denied")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_EXIST,                "Vnode not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST,            "Vnode already exist")

From f3e980de149c5053bffa7d861596d4c4d506631b Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 2 Dec 2022 22:45:11 +0800
Subject: [PATCH 30/32] enh: remove TSDB_CODE_RPC_REDIRECT

---
 include/libs/qcom/query.h                     | 12 +++++-------
 include/util/taoserror.h                      |  2 +-
 source/dnode/mgmt/node_mgmt/src/dmTransport.c |  6 +++---
 source/dnode/vnode/src/inc/vnd.h              |  2 +-
 source/dnode/vnode/src/vnd/vnodeSvr.c         |  6 +++---
 source/dnode/vnode/src/vnd/vnodeSync.c        |  8 +++++---
 source/libs/transport/src/transCli.c          |  4 +++-
 7 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h
index cd61c29559..97ce5ae032 100644
--- a/include/libs/qcom/query.h
+++ b/include/libs/qcom/query.h
@@ -261,25 +261,23 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
    NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
 
 #define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || (_code) == TSDB_CODE_VND_STOPPED)
-#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR)
+#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR)
 #define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false)  // used later
 
 #define NEED_REDIRECT_ERROR(_code)                                                      \
   ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL ||  \
-   (_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ||   \
+   (_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
    SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \
-   (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING ||      \
-   (_code) == TSDB_CODE_APP_IS_STOPPING)
+   (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING)
 
 #define NEED_CLIENT_RM_TBLMETA_REQ(_type)                                                                  \
   ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
    (_type) == TDMT_MND_DROP_STB)
 
 #define NEED_SCHEDULER_REDIRECT_ERROR(_code)                                            \
-  ((_code) == TSDB_CODE_SYN_RESTORING || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ||   \
+  ((_code) == TSDB_CODE_MNODE_NOT_FOUND || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
    SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \
-   (_code) == TSDB_CODE_MNODE_NOT_FOUND || (_code) == TSDB_CODE_APP_IS_STARTING ||      \
-   (_code) == TSDB_CODE_APP_IS_STOPPING)
+   (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING)
 
 #define REQUEST_TOTAL_EXEC_TIMES 2
 
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index d9e9f5a4fd..2baf4d14e2 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -43,7 +43,7 @@ int32_t* taosGetErrno();
 // #define TSDB_CODE_RPC_ACTION_IN_PROGRESS     TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x
 // #define TSDB_CODE_RPC_AUTH_REQUIRED          TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x
 // #define TSDB_CODE_RPC_AUTH_FAILURE           TAOS_DEF_ERROR_CODE(0, 0x0003) //2.x
-// #define TSDB_CODE_RPC_REDIRECT               TAOS_DEF_ERROR_CODE(0, 0x0004)
+// #define TSDB_CODE_RPC_REDIRECT               TAOS_DEF_ERROR_CODE(0, 0x0004) //2.x
 // #define TSDB_CODE_RPC_NOT_READY              TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x
 // #define TSDB_CODE_RPC_ALREADY_PROCESSED      TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x
 // #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x
diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index a3430d0bde..12aba130d5 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -226,9 +226,9 @@ static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { rpcRegisterBrokenLin
 static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcReleaseHandle(pHandle, type); }
 
 static bool rpcRfp(int32_t code, tmsg_t msgType) {
-  if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_MNODE_NOT_FOUND ||
-      code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_RPC_BROKEN_LINK ||
-      code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) {
+  if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_MNODE_NOT_FOUND ||
+      code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_VND_STOPPED ||
+      code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) {
     if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH ||
         msgType == TDMT_SCH_MERGE_FETCH) {
       return false;
diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h
index e2b7327e8f..15e64ffaed 100644
--- a/source/dnode/vnode/src/inc/vnd.h
+++ b/source/dnode/vnode/src/inc/vnd.h
@@ -100,7 +100,7 @@ int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
 int32_t vnodeSyncStart(SVnode* pVnode);
 void    vnodeSyncPreClose(SVnode* pVnode);
 void    vnodeSyncClose(SVnode* pVnode);
-void    vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg);
+void    vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg, int32_t code);
 bool    vnodeIsLeader(SVnode* pVnode);
 bool    vnodeIsRoleLeader(SVnode* pVnode);
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index d63c45fc0e..fc6b126cc5 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -344,7 +344,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   vTrace("message in vnode query queue is processing");
   // if ((pMsg->msgType == TDMT_SCH_QUERY) && !vnodeIsLeader(pVnode)) {
   if ((pMsg->msgType == TDMT_SCH_QUERY) && !syncIsReadyForRead(pVnode->sync)) {
-    vnodeRedirectRpcMsg(pVnode, pMsg);
+    vnodeRedirectRpcMsg(pVnode, pMsg, terrno);
     return 0;
   }
 
@@ -367,12 +367,12 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
        pMsg->msgType == TDMT_VND_BATCH_META) &&
       !syncIsReadyForRead(pVnode->sync)) {
     //      !vnodeIsLeader(pVnode)) {
-    vnodeRedirectRpcMsg(pVnode, pMsg);
+    vnodeRedirectRpcMsg(pVnode, pMsg, terrno);
     return 0;
   }
 
   if (pMsg->msgType == TDMT_VND_TMQ_CONSUME && !pVnode->restored) {
-    vnodeRedirectRpcMsg(pVnode, pMsg);
+    vnodeRedirectRpcMsg(pVnode, pMsg, TSDB_CODE_SYN_RESTORING);
     return 0;
   }
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index ed612ff6a6..a155e8da86 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -53,7 +53,7 @@ static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
   }
 }
 
-void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) {
+void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) {
   SEpSet newEpSet = {0};
   syncGetRetryEpSet(pVnode->sync, &newEpSet);
 
@@ -66,7 +66,9 @@ void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) {
   }
   pMsg->info.hasEpSet = 1;
 
-  SRpcMsg rsp = {.code = TSDB_CODE_SYN_NOT_LEADER, .info = pMsg->info, .msgType = pMsg->msgType + 1};
+  if (code == 0) code = TSDB_CODE_SYN_NOT_LEADER;
+
+  SRpcMsg rsp = {.code = code, .info = pMsg->info, .msgType = pMsg->msgType + 1};
   int32_t contLen = tSerializeSEpSet(NULL, 0, &newEpSet);
 
   rsp.pCont = rpcMallocCont(contLen);
@@ -98,7 +100,7 @@ static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
 
 static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) {
   if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) {
-    vnodeRedirectRpcMsg(pVnode, pMsg);
+    vnodeRedirectRpcMsg(pVnode, pMsg, code);
   } else {
     const STraceId *trace = &pMsg->info.traceId;
     vGError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", pVnode->config.vgId, pMsg, tstrerror(code), code);
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index d588e2db1c..7339d487d1 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -1519,7 +1519,9 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
     transFreeMsg(pResp->pCont);
     transUnrefCliHandle(pConn);
   } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR ||
-             code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED) {
+             code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED ||
+             code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING ||
+             code == TSDB_CODE_APP_IS_STOPPING) {
     tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen);
     noDelay = cliResetEpset(pCtx, pResp, true);
     transFreeMsg(pResp->pCont);

From 9e803332fce3d007fee29244b116eec10f8697bc Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 2 Dec 2022 23:10:12 +0800
Subject: [PATCH 31/32] refact: adjust error codes

---
 include/util/taoserror.h              | 94 +++++++++++++--------------
 source/dnode/mnode/impl/src/mndAcct.c |  6 +-
 source/util/src/terror.c              |  3 -
 3 files changed, 50 insertions(+), 53 deletions(-)

diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 2baf4d14e2..490dbf7744 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -40,61 +40,61 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_FAILED                    -1   // unknown or needn't tell detail error
 
 // rpc
-// #define TSDB_CODE_RPC_ACTION_IN_PROGRESS     TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x
-// #define TSDB_CODE_RPC_AUTH_REQUIRED          TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x
-// #define TSDB_CODE_RPC_AUTH_FAILURE           TAOS_DEF_ERROR_CODE(0, 0x0003) //2.x
-// #define TSDB_CODE_RPC_REDIRECT               TAOS_DEF_ERROR_CODE(0, 0x0004) //2.x
-// #define TSDB_CODE_RPC_NOT_READY              TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x
-// #define TSDB_CODE_RPC_ALREADY_PROCESSED      TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x
-// #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x
-// #define TSDB_CODE_RPC_MISMATCHED_LINK_ID     TAOS_DEF_ERROR_CODE(0, 0x0008) //2.x
-// #define TSDB_CODE_RPC_TOO_SLOW               TAOS_DEF_ERROR_CODE(0, 0x0009) //2.x
-// #define TSDB_CODE_RPC_MAX_SESSIONS           TAOS_DEF_ERROR_CODE(0, 0x000A) //2.x
+// #define TSDB_CODE_RPC_ACTION_IN_PROGRESS     TAOS_DEF_ERROR_CODE(0, 0x0001) // 2.x
+// #define TSDB_CODE_RPC_AUTH_REQUIRED          TAOS_DEF_ERROR_CODE(0, 0x0002) // 2.x
+// #define TSDB_CODE_RPC_AUTH_FAILURE           TAOS_DEF_ERROR_CODE(0, 0x0003) // 2.x
+// #define TSDB_CODE_RPC_REDIRECT               TAOS_DEF_ERROR_CODE(0, 0x0004) // 2.x
+// #define TSDB_CODE_RPC_NOT_READY              TAOS_DEF_ERROR_CODE(0, 0x0005) // 2.x
+// #define TSDB_CODE_RPC_ALREADY_PROCESSED      TAOS_DEF_ERROR_CODE(0, 0x0006) // 2.x
+// #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) // 2.x
+// #define TSDB_CODE_RPC_MISMATCHED_LINK_ID     TAOS_DEF_ERROR_CODE(0, 0x0008) // 2.x
+// #define TSDB_CODE_RPC_TOO_SLOW               TAOS_DEF_ERROR_CODE(0, 0x0009) // 2.x
+// #define TSDB_CODE_RPC_MAX_SESSIONS           TAOS_DEF_ERROR_CODE(0, 0x000A) // 2.x
 #define TSDB_CODE_RPC_NETWORK_UNAVAIL           TAOS_DEF_ERROR_CODE(0, 0x000B)
-// #define TSDB_CODE_RPC_APP_ERROR              TAOS_DEF_ERROR_CODE(0, 0x000C) //2.x
-// #define TSDB_CODE_RPC_UNEXPECTED_RESPONSE    TAOS_DEF_ERROR_CODE(0, 0x000D) //2.x
-// #define TSDB_CODE_RPC_INVALID_VALUE          TAOS_DEF_ERROR_CODE(0, 0x000E) //2.x
-// #define TSDB_CODE_RPC_INVALID_TRAN_ID        TAOS_DEF_ERROR_CODE(0, 0x000F) //2.x
-// #define TSDB_CODE_RPC_INVALID_SESSION_ID     TAOS_DEF_ERROR_CODE(0, 0x0010) //2.x
-// #define TSDB_CODE_RPC_INVALID_MSG_TYPE       TAOS_DEF_ERROR_CODE(0, 0x0011) //2.x
-// #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE  TAOS_DEF_ERROR_CODE(0, 0x0012) //2.x
-#define TSDB_CODE_TIME_UNSYNCED                 TAOS_DEF_ERROR_CODE(0, 0x0013)
-// #define TSDB_CODE_APP_NOT_READY              TAOS_DEF_ERROR_CODE(0, 0x0014) //2.x
+// #define TSDB_CODE_RPC_APP_ERROR              TAOS_DEF_ERROR_CODE(0, 0x000C) // 2.x
+// #define TSDB_CODE_RPC_UNEXPECTED_RESPONSE    TAOS_DEF_ERROR_CODE(0, 0x000D) // 2.x
+// #define TSDB_CODE_RPC_INVALID_VALUE          TAOS_DEF_ERROR_CODE(0, 0x000E) // 2.x
+// #define TSDB_CODE_RPC_INVALID_TRAN_ID        TAOS_DEF_ERROR_CODE(0, 0x000F) // 2.x
+// #define TSDB_CODE_RPC_INVALID_SESSION_ID     TAOS_DEF_ERROR_CODE(0, 0x0010) // 2.x
+// #define TSDB_CODE_RPC_INVALID_MSG_TYPE       TAOS_DEF_ERROR_CODE(0, 0x0011) // 2.x
+// #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE  TAOS_DEF_ERROR_CODE(0, 0x0012) // 2.x
+#define TSDB_CODE_TIME_UNSYNCED                 TAOS_DEF_ERROR_CODE(0, 0x0013) //
+// #define TSDB_CODE_APP_NOT_READY              TAOS_DEF_ERROR_CODE(0, 0x0014) // 2.x
 #define TSDB_CODE_RPC_FQDN_ERROR                TAOS_DEF_ERROR_CODE(0, 0x0015)
-// #define TSDB_CODE_RPC_INVALID_VERSION        TAOS_DEF_ERROR_CODE(0, 0x0016) //2.x
-#define TSDB_CODE_RPC_PORT_EADDRINUSE           TAOS_DEF_ERROR_CODE(0, 0x0017)
-#define TSDB_CODE_RPC_BROKEN_LINK               TAOS_DEF_ERROR_CODE(0, 0x0018)
-#define TSDB_CODE_RPC_TIMEOUT                   TAOS_DEF_ERROR_CODE(0, 0x0019)
+// #define TSDB_CODE_RPC_INVALID_VERSION        TAOS_DEF_ERROR_CODE(0, 0x0016) // 2.x
+#define TSDB_CODE_RPC_PORT_EADDRINUSE           TAOS_DEF_ERROR_CODE(0, 0x0017) //
+#define TSDB_CODE_RPC_BROKEN_LINK               TAOS_DEF_ERROR_CODE(0, 0x0018) //
+#define TSDB_CODE_RPC_TIMEOUT                   TAOS_DEF_ERROR_CODE(0, 0x0019) //
 
 //common & util
-#define TSDB_CODE_OPS_NOT_SUPPORT               TAOS_DEF_ERROR_CODE(0, 0x0100)
-#define TSDB_CODE_MEMORY_CORRUPTED              TAOS_DEF_ERROR_CODE(0, 0x0101)
+#define TSDB_CODE_OPS_NOT_SUPPORT               TAOS_DEF_ERROR_CODE(0, 0x0100) //
+// #define TSDB_CODE_MEMORY_CORRUPTED           TAOS_DEF_ERROR_CODE(0, 0x0101) // 2.x
 #define TSDB_CODE_OUT_OF_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0102)
 // #define TSDB_CODE_COM_INVALID_CFG_MSG        TAOS_DEF_ERROR_CODE(0, 0x0103) // 2.x
-#define TSDB_CODE_FILE_CORRUPTED                TAOS_DEF_ERROR_CODE(0, 0x0104)
-#define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0105)
-#define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0106)
-#define TSDB_CODE_REF_ID_REMOVED                TAOS_DEF_ERROR_CODE(0, 0x0107)
-#define TSDB_CODE_REF_INVALID_ID                TAOS_DEF_ERROR_CODE(0, 0x0108)
-#define TSDB_CODE_REF_ALREADY_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0109)
-#define TSDB_CODE_REF_NOT_EXIST                 TAOS_DEF_ERROR_CODE(0, 0x010A)
+#define TSDB_CODE_FILE_CORRUPTED                TAOS_DEF_ERROR_CODE(0, 0x0104) //
+#define TSDB_CODE_REF_NO_MEMORY                 TAOS_DEF_ERROR_CODE(0, 0x0105) // internal
+#define TSDB_CODE_REF_FULL                      TAOS_DEF_ERROR_CODE(0, 0x0106) // internal
+#define TSDB_CODE_REF_ID_REMOVED                TAOS_DEF_ERROR_CODE(0, 0x0107) // internal
+#define TSDB_CODE_REF_INVALID_ID                TAOS_DEF_ERROR_CODE(0, 0x0108) // internal
+#define TSDB_CODE_REF_ALREADY_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0109) // internal
+#define TSDB_CODE_REF_NOT_EXIST                 TAOS_DEF_ERROR_CODE(0, 0x010A) // internal
 
-#define TSDB_CODE_APP_ERROR                     TAOS_DEF_ERROR_CODE(0, 0x0110)
-#define TSDB_CODE_ACTION_IN_PROGRESS            TAOS_DEF_ERROR_CODE(0, 0x0111)
-#define TSDB_CODE_OUT_OF_RANGE                  TAOS_DEF_ERROR_CODE(0, 0x0112)
-#define TSDB_CODE_OUT_OF_SHM_MEM                TAOS_DEF_ERROR_CODE(0, 0x0113)
-#define TSDB_CODE_INVALID_SHM_ID                TAOS_DEF_ERROR_CODE(0, 0x0114)
-#define TSDB_CODE_INVALID_MSG                   TAOS_DEF_ERROR_CODE(0, 0x0115)
+#define TSDB_CODE_APP_ERROR                     TAOS_DEF_ERROR_CODE(0, 0x0110) //
+#define TSDB_CODE_ACTION_IN_PROGRESS            TAOS_DEF_ERROR_CODE(0, 0x0111) // internal
+#define TSDB_CODE_OUT_OF_RANGE                  TAOS_DEF_ERROR_CODE(0, 0x0112) //
+// #define TSDB_CODE_OUT_OF_SHM_MEM             TAOS_DEF_ERROR_CODE(0, 0x0113)
+// #define TSDB_CODE_INVALID_SHM_ID             TAOS_DEF_ERROR_CODE(0, 0x0114)
+#define TSDB_CODE_INVALID_MSG                   TAOS_DEF_ERROR_CODE(0, 0x0115) //
 #define TSDB_CODE_INVALID_MSG_LEN               TAOS_DEF_ERROR_CODE(0, 0x0116) //
-#define TSDB_CODE_INVALID_PTR                   TAOS_DEF_ERROR_CODE(0, 0x0117)
-#define TSDB_CODE_INVALID_PARA                  TAOS_DEF_ERROR_CODE(0, 0x0118)
-#define TSDB_CODE_INVALID_CFG                   TAOS_DEF_ERROR_CODE(0, 0x0119)
-#define TSDB_CODE_INVALID_OPTION                TAOS_DEF_ERROR_CODE(0, 0x011A)
-#define TSDB_CODE_INVALID_JSON_FORMAT           TAOS_DEF_ERROR_CODE(0, 0x011B)
-#define TSDB_CODE_INVALID_VERSION_NUMBER        TAOS_DEF_ERROR_CODE(0, 0x011C)
-#define TSDB_CODE_INVALID_VERSION_STRING        TAOS_DEF_ERROR_CODE(0, 0x011D)
-#define TSDB_CODE_VERSION_NOT_COMPATIBLE        TAOS_DEF_ERROR_CODE(0, 0x011E)
-#define TSDB_CODE_CHECKSUM_ERROR                TAOS_DEF_ERROR_CODE(0, 0x011F)
+#define TSDB_CODE_INVALID_PTR                   TAOS_DEF_ERROR_CODE(0, 0x0117) // internal
+#define TSDB_CODE_INVALID_PARA                  TAOS_DEF_ERROR_CODE(0, 0x0118) //
+#define TSDB_CODE_INVALID_CFG                   TAOS_DEF_ERROR_CODE(0, 0x0119) // internal
+#define TSDB_CODE_INVALID_OPTION                TAOS_DEF_ERROR_CODE(0, 0x011A) // internal
+#define TSDB_CODE_INVALID_JSON_FORMAT           TAOS_DEF_ERROR_CODE(0, 0x011B) // internal
+#define TSDB_CODE_INVALID_VERSION_NUMBER        TAOS_DEF_ERROR_CODE(0, 0x011C) // internal
+#define TSDB_CODE_INVALID_VERSION_STRING        TAOS_DEF_ERROR_CODE(0, 0x011D) // internal
+#define TSDB_CODE_VERSION_NOT_COMPATIBLE        TAOS_DEF_ERROR_CODE(0, 0x011E) // internal
+#define TSDB_CODE_CHECKSUM_ERROR                TAOS_DEF_ERROR_CODE(0, 0x011F) // internal
 
 #define TSDB_CODE_COMPRESS_ERROR                TAOS_DEF_ERROR_CODE(0, 0x0120)
 #define TSDB_CODE_MSG_NOT_PROCESSED             TAOS_DEF_ERROR_CODE(0, 0x0121) //
diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c
index ce2ec7b23a..148e21b507 100644
--- a/source/dnode/mnode/impl/src/mndAcct.c
+++ b/source/dnode/mnode/impl/src/mndAcct.c
@@ -220,7 +220,7 @@ static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) {
     return -1;
   }
 
-  terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+  terrno = TSDB_CODE_OPS_NOT_SUPPORT;
   mError("failed to process create acct request since %s", terrstr());
   return -1;
 }
@@ -230,7 +230,7 @@ static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) {
     return -1;
   }
 
-  terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+  terrno = TSDB_CODE_OPS_NOT_SUPPORT;
   mError("failed to process create acct request since %s", terrstr());
   return -1;
 }
@@ -240,7 +240,7 @@ static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) {
     return -1;
   }
 
-  terrno = TSDB_CODE_MSG_NOT_PROCESSED;
+  terrno = TSDB_CODE_OPS_NOT_SUPPORT;
   mError("failed to process create acct request since %s", terrstr());
   return -1;
 }
\ No newline at end of file
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 2110c0b3aa..18eeca030e 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -55,7 +55,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT,                  "Conn read timeout")
 //common & util
 TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED,                "Client and server's time is not synchronized")
 TAOS_DEFINE_ERROR(TSDB_CODE_OPS_NOT_SUPPORT,              "Operation not supported")
-TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED,             "Memory corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_MEMORY,                "Out of Memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED,               "Data file corrupted")
 TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY,                "Ref out of memory")
@@ -68,8 +67,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST,                "Ref is not there")
 TAOS_DEFINE_ERROR(TSDB_CODE_APP_ERROR,                    "Unexpected generic error")
 TAOS_DEFINE_ERROR(TSDB_CODE_ACTION_IN_PROGRESS,           "Action in progress")
 TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RANGE,                 "Out of range")
-TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_SHM_MEM,               "Out of Shared memory")
-TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SHM_ID,               "Invalid SHM ID")
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG,                  "Invalid message")
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG_LEN,              "Invalid message len")
 TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PTR,                  "Invalid pointer")

From c20e6078259ba0090da7a0759f09acdb3d1a9f5e Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Fri, 2 Dec 2022 23:32:31 +0800
Subject: [PATCH 32/32] fix: compile error

---
 source/dnode/mnode/impl/test/acct/acct.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp
index 46a9a465eb..1f59a7fcca 100644
--- a/source/dnode/mnode/impl/test/acct/acct.cpp
+++ b/source/dnode/mnode/impl/test/acct/acct.cpp
@@ -32,7 +32,7 @@ TEST_F(MndTestAcct, 01_Create_Acct) {
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
-  ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
+  ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
 }
 
 TEST_F(MndTestAcct, 02_Alter_Acct) {
@@ -42,7 +42,7 @@ TEST_F(MndTestAcct, 02_Alter_Acct) {
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
-  ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
+  ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
 }
 
 TEST_F(MndTestAcct, 03_Drop_Acct) {
@@ -52,5 +52,5 @@ TEST_F(MndTestAcct, 03_Drop_Acct) {
 
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
-  ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
+  ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
 }