From 797af5373ddca04177a64da4c78ebcab517e3b7c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 29 Nov 2022 10:59:40 +0800
Subject: [PATCH 01/23] enh(query): improve the multi-way merge performance.

---
 source/libs/executor/src/exchangeoperator.c |  7 +---
 source/libs/executor/src/sortoperator.c     |  5 ++-
 source/libs/executor/src/tsort.c            | 46 +++++++++++----------
 3 files changed, 30 insertions(+), 28 deletions(-)

diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c
index b2ddff45a4..001d668749 100644
--- a/source/libs/executor/src/exchangeoperator.c
+++ b/source/libs/executor/src/exchangeoperator.c
@@ -209,7 +209,7 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) {
   }
 }
 
-static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) {
+static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) {
   SExchangeInfo* pExchangeInfo = pOperator->info;
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
 
@@ -318,7 +318,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
   pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock);
 
   pOperator->fpSet =
-      createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL);
+      createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, NULL);
   return pOperator;
 
 _error:
@@ -581,13 +581,10 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) {
 
   pOperator->status = OP_RES_TO_RETURN;
   pOperator->cost.openCost = taosGetTimestampUs() - startTs;
-
-  tsem_wait(&pExchangeInfo->ready);
   if (isTaskKilled(pTaskInfo)) {
     longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
   }
 
-  tsem_post(&pExchangeInfo->ready);
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index ec754f31b0..2f504cee3d 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -559,7 +559,7 @@ typedef struct SMultiwayMergeOperatorInfo {
   STupleHandle*  prefetchedTuple;
 } SMultiwayMergeOperatorInfo;
 
-int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
+int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) {
   SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
   SExecTaskInfo*              pTaskInfo = pOperator->pTaskInfo;
 
@@ -580,6 +580,7 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
     SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
     ps->param = pOperator->pDownstream[i];
     ps->onlyRef = true;
+
     tsortAddSource(pInfo->pSortHandle, ps);
   }
 
@@ -792,7 +793,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
   pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1);  // one additional is reserved for merged result.
 
   setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo);
-  pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL,
+  pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL,
                                          destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo);
 
   code = appendDownstream(pOperator, downStreams, numStreams);
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 3f91142708..4db76b82f5 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -34,14 +34,12 @@ struct SSortHandle {
   int32_t        pageSize;
   int32_t        numOfPages;
   SDiskbasedBuf* pBuf;
-
-  SArray* pSortInfo;
-  SArray* pOrderedSource;
-
-  int32_t  loops;
-  uint64_t sortElapsed;
-  int64_t  startTs;
-  uint64_t totalElapsed;
+  SArray*        pSortInfo;
+  SArray*        pOrderedSource;
+  int32_t        loops;
+  uint64_t       sortElapsed;
+  int64_t        startTs;
+  uint64_t       totalElapsed;
 
   int32_t           sourceId;
   SSDataBlock*      pDataBlock;
@@ -99,9 +97,9 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page
 }
 
 static int32_t sortComparCleanup(SMsortComparParam* cmpParam) {
+  // NOTICE: pSource may be, if it is SORT_MULTISOURCE_MERGE
   for (int32_t i = 0; i < cmpParam->numOfSources; ++i) {
-    SSortSource* pSource =
-        cmpParam->pSources[i];  // NOTICE: pSource may be SGenericSource *, if it is SORT_MULTISOURCE_MERGE
+    SSortSource* pSource = cmpParam->pSources[i];
     blockDataDestroy(pSource->src.pBlock);
     taosMemoryFreeClear(pSource);
   }
@@ -228,15 +226,15 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
   return doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId, pPageIdList);
 }
 
-static void setCurrentSourceIsDone(SSortSource* pSource, SSortHandle* pHandle) {
+static void setCurrentSourceDone(SSortSource* pSource, SSortHandle* pHandle) {
   pSource->src.rowIndex = -1;
   ++pHandle->numOfCompletedSources;
 }
 
-static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex,
+static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32_t startIndex, int32_t endIndex,
                               SSortHandle* pHandle) {
-  cmpParam->pSources = taosArrayGet(pSources, startIndex);
-  cmpParam->numOfSources = (endIndex - startIndex + 1);
+  pParam->pSources = taosArrayGet(pSources, startIndex);
+  pParam->numOfSources = (endIndex - startIndex + 1);
 
   int32_t code = 0;
 
@@ -244,7 +242,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
   if (pHandle->pBuf == NULL) {
     if (!osTempSpaceAvailable()) {
       code = TSDB_CODE_NO_AVAIL_DISK;
-      qError("Sort compare init failed since %s", terrstr(code));
+      qError("Sort compare init failed since %s, %s", terrstr(code), pHandle->idStr);
       return code;
     }
 
@@ -257,12 +255,12 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
   }
 
   if (pHandle->type == SORT_SINGLESOURCE_SORT) {
-    for (int32_t i = 0; i < cmpParam->numOfSources; ++i) {
-      SSortSource* pSource = cmpParam->pSources[i];
+    for (int32_t i = 0; i < pParam->numOfSources; ++i) {
+      SSortSource* pSource = pParam->pSources[i];
 
       // set current source is done
       if (taosArrayGetSize(pSource->pageIdList) == 0) {
-        setCurrentSourceIsDone(pSource, pHandle);
+        setCurrentSourceDone(pSource, pHandle);
         continue;
       }
 
@@ -277,15 +275,21 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int
       releaseBufPage(pHandle->pBuf, pPage);
     }
   } else {
-    for (int32_t i = 0; i < cmpParam->numOfSources; ++i) {
-      SSortSource* pSource = cmpParam->pSources[i];
+    qDebug("start init for the multiway merge sort, %s", pHandle->idStr);
+    int64_t st = taosGetTimestampUs();
+
+    for (int32_t i = 0; i < pParam->numOfSources; ++i) {
+      SSortSource* pSource = pParam->pSources[i];
       pSource->src.pBlock = pHandle->fetchfp(pSource->param);
 
       // set current source is done
       if (pSource->src.pBlock == NULL) {
-        setCurrentSourceIsDone(pSource, pHandle);
+        setCurrentSourceDone(pSource, pHandle);
       }
     }
+
+    int64_t et = taosGetTimestampUs();
+    qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr);
   }
 
   return code;

From a1585f16f10076cffc0d2c82737b4e42e5c3fedb Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 29 Nov 2022 11:06:20 +0800
Subject: [PATCH 02/23] ehn(query): improve the merge sort performance.

---
 source/libs/executor/src/sortoperator.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 2f504cee3d..0287af62d2 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -577,8 +577,13 @@ int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) {
   tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort);
 
   for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
+    SOperatorInfo* pDownstream = pOperator->pDownstream[i];
+    if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) {
+      pDownstream->fpSet._openFn(pDownstream);
+    }
+
     SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
-    ps->param = pOperator->pDownstream[i];
+    ps->param = pDownstream;
     ps->onlyRef = true;
 
     tsortAddSource(pInfo->pSortHandle, ps);

From dac930352b7764c3f0b36b9703a8251ad1d31784 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 29 Nov 2022 12:22:04 +0800
Subject: [PATCH 03/23] enh(query): optimize the perf in compare.

---
 include/common/ttypes.h            |  2 -
 source/libs/scalar/src/sclvector.c | 62 +++++++++++++++++++-----------
 2 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/include/common/ttypes.h b/include/common/ttypes.h
index 761ffd0f1c..6350057c1f 100644
--- a/include/common/ttypes.h
+++ b/include/common/ttypes.h
@@ -278,11 +278,9 @@ typedef struct {
 #define IS_VALID_TINYINT(_t)   ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
 #define IS_VALID_SMALLINT(_t)  ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
 #define IS_VALID_INT(_t)       ((_t) >= INT32_MIN && (_t) <= INT32_MAX)
-#define IS_VALID_BIGINT(_t)    ((_t) >= INT64_MIN && (_t) <= INT64_MAX)
 #define IS_VALID_UTINYINT(_t)  ((_t) >= 0 && (_t) <= UINT8_MAX)
 #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX)
 #define IS_VALID_UINT(_t)      ((_t) >= 0 && (_t) <= UINT32_MAX)
-#define IS_VALID_UBIGINT(_t)   ((_t) >= 0 && (_t) <= UINT64_MAX)
 #define IS_VALID_FLOAT(_t)     ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)    ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index c4ff5b2b01..98391d95f4 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -1543,9 +1543,44 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut,
 int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t startIndex,
                             int32_t numOfRows, int32_t step, __compar_fn_t fp, int32_t optr) {
   int32_t num = 0;
-  bool *  pRes = (bool *)pOut->columnData->pData;
+  bool   *pRes = (bool *)pOut->columnData->pData;
 
-  if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_JSON || GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_JSON) {
+  if (IS_MATHABLE_TYPE(GET_PARAM_TYPE(pLeft)) && IS_MATHABLE_TYPE(GET_PARAM_TYPE(pRight))) {
+    if (!(pLeft->columnData->hasNull || pRight->columnData->hasNull)) {
+      for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) {
+        int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i;
+        int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i;
+
+        char *pLeftData = colDataGetData(pLeft->columnData, leftIndex);
+        char *pRightData = colDataGetData(pRight->columnData, rightIndex);
+
+        pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData);
+        if (pRes[i]) {
+          ++num;
+        }
+      }
+    } else {
+      for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) {
+        int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i;
+        int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i;
+
+        if (colDataIsNull_f(pLeft->columnData->nullbitmap, leftIndex) ||
+            colDataIsNull_f(pRight->columnData->nullbitmap, rightIndex)) {
+          pRes[i] = false;
+          continue;
+        }
+
+        char *pLeftData = colDataGetData(pLeft->columnData, leftIndex);
+        char *pRightData = colDataGetData(pRight->columnData, rightIndex);
+
+        pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData);
+        if (pRes[i]) {
+          ++num;
+        }
+      }
+    }
+  } else {
+    //  if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_JSON || GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_JSON) {
     for (int32_t i = startIndex; i < numOfRows && i >= startIndex; i += step) {
       int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i;
       int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i;
@@ -1556,8 +1591,8 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa
         continue;
       }
 
-      char *  pLeftData = colDataGetData(pLeft->columnData, leftIndex);
-      char *  pRightData = colDataGetData(pRight->columnData, rightIndex);
+      char   *pLeftData = colDataGetData(pLeft->columnData, leftIndex);
+      char   *pRightData = colDataGetData(pRight->columnData, rightIndex);
       int64_t leftOut = 0;
       int64_t rightOut = 0;
       bool    freeLeft = false;
@@ -1592,25 +1627,6 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa
         taosMemoryFreeClear(pRightData);
       }
     }
-  } else {
-    for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) {
-      int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i;
-      int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i;
-
-      if (colDataIsNull_s(pLeft->columnData, leftIndex) ||
-          colDataIsNull_s(pRight->columnData, rightIndex)) {
-        pRes[i] = false;
-        continue;
-      }
-
-      char *pLeftData = colDataGetData(pLeft->columnData, leftIndex);
-      char *pRightData = colDataGetData(pRight->columnData, rightIndex);
-
-      pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData);
-      if (pRes[i]) {
-        ++num;
-      }
-    }
   }
 
   return num;

From 445c4f28f0d09f3f38813f5c26f016c10350f265 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 29 Nov 2022 23:34:22 +0800
Subject: [PATCH 04/23] enh(query): optimize the perf.

---
 source/common/src/tdatablock.c          | 5 ++++-
 source/dnode/vnode/src/tsdb/tsdbRead.c  | 2 +-
 source/libs/executor/src/sortoperator.c | 3 +--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index a0f795a729..20eda52aac 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1139,6 +1139,9 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
 void blockDataCleanup(SSDataBlock* pDataBlock) {
   SDataBlockInfo* pInfo = &pDataBlock->info;
 
+  int32_t existedRows = pInfo->rows;
+  ASSERT(existedRows <= pDataBlock->info.capacity);
+
   pInfo->rows = 0;
   pInfo->id.uid = 0;
   pInfo->id.groupId = 0;
@@ -1152,7 +1155,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) {
   size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    colInfoDataCleanup(p, pInfo->capacity);
+    colInfoDataCleanup(p, existedRows);
   }
 }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 4ba311212a..660d189451 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -3589,7 +3589,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
 
   SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i);
   if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-    colDataAppendInt64(pColData, outputRowIndex, &pBlockData->aTSKEY[rowIndex]);
+    ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex];
     i += 1;
   }
 
diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c
index 0287af62d2..b355b85861 100644
--- a/source/libs/executor/src/sortoperator.c
+++ b/source/libs/executor/src/sortoperator.c
@@ -720,7 +720,6 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) {
   }
 
   qDebug("start to merge final sorted rows, %s", GET_TASKID(pTaskInfo));
-
   SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->matchInfo.pList, pOperator);
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
@@ -787,7 +786,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
   SPhysiNode*  pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
   SSDataBlock* pInputBlock = createDataBlockFromDescNode(pChildNode->pOutputDataBlockDesc);
 
-  initResultSizeInfo(&pOperator->resultInfo, 4096);
+  initResultSizeInfo(&pOperator->resultInfo, 1024);
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
 
   pInfo->groupSort = pMergePhyNode->groupSort;

From a8703c7fc22d661d1fbd7e196ddeb475b4ffbfd7 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 00:11:21 +0800
Subject: [PATCH 05/23] enh(query): optimize the perf.

---
 source/libs/executor/src/timewindowoperator.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index a6a477a9e3..dae2dafe9e 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1286,7 +1286,6 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
     return NULL;
   }
 
-  blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
   while (1) {
     doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
     doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL);
@@ -1737,7 +1736,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
   pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId;
 
   size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
-  initResultSizeInfo(&pOperator->resultInfo, 4096);
+  initResultSizeInfo(&pOperator->resultInfo, 1024);
+  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
 
   int32_t    num = 0;
   SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);

From 3f0c93a62308112a1ba2a5093277fb8c342ff509 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 14:53:25 +0800
Subject: [PATCH 06/23] enh(query): optimize query perf.

---
 source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index 294a4bd3e4..c149e52fed 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -40,7 +40,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd
   }
   pFD->szPage = szPage;
   pFD->pgno = 0;
-  pFD->pBuf = taosMemoryCalloc(1, szPage);
+  pFD->pBuf = taosMemoryMalloc(szPage);
   if (pFD->pBuf == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     taosCloseFile(&pFD->pFD);

From ae816331441c1602d500a5f3d208da76e1fe323b Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 14:55:27 +0800
Subject: [PATCH 07/23] enh(query): optimize query perf.

---
 source/common/src/tdatablock.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 20eda52aac..af45db053a 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1242,7 +1242,7 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
   size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, true);
+    code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, false);
     if (code) {
       return code;
     }

From 8dcbafb78066547b8f8f04c4772e2f7a559f1415 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 15:24:03 +0800
Subject: [PATCH 08/23] enh(query): avoiding unnecessary remove operation.

---
 source/util/src/tpagedbuf.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c
index e1a43ace47..b7dbdad053 100644
--- a/source/util/src/tpagedbuf.c
+++ b/source/util/src/tpagedbuf.c
@@ -507,7 +507,9 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
 
   dBufPrintStatis(pBuf);
 
+  bool needRemoveFile = false;
   if (pBuf->pFile != NULL) {
+    needRemoveFile = true;
     uDebug(
         "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page "
         "size:%.2f Kb, %s\n",
@@ -534,9 +536,14 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
     }
   }
 
-  if (taosRemoveFile(pBuf->path) < 0) {
-    uDebug("WARNING tPage remove file failed. path=%s", pBuf->path);
+  if (needRemoveFile) {
+    int32_t ret = taosRemoveFile(pBuf->path);
+    if (ret != 0) {  // print the error and discard this error info
+      int32_t code = TAOS_SYSTEM_ERROR(errno);
+      uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(code));
+    }
   }
+
   taosMemoryFreeClear(pBuf->path);
 
   size_t n = taosArrayGetSize(pBuf->pIdList);

From d66b7f3de07d6bb1149b5be41b5aab6ac0fa2e7e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 15:25:19 +0800
Subject: [PATCH 09/23] refactor: do some internal refactor.

---
 source/util/src/tpagedbuf.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c
index b7dbdad053..0f4c51060b 100644
--- a/source/util/src/tpagedbuf.c
+++ b/source/util/src/tpagedbuf.c
@@ -539,8 +539,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
   if (needRemoveFile) {
     int32_t ret = taosRemoveFile(pBuf->path);
     if (ret != 0) {  // print the error and discard this error info
-      int32_t code = TAOS_SYSTEM_ERROR(errno);
-      uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(code));
+      uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(errno));
     }
   }
 

From 4fef284cfe07d71afe4f088b0d7767f79f1ae636 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 21:26:27 +0800
Subject: [PATCH 10/23] refactor: do some internal refactor.

---
 include/libs/function/function.h         | 59 +++++++-----------------
 include/libs/function/tudf.h             | 26 +++++++++++
 source/dnode/mgmt/node_util/inc/dmUtil.h |  2 +-
 3 files changed, 43 insertions(+), 44 deletions(-)

diff --git a/include/libs/function/function.h b/include/libs/function/function.h
index 240772bfc2..8fe9d98e57 100644
--- a/include/libs/function/function.h
+++ b/include/libs/function/function.h
@@ -137,22 +137,22 @@ typedef struct SqlFunctionCtx {
   int16_t              functionId;  // function id
   char                *pOutput;     // final result output buffer, point to sdata->data
   int32_t              numOfParams;
-  SFunctParam     *param;  // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
-  SColumnInfoData *pTsOutput;  // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
-  int32_t          offset;
-  struct SResultRowEntryInfo *resultInfo;
-  SSubsidiaryResInfo          subsidiaries;
-  SPoint1                     start;
-  SPoint1                     end;
-  SFuncExecFuncs              fpSet;
-  SScalarFuncExecFuncs        sfp;
-  struct SExprInfo           *pExpr;
-  struct SSDataBlock         *pSrcBlock;
-  struct SSDataBlock         *pDstBlock;  // used by indefinite rows function to set selectivity
-  SSerializeDataHandle        saveHandle;
-  bool                        isStream;
-
-  char udfName[TSDB_FUNC_NAME_LEN];
+  // input parameter, e.g., top(k, 20), the number of results of top query is kept in param
+  SFunctParam         *param;
+  // corresponding output buffer for timestamp of each result, e.g., diff/csum
+  SColumnInfoData     *pTsOutput;
+  int32_t              offset;
+  SResultRowEntryInfo *resultInfo;
+  SSubsidiaryResInfo   subsidiaries;
+  SPoint1              start;
+  SPoint1              end;
+  SFuncExecFuncs       fpSet;
+  SScalarFuncExecFuncs sfp;
+  struct SExprInfo    *pExpr;
+  struct SSDataBlock  *pSrcBlock;
+  struct SSDataBlock  *pDstBlock;  // used by indefinite rows function to set selectivity
+  SSerializeDataHandle saveHandle;
+  char                 udfName[TSDB_FUNC_NAME_LEN];
 } SqlFunctionCtx;
 
 typedef struct tExprNode {
@@ -182,7 +182,6 @@ struct SScalarParam {
 };
 
 void    cleanupResultRowEntry(struct SResultRowEntryInfo *pCell);
-//int32_t getNumOfResult(SqlFunctionCtx *pCtx, int32_t num, SSDataBlock *pResBlock);
 bool    isRowEntryCompleted(struct SResultRowEntryInfo *pEntry);
 bool    isRowEntryInitialized(struct SResultRowEntryInfo *pEntry);
 
@@ -194,32 +193,6 @@ typedef struct SPoint {
 int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2,
                                       int32_t inputType);
 
-///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-// udf api
-/**
- * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf
- * @return error code
- */
-int32_t udfcOpen();
-
-/**
- * destroy udfd proxy
- * @return error code
- */
-int32_t udfcClose();
-
-/**
- * start udfd that serves udf function invocation under dnode startDnodeId
- * @param startDnodeId
- * @return
- */
-int32_t udfStartUdfd(int32_t startDnodeId);
-/**
- * stop udfd
- * @return
- */
-int32_t udfStopUdfd();
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h
index 31cc53bb9f..b71d50d43c 100644
--- a/include/libs/function/tudf.h
+++ b/include/libs/function/tudf.h
@@ -85,6 +85,32 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols,
 
 int32_t cleanUpUdfs();
 
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// udf api
+/**
+ * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf
+ * @return error code
+ */
+int32_t udfcOpen();
+
+/**
+ * destroy udfd proxy
+ * @return error code
+ */
+int32_t udfcClose();
+
+/**
+ * start udfd that serves udf function invocation under dnode startDnodeId
+ * @param startDnodeId
+ * @return
+ */
+int32_t udfStartUdfd(int32_t startDnodeId);
+/**
+ * stop udfd
+ * @return
+ */
+int32_t udfStopUdfd();
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h
index 8719e988e7..2124b387ec 100644
--- a/source/dnode/mgmt/node_util/inc/dmUtil.h
+++ b/source/dnode/mgmt/node_util/inc/dmUtil.h
@@ -39,7 +39,7 @@
 #include "sync.h"
 #include "wal.h"
 
-#include "libs/function/function.h"
+#include "libs/function/tudf.h"
 #ifdef __cplusplus
 extern "C" {
 #endif

From 26ab0894a88f97699cbf9cb6a05bd666e0e618b6 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 30 Nov 2022 21:56:02 +0800
Subject: [PATCH 11/23] fix(query): fix syntax error.

---
 source/libs/executor/src/executil.c           | 2 --
 source/libs/executor/src/timewindowoperator.c | 7 +------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index b135566caa..7c37d66eb6 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1536,8 +1536,6 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
     pCtx->start.key = INT64_MIN;
     pCtx->end.key = INT64_MIN;
     pCtx->numOfParams = pExpr->base.numOfParams;
-    pCtx->isStream = false;
-
     pCtx->param = pFunct->pParam;
     pCtx->saveHandle.currentPage = -1;
   }
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index dae2dafe9e..35548a6ade 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1717,7 +1717,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSuppor
 
 void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
   for (int32_t i = 0; i < numOfExpr; i++) {
-    pCtx[i].isStream = true;
+//    pCtx[i].isStream = true;
   }
 }
 
@@ -1783,11 +1783,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
     goto _error;
   }
 
-  if (isStream) {
-    ASSERT(num > 0);
-    initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
-  }
-
   initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win);
   pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, pInfo);
   if (pInfo->timeWindowInterpo) {

From cefe4be1abef27af09104d7a053ecc3478d93bfb Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 00:34:06 +0800
Subject: [PATCH 12/23] refactor: do some internal refactor.

---
 include/common/tdatablock.h                   |  1 +
 source/common/src/tdatablock.c                | 19 +++++++++
 .../dnode/vnode/src/tsdb/tsdbReaderWriter.c   |  2 +-
 source/libs/executor/src/executorimpl.c       |  2 +-
 source/libs/executor/src/scanoperator.c       |  2 +-
 source/libs/executor/src/timewindowoperator.c | 41 ++++++++++---------
 source/util/src/tarray.c                      |  2 +-
 source/util/src/thash.c                       |  4 +-
 8 files changed, 48 insertions(+), 25 deletions(-)

diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index db6ca65cf0..a5d3a0829d 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -234,6 +234,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
 
 int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload);
 int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
+int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows);
 
 void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
 void blockDataCleanup(SSDataBlock* pDataBlock);
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index af45db053a..f5e73d8f0e 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1239,6 +1239,25 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
     return TSDB_CODE_SUCCESS;
   }
 
+  size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
+    code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, true);
+    if (code) {
+      return code;
+    }
+  }
+
+  pDataBlock->info.capacity = numOfRows;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows) {
+  int32_t code = 0;
+  if (numOfRows == 0 || numOfRows <= pDataBlock->info.capacity) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index c149e52fed..294a4bd3e4 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -40,7 +40,7 @@ static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsd
   }
   pFD->szPage = szPage;
   pFD->pgno = 0;
-  pFD->pBuf = taosMemoryMalloc(szPage);
+  pFD->pBuf = taosMemoryCalloc(1, szPage);
   if (pFD->pBuf == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     taosCloseFile(&pFD->pFD);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 300ba52934..3e28798a3a 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -1749,7 +1749,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n
   pAggSup->currentPageId = -1;
   pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput);
   pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t));
-  pAggSup->pResultRowHashTable = tSimpleHashInit(10, hashFn);
+  pAggSup->pResultRowHashTable = tSimpleHashInit(100, hashFn);
 
   if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 84b7678b9f..ca3734d91e 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -886,7 +886,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
 
   initResultSizeInfo(&pOperator->resultInfo, 4096);
   pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
-  blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
+  blockDataEnsureCapacityNoClear(pInfo->pResBlock, pOperator->resultInfo.capacity);
 
   code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
   if (code != TSDB_CODE_SUCCESS) {
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 35548a6ade..551d668763 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1658,24 +1658,6 @@ static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) {
 static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) {
   // the primary timestamp column
   bool needed = false;
-  pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
-  pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
-
-  {  // ts column
-    SColumn c = {0};
-    c.colId = 1;
-    c.slotId = pInfo->primaryTsIndex;
-    c.type = TSDB_DATA_TYPE_TIMESTAMP;
-    c.bytes = sizeof(int64_t);
-    taosArrayPush(pInfo->pInterpCols, &c);
-
-    SGroupKeys key = {0};
-    key.bytes = c.bytes;
-    key.type = c.type;
-    key.isNull = true;  // to denote no value is assigned yet
-    key.pData = taosMemoryCalloc(1, c.bytes);
-    taosArrayPush(pInfo->pPrevValues, &key);
-  }
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SExprInfo* pExpr = pCtx[i].pExpr;
@@ -1696,6 +1678,27 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
     }
   }
 
+  if (needed) {
+    pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn));
+    pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys));
+
+    {  // ts column
+      SColumn c = {0};
+      c.colId = 1;
+      c.slotId = pInfo->primaryTsIndex;
+      c.type = TSDB_DATA_TYPE_TIMESTAMP;
+      c.bytes = sizeof(int64_t);
+      taosArrayPush(pInfo->pInterpCols, &c);
+
+      SGroupKeys key;
+      key.bytes = c.bytes;
+      key.type = c.type;
+      key.isNull = true;  // to denote no value is assigned yet
+      key.pData = taosMemoryCalloc(1, c.bytes);
+      taosArrayPush(pInfo->pPrevValues, &key);
+    }
+  }
+
   return needed;
 }
 
@@ -1737,7 +1740,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
 
   size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
   initResultSizeInfo(&pOperator->resultInfo, 1024);
-  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+  blockDataEnsureCapacityNoClear(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
 
   int32_t    num = 0;
   SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index f53b1cfd7f..6a16c044f9 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -33,7 +33,7 @@ SArray* taosArrayInit(size_t size, size_t elemSize) {
   }
 
   pArray->size = 0;
-  pArray->pData = taosMemoryCalloc(size, elemSize);
+  pArray->pData = taosMemoryMalloc(size * elemSize);
   if (pArray->pData == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     taosMemoryFree(pArray);
diff --git a/source/util/src/thash.c b/source/util/src/thash.c
index a0411483ca..b07d485c87 100644
--- a/source/util/src/thash.c
+++ b/source/util/src/thash.c
@@ -244,7 +244,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
     capacity = 4;
   }
 
-  SHashObj *pHashObj = (SHashObj *)taosMemoryCalloc(1, sizeof(SHashObj));
+  SHashObj *pHashObj = (SHashObj *)taosMemoryMalloc(sizeof(SHashObj));
   if (pHashObj == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
@@ -264,7 +264,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
 
   ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0);
 
-  pHashObj->hashList = (SHashEntry **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *));
+  pHashObj->hashList = (SHashEntry **)taosMemoryMalloc(pHashObj->capacity * sizeof(void *));
   if (pHashObj->hashList == NULL) {
     taosMemoryFree(pHashObj);
     terrno = TSDB_CODE_OUT_OF_MEMORY;

From c996837fab936338a81491e9c4ae88412091f7ef Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 01:10:37 +0800
Subject: [PATCH 13/23] refactor: do some internal refactor.

---
 source/util/src/tarray.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index 6a16c044f9..f53b1cfd7f 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -33,7 +33,7 @@ SArray* taosArrayInit(size_t size, size_t elemSize) {
   }
 
   pArray->size = 0;
-  pArray->pData = taosMemoryMalloc(size * elemSize);
+  pArray->pData = taosMemoryCalloc(size, elemSize);
   if (pArray->pData == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     taosMemoryFree(pArray);

From f8a2ab8338cedbdb68d3f2c1fc4236c1d95fee57 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 17:24:26 +0800
Subject: [PATCH 14/23] refactor: do some internal refactor.

---
 include/common/tcommon.h                      |   5 +-
 include/common/tdatablock.h                   |   6 +-
 source/common/src/tdatablock.c                |  34 +++--
 source/dnode/vnode/inc/vnode.h                |   3 +-
 source/dnode/vnode/src/tsdb/tsdbRead.c        | 123 ++++++++----------
 source/libs/executor/src/executil.c           |  13 +-
 source/libs/executor/src/executor.c           |   4 +-
 source/libs/executor/src/executorimpl.c       |   2 -
 source/libs/executor/src/scanoperator.c       |  39 ++----
 source/libs/executor/src/sysscanoperator.c    |  85 ++++++------
 source/libs/executor/src/timewindowoperator.c |   6 +-
 source/libs/scalar/src/sclvector.c            |   2 +-
 source/util/src/thash.c                       |   5 +-
 13 files changed, 156 insertions(+), 171 deletions(-)

diff --git a/include/common/tcommon.h b/include/common/tcommon.h
index 77f1879b81..6ec3d5db10 100644
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@@ -249,10 +249,11 @@ typedef struct SColumnInfoData {
 
 typedef struct SQueryTableDataCond {
   uint64_t     suid;
-  int32_t      order;  // desc|asc order to iterate the data block
+  int32_t      order;    // desc|asc order to iterate the data block
   int32_t      numOfCols;
   SColumnInfo* colList;
-  int32_t      type;  // data block load type:
+  int32_t*     pSlotList; // the column output destation slot, and it may be null
+  int32_t      type;     // data block load type:
   STimeWindow  twindows;
   int64_t      startVersion;
   int64_t      endVersion;
diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index a5d3a0829d..4208d85648 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -41,7 +41,7 @@ typedef struct SBlockOrderInfo {
     BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
   } while (0)
 
-#define colDataSetNotNull_f(bm_, r_)                  \
+#define colDataClearNull_f(bm_, r_)                  \
   do {                                                \
     BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \
   } while (0)
@@ -151,9 +151,6 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui
     for (int32_t i = start; i < start + nRows; ++i) {
       colDataSetNull_f(pColumnInfoData->nullbitmap, i);
     }
-
-    int32_t bytes = pColumnInfoData->info.bytes;
-    memset(pColumnInfoData->pData + start * bytes, 0, nRows * bytes);
   }
 
   pColumnInfoData->hasNull = true;
@@ -238,6 +235,7 @@ int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRo
 
 void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
 void blockDataCleanup(SSDataBlock* pDataBlock);
+void blockDataEmpty(SSDataBlock* pDataBlock);
 
 size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
 
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index f5e73d8f0e..c300c3e6f0 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1137,17 +1137,15 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
 }
 
 void blockDataCleanup(SSDataBlock* pDataBlock) {
+  blockDataEmpty(pDataBlock);
   SDataBlockInfo* pInfo = &pDataBlock->info;
-
-  int32_t existedRows = pInfo->rows;
-  ASSERT(existedRows <= pDataBlock->info.capacity);
-
-  pInfo->rows = 0;
   pInfo->id.uid = 0;
   pInfo->id.groupId = 0;
-  pInfo->window.ekey = 0;
-  pInfo->window.skey = 0;
+}
 
+void blockDataEmpty(SSDataBlock* pDataBlock) {
+  SDataBlockInfo* pInfo = &pDataBlock->info;
+  ASSERT(pInfo->rows <= pDataBlock->info.capacity);
   if (pInfo->capacity == 0) {
     return;
   }
@@ -1155,8 +1153,12 @@ void blockDataCleanup(SSDataBlock* pDataBlock) {
   size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
-    colInfoDataCleanup(p, existedRows);
+    colInfoDataCleanup(p, pInfo->capacity);
   }
+
+  pInfo->rows = 0;
+  pInfo->window.ekey = 0;
+  pInfo->window.skey = 0;
 }
 
 // todo temporarily disable it
@@ -1643,6 +1645,8 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s
 static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
     pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, n, total);
+
+    // clear the offset value of the unused entries.
     memset(&pColInfoData->varmeta.offset[total - n], 0, n);
   } else {
     int32_t bytes = pColInfoData->info.bytes;
@@ -1657,7 +1661,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
   }
 
   if (pBlock->info.rows <= n) {
-    blockDataCleanup(pBlock);
+    blockDataEmpty(pBlock);
   } else {
     size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
     for (int32_t i = 0; i < numOfCols; ++i) {
@@ -1674,12 +1678,22 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
     pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
     memset(&pColInfoData->varmeta.offset[n], 0, total - n);
+  } else {  // reset the bitmap value
+    int32_t stopIndex = BitmapLen(n) * 8;
+    for(int32_t i = n + 1; i < stopIndex; ++i) {
+      colDataClearNull_f(pColInfoData->nullbitmap, i);
+    }
+
+    int32_t remain = BitmapLen(total) - BitmapLen(n);
+    if (remain > 0) {
+      memset(pColInfoData->nullbitmap, 0, remain);
+    }
   }
 }
 
 int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) {
   if (n == 0) {
-    blockDataCleanup(pBlock);
+    blockDataEmpty(pBlock);
     return TSDB_CODE_SUCCESS;
   }
 
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 7ef3207b4d..53b8eadb33 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -159,11 +159,10 @@ typedef struct STsdbReader STsdbReader;
 
 int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num);
 int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
-                       STsdbReader **ppReader, const char *idstr);
+                       SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr);
 
 void     tsdbReaderClose(STsdbReader *pReader);
 bool     tsdbNextDataBlock(STsdbReader *pReader);
-bool     tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid);
 void     tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow);
 int32_t  tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
 SArray  *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 660d189451..27d99bd42d 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -85,7 +85,8 @@ typedef struct SBlockLoadSuppInfo {
   SArray*          pColAgg;
   SColumnDataAgg   tsColAgg;
   SColumnDataAgg** plist;
-  int16_t*         colIds;  // column ids for loading file block data
+  int16_t*         colIds;    // column ids for loading file block data
+  int16_t*         slotIds;   // the ordinal index in the destination SSDataBlock
   int32_t          numOfCols;
   char**           buildBuf;  // build string tmp buffer, todo remove it later after all string format being updated.
   bool             smaValid;  // the sma on all queried columns are activated
@@ -214,25 +215,25 @@ static bool          hasDataInFileBlock(const SBlockData* pBlockData, const SFil
 
 static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
 
-static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SSDataBlock* pBlock) {
-  size_t numOfCols = blockDataGetNumOfCols(pBlock);
-
+static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) {
   pSupInfo->smaValid = true;
   pSupInfo->numOfCols = numOfCols;
   pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t));
   pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES);
-  if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL) {
+  pSupInfo->slotIds = taosMemoryMalloc(numOfCols * sizeof(int16_t));
+  if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL || pSupInfo->slotIds == NULL) {
     taosMemoryFree(pSupInfo->colIds);
     taosMemoryFree(pSupInfo->buildBuf);
+    taosMemoryFree(pSupInfo->slotIds);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
   for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
-    pSupInfo->colIds[i] = pCol->info.colId;
+    pSupInfo->colIds[i] = pCols[i].colId;
+    pSupInfo->slotIds[i] = pSlotIdList[i];
 
-    if (IS_VAR_DATA_TYPE(pCol->info.type)) {
-      pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes);
+    if (IS_VAR_DATA_TYPE(pCols[i].type)) {
+      pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes);
     }
   }
 
@@ -566,7 +567,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity)
 }
 
 static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, int32_t capacity,
-                                const char* idstr) {
+                                SSDataBlock* pResBlock, const char* idstr) {
   int32_t      code = 0;
   int8_t       level = 0;
   STsdbReader* pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader));
@@ -585,6 +586,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
   pReader->suid = pCond->suid;
   pReader->order = pCond->order;
   pReader->capacity = capacity;
+  pReader->pResBlock = pResBlock;
   pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL;
   pReader->verRange = getQueryVerRange(pVnode, pCond, level);
   pReader->type = pCond->type;
@@ -592,6 +594,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
   pReader->blockInfoBuf.numPerBucket = 1000;  // 1000 tables per bucket
   ASSERT(pCond->numOfCols > 0);
 
+  // todo refactor.
   limitOutputBufferSize(pCond, &pReader->capacity);
 
   // allocate buffer in order to load data blocks from file
@@ -611,13 +614,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
     goto _end;
   }
 
-  pReader->pResBlock = createResBlock(pCond, pReader->capacity);
-  if (pReader->pResBlock == NULL) {
-    code = terrno;
-    goto _end;
-  }
-
-  setColumnIdSlotList(&pReader->suppInfo, pReader->pResBlock);
+  setColumnIdSlotList(&pReader->suppInfo, pCond->colList, pCond->pSlotList, pCond->numOfCols);
 
   *ppReader = pReader;
   return code;
@@ -1041,17 +1038,16 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo
 }
 
 static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) {
-  SReaderStatus*  pStatus = &pReader->status;
-  SDataBlockIter* pBlockIter = &pStatus->blockIter;
+  SReaderStatus*      pStatus = &pReader->status;
+  SDataBlockIter*     pBlockIter = &pStatus->blockIter;
+  SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
+  SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
 
   SBlockData*         pBlockData = &pStatus->fileBlockData;
   SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter);
   SDataBlk*           pBlock = getCurrentBlock(pBlockIter);
   SSDataBlock*        pResBlock = pReader->pResBlock;
-  int32_t             numOfOutputCols = blockDataGetNumOfCols(pResBlock);
-
-  SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
-  SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
+  int32_t             numOfOutputCols = pSupInfo->numOfCols;
 
   SColVal cv = {0};
   int64_t st = taosGetTimestampUs();
@@ -1087,8 +1083,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
   int32_t i = 0;
   int32_t rowIndex = 0;
 
-  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i);
-  if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+  if (pSupInfo->colIds[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
     copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);
     i += 1;
   }
@@ -1097,12 +1093,13 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
   int32_t num = pBlockData->nColData;
   while (i < numOfOutputCols && colIndex < num) {
     rowIndex = 0;
-    pColData = taosArrayGet(pResBlock->pDataBlock, i);
 
     SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);
-    if (pData->cid < pColData->info.colId) {
+    if (pData->cid < pSupInfo->colIds[i]) {
       colIndex += 1;
-    } else if (pData->cid == pColData->info.colId) {
+    } else if (pData->cid == pSupInfo->colIds[i]) {
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+
       if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {
         colDataAppendNNULL(pColData, 0, dumpedRows);
       } else {
@@ -1119,6 +1116,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
       colIndex += 1;
       i += 1;
     } else {  // the specified column does not exist in file block, fill with null data
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
       colDataAppendNNULL(pColData, 0, dumpedRows);
       i += 1;
     }
@@ -1126,7 +1124,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
 
   // fill the mis-matched columns with null value
   while (i < numOfOutputCols) {
-    pColData = taosArrayGet(pResBlock->pDataBlock, i);
+    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
     colDataAppendNNULL(pColData, 0, dumpedRows);
     i += 1;
   }
@@ -3535,7 +3533,7 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
 
 int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow,
                              STableBlockScanInfo* pScanInfo) {
-  int32_t numOfRows = pBlock->info.rows;
+  int32_t outputRowIndex = pBlock->info.rows;
   int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
   int64_t uid = pScanInfo->uid;
 
@@ -3545,23 +3543,26 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
   SColVal colVal = {0};
   int32_t i = 0, j = 0;
 
-  SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-  if (pColInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-    colDataAppend(pColInfoData, numOfRows, (const char*)&pTSRow->ts, false);
+  if (pSupInfo->colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+    ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts;
     i += 1;
   }
 
   while (i < numOfCols && j < pSchema->numOfCols) {
-    pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-    col_id_t colId = pColInfoData->info.colId;
+    col_id_t colId = pSupInfo->colIds[i];
 
     if (colId == pSchema->columns[j].colId) {
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+
       tTSRowGetVal(pTSRow, pSchema, j, &colVal);
-      doCopyColVal(pColInfoData, numOfRows, i, &colVal, pSupInfo);
+      doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo);
       i += 1;
       j += 1;
     } else if (colId < pSchema->columns[j].colId) {
-      colDataAppendNULL(pColInfoData, numOfRows);
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+
+      colDataAppendNULL(pColInfoData, outputRowIndex);
       i += 1;
     } else if (colId > pSchema->columns[j].colId) {
       j += 1;
@@ -3570,8 +3571,8 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
 
   // set null value since current column does not exist in the "pSchema"
   while (i < numOfCols) {
-    pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
-    colDataAppendNULL(pColInfoData, numOfRows);
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+    colDataAppendNULL(pColInfoData, outputRowIndex);
     i += 1;
   }
 
@@ -3586,27 +3587,25 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   int32_t outputRowIndex = pResBlock->info.rows;
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
-
-  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i);
-  if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+  if (pReader->suppInfo.colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
     ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex];
     i += 1;
   }
 
   SColVal cv = {0};
   int32_t numOfInputCols = pBlockData->nColData;
-  int32_t numOfOutputCols = pResBlock->pDataBlock->size;
+  int32_t numOfOutputCols = pSupInfo->numOfCols;
 
   while (i < numOfOutputCols && j < numOfInputCols) {
-    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, i);
-    SColData*        pData = tBlockDataGetColDataByIdx(pBlockData, j);
-
-    if (pData->cid < pCol->info.colId) {
+    SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j);
+    if (pData->cid < pSupInfo->colIds[i]) {
       j += 1;
       continue;
     }
 
-    if (pData->cid == pCol->info.colId) {
+    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+    if (pData->cid == pSupInfo->colIds[i]) {
       tColDataGetValue(pData, rowIndex, &cv);
       doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
       j += 1;
@@ -3619,7 +3618,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   }
 
   while (i < numOfOutputCols) {
-    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i);
+    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
     colDataAppendNULL(pCol, outputRowIndex);
     i += 1;
   }
@@ -3718,14 +3717,14 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
 
 // ====================================== EXPOSED APIs ======================================
 int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
-                       STsdbReader** ppReader, const char* idstr) {
+                       SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) {
   STimeWindow window = pCond->twindows;
   if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) {
     pCond->twindows.skey += 1;
     pCond->twindows.ekey -= 1;
   }
 
-  int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, 4096, idstr);
+  int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, pResBlock->info.capacity, pResBlock, idstr);
   if (code != TSDB_CODE_SUCCESS) {
     goto _err;
   }
@@ -3751,7 +3750,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
     }
 
     // here we only need one more row, so the capacity is set to be ONE.
-    code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, idstr);
+    code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, pResBlock, idstr);
     if (code != TSDB_CODE_SUCCESS) {
       goto _err;
     }
@@ -3765,7 +3764,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
     }
     pCond->order = order;
 
-    code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, idstr);
+    code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, pResBlock, idstr);
     if (code != TSDB_CODE_SUCCESS) {
       goto _err;
     }
@@ -3837,10 +3836,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
   tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr);
   return code;
 
-_err:
+  _err:
   tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr);
   return code;
-}
+  }
 
 void tsdbReaderClose(STsdbReader* pReader) {
   if (pReader == NULL) {
@@ -3874,7 +3873,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
   taosMemoryFree(pSupInfo->colIds);
 
   taosArrayDestroy(pSupInfo->pColAgg);
-  for (int32_t i = 0; i < blockDataGetNumOfCols(pReader->pResBlock); ++i) {
+  for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) {
     if (pSupInfo->buildBuf[i] != NULL) {
       taosMemoryFreeClear(pSupInfo->buildBuf[i]);
     }
@@ -3891,8 +3890,6 @@ void tsdbReaderClose(STsdbReader* pReader) {
     clearBlockScanInfoBuf(&pReader->blockInfoBuf);
   }
 
-  blockDataDestroy(pReader->pResBlock);
-
   if (pReader->pFileReader != NULL) {
     tsdbDataFReaderClose(&pReader->pFileReader);
   }
@@ -4007,16 +4004,6 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
   return false;
 }
 
-bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) {
-  STableBlockScanInfo* pBlockScanInfo =
-      *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid));
-  if (pBlockScanInfo == NULL) {  // no data block for the table of given uid
-    return false;
-  }
-
-  return true;
-}
-
 static void setBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow) {
   ASSERT(pReader != NULL);
   *rows = pReader->pResBlock->info.rows;
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 7c37d66eb6..3f62191ca1 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1241,6 +1241,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
     }
   }
 
+  // set the output flag for each column in SColMatchInfo, according to the
   *numOfOutputCols = 0;
   int32_t num = LIST_LENGTH(pOutputNodeList->pSlots);
   for (int32_t i = 0; i < num; ++i) {
@@ -1599,20 +1600,22 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) {
 int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) {
   pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
   pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols);
+
   pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo));
-  if (pCond->colList == NULL) {
+  pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)*pCond->numOfCols);
+  if (pCond->colList == NULL || pCond->pSlotList == NULL) {
     terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    taosMemoryFreeClear(pCond->colList);
+    taosMemoryFreeClear(pCond->pSlotList);
     return terrno;
   }
 
-  // pCond->twindow = pTableScanNode->scanRange;
   // TODO: get it from stable scan node
   pCond->twindows = pTableScanNode->scanRange;
   pCond->suid = pTableScanNode->scan.suid;
   pCond->type = TIMEWINDOW_RANGE_CONTAINED;
   pCond->startVersion = -1;
   pCond->endVersion = -1;
-  //  pCond->type = pTableScanNode->scanFlag;
 
   int32_t j = 0;
   for (int32_t i = 0; i < pCond->numOfCols; ++i) {
@@ -1625,6 +1628,8 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
     pCond->colList[j].type = pColNode->node.resType.type;
     pCond->colList[j].bytes = pColNode->node.resType.bytes;
     pCond->colList[j].colId = pColNode->colId;
+
+    pCond->pSlotList[j] = pNode->slotId;
     j += 1;
   }
 
@@ -1962,7 +1967,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
 
 int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle,
                                 STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond,
-                                struct SExecTaskInfo* pTaskInfo) {
+                                SExecTaskInfo* pTaskInfo) {
   int64_t     st = taosGetTimestampUs();
   const char* idStr = GET_TASKID(pTaskInfo);
 
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 34bd9cf8ca..d675ac05d5 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1064,7 +1064,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
         int32_t        num = tableListGetSize(pTaskInfo->pTableInfoList);
 
         if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num,
-                           &pTableScanInfo->base.dataReader, NULL) < 0 ||
+                           pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 ||
             pTableScanInfo->base.dataReader == NULL) {
           ASSERT(0);
         }
@@ -1116,7 +1116,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
     int32_t        size = tableListGetSize(pTaskInfo->pTableInfoList);
     ASSERT(size == 1);
 
-    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->dataReader, NULL);
+    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->pRes, &pInfo->dataReader, NULL);
 
     cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
     strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 3e28798a3a..aa2682c490 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -1190,8 +1190,6 @@ void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) {
 //       T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
 //     }
 //
-//     tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo);
-//
 //     if (pQueryAttr->limit.offset > blockInfo.rows) {
 //       pQueryAttr->limit.offset -= blockInfo.rows;
 //       pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index ca3734d91e..1729c9f1e8 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -285,7 +285,7 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
   if (pLimit->offset > 0 && pLimitInfo->remainOffset > 0) {
     if (pLimitInfo->remainOffset >= pBlock->info.rows) {
       pLimitInfo->remainOffset -= pBlock->info.rows;
-      pBlock->info.rows = 0;
+      blockDataEmpty(pBlock);
       qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset, id);
     } else {
       blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset);
@@ -390,7 +390,6 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
     return terrno;
   }
 
-  relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
   doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
 
   // restore the previous value
@@ -638,16 +637,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
       continue;
     }
 
-    blockDataCleanup(pBlock);
-    SDataBlockInfo* pBInfo = &pBlock->info;
-
-    int32_t rows = 0;
-    tsdbRetrieveDataBlockInfo(pTableScanInfo->base.dataReader, &rows, &pBInfo->id.uid, &pBInfo->window);
-
-    blockDataEnsureCapacity(pBlock, rows);  // todo remove it latter
-    pBInfo->rows = rows;
-
-    ASSERT(pBInfo->id.uid != 0);
+    ASSERT(pBlock->info.id.uid != 0);
     pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
 
     uint32_t status = 0;
@@ -778,7 +768,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
       ASSERT(pInfo->base.dataReader == NULL);
 
       int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num,
-                                    (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo));
+                                    pInfo->pResBlock, (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo));
       if (code != TSDB_CODE_SUCCESS) {
         T_LONG_JMP(pTaskInfo->env, code);
       }
@@ -879,14 +869,14 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
   pInfo->base.scanFlag = MAIN_SCAN;
   pInfo->base.pdInfo.interval = extractIntervalInfo(pTableScanNode);
   pInfo->base.readHandle = *readHandle;
+  pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired;
+
   pInfo->sample.sampleRatio = pTableScanNode->ratio;
   pInfo->sample.seed = taosGetTimestampSec();
 
-  pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired;
-
   initResultSizeInfo(&pOperator->resultInfo, 4096);
   pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
-  blockDataEnsureCapacityNoClear(pInfo->pResBlock, pOperator->resultInfo.capacity);
+  blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
 
   code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
   if (code != TSDB_CODE_SUCCESS) {
@@ -993,10 +983,8 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
   SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo;
 
   SSDataBlock* pBlock = pTableScanInfo->pResBlock;
-  blockDataCleanup(pBlock);
-
   STsdbReader* pReader = NULL;
-  int32_t      code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader,
+  int32_t      code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, (STsdbReader**)&pReader,
                                      GET_TASKID(pTaskInfo));
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
@@ -2301,7 +2289,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
     if (pHandle->initTableReader) {
       pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
       pTSInfo->base.dataReader = NULL;
-      code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, &pTSInfo->base.dataReader, NULL);
+      code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, pTSInfo->pResBlock, &pTSInfo->base.dataReader, NULL);
       if (code != 0) {
         terrno = code;
         destroyTableScanOperatorInfo(pTableScanOp);
@@ -2531,11 +2519,10 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
   blockDataCleanup(pBlock);
 
   int64_t st = taosGetTimestampUs();
-
-  void*        p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex);
+  void*   p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex);
   SReadHandle* pHandle = &pInfo->base.readHandle;
 
-  int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->base.dataReader, GET_TASKID(pTaskInfo));
+  int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &pInfo->base.dataReader, GET_TASKID(pTaskInfo));
   if (code != 0) {
     T_LONG_JMP(pTaskInfo->env, code);
   }
@@ -2553,12 +2540,6 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
     }
 
     blockDataCleanup(pBlock);
-
-    int32_t rows = 0;
-    tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.id.uid, &pBlock->info.window);
-    blockDataEnsureCapacity(pBlock, rows);
-    pBlock->info.rows = rows;
-
     if (pQueryCond->order == TSDB_ORDER_ASC) {
       pQueryCond->twindows.skey = pBlock->info.window.ekey + 1;
     } else {
diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c
index 3d35326749..7ac6cee2af 100644
--- a/source/libs/executor/src/sysscanoperator.c
+++ b/source/libs/executor/src/sysscanoperator.c
@@ -1899,53 +1899,52 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC
 
 SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode,
                                                SExecTaskInfo* pTaskInfo) {
-    SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo));
-    SOperatorInfo*  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
-    if (pInfo == NULL || pOperator == NULL) {
-        pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _error;
-    }
+  SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo));
+  SOperatorInfo*  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  if (pInfo == NULL || pOperator == NULL) {
+    pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _error;
+  }
 
-    {
-        SQueryTableDataCond cond = {0};
+  pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc);
+  blockDataEnsureCapacity(pInfo->pResBlock, 1);
 
-        int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond);
-        if (code != TSDB_CODE_SUCCESS) {
-            goto _error;
-        }
-
-        STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
-        size_t          num = tableListGetSize(pTableListInfo);
-        void*           pList = tableListGetInfo(pTableListInfo, 0);
-
-        code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str);
-        cleanupQueryTableDataCond(&cond);
-        if (code != 0) {
-            goto _error;
-        }
-    }
-
-    pInfo->readHandle = *readHandle;
-    pInfo->uid = pBlockScanNode->suid;
-
-    pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc);
-    blockDataEnsureCapacity(pInfo->pResBlock, 1);
-
-    int32_t    numOfCols = 0;
-    SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols);
-    int32_t    code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
+  {
+    SQueryTableDataCond cond = {0};
+    int32_t             code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond);
     if (code != TSDB_CODE_SUCCESS) {
-        goto _error;
+      goto _error;
     }
 
-    setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false,
-                    OP_NOT_OPENED, pInfo, pTaskInfo);
-    pOperator->fpSet =
-            createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL);
-    return pOperator;
+    STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
+    size_t          num = tableListGetSize(pTableListInfo);
+    void*           pList = tableListGetInfo(pTableListInfo, 0);
 
-    _error:
-    taosMemoryFreeClear(pInfo);
-    taosMemoryFreeClear(pOperator);
-    return NULL;
+    code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str);
+    cleanupQueryTableDataCond(&cond);
+    if (code != 0) {
+      goto _error;
+    }
+  }
+
+  pInfo->readHandle = *readHandle;
+  pInfo->uid = pBlockScanNode->suid;
+
+  int32_t    numOfCols = 0;
+  SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols);
+  int32_t    code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false,
+                  OP_NOT_OPENED, pInfo, pTaskInfo);
+  pOperator->fpSet =
+      createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL);
+  return pOperator;
+
+_error:
+  taosMemoryFreeClear(pInfo);
+  taosMemoryFreeClear(pOperator);
+  return NULL;
 }
\ No newline at end of file
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 551d668763..5f077c98d7 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1739,8 +1739,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
   pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId;
 
   size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
-  initResultSizeInfo(&pOperator->resultInfo, 1024);
-  blockDataEnsureCapacityNoClear(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+  initResultSizeInfo(&pOperator->resultInfo, 512);
+  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
 
   int32_t    num = 0;
   SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
@@ -4319,7 +4319,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
   iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock;
 
   size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
-  initResultSizeInfo(&pOperator->resultInfo, 4096);
+  initResultSizeInfo(&pOperator->resultInfo, 512);
 
   int32_t    num = 0;
   SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c
index 98391d95f4..6074c72424 100644
--- a/source/libs/scalar/src/sclvector.c
+++ b/source/libs/scalar/src/sclvector.c
@@ -1782,7 +1782,7 @@ void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut,
     if (colDataIsNull_s(pOut->columnData, i)) {
       int8_t v = 0;
       colDataAppendInt8(pOut->columnData, i, &v);
-      colDataSetNotNull_f(pOut->columnData->nullbitmap, i);
+      colDataClearNull_f(pOut->columnData->nullbitmap, i);
     }
   }
   pOut->columnData->hasNull = false;
diff --git a/source/util/src/thash.c b/source/util/src/thash.c
index b07d485c87..e9548613aa 100644
--- a/source/util/src/thash.c
+++ b/source/util/src/thash.c
@@ -279,7 +279,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
     return NULL;
   }
 
-  void *p = taosMemoryCalloc(pHashObj->capacity, sizeof(SHashEntry));
+  void *p = taosMemoryMalloc(pHashObj->capacity * sizeof(SHashEntry));
   if (p == NULL) {
     taosArrayDestroy(pHashObj->pMemBlock);
     taosMemoryFree(pHashObj->hashList);
@@ -290,6 +290,9 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp
 
   for (int32_t i = 0; i < pHashObj->capacity; ++i) {
     pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry));
+    pHashObj->hashList[i]->num = 0;
+    pHashObj->hashList[i]->latch = 0;
+    pHashObj->hashList[i]->next = NULL;
   }
 
   taosArrayPush(pHashObj->pMemBlock, &p);

From 9f55ca7f80fa3c8ff83fbc471c74353264fa82e3 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 17:45:33 +0800
Subject: [PATCH 15/23] test:update the test cases.

---
 tests/script/tsim/parser/groupby-basic.sim | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tests/script/tsim/parser/groupby-basic.sim b/tests/script/tsim/parser/groupby-basic.sim
index 4c70399e4b..2e436945e0 100644
--- a/tests/script/tsim/parser/groupby-basic.sim
+++ b/tests/script/tsim/parser/groupby-basic.sim
@@ -78,8 +78,8 @@ $ts2 = $tb2 . .ts
 
 print ===============================groupby_operation
 print
-print ==== select count(*), c1 from group_tb0 group by c1
-sql select count(*), c1 from group_tb0 group by c1
+print ==== select count(*), c1 from group_tb0 group by c1 order by c1
+sql select count(*), c1 from group_tb0 group by c1 order by c1
 print rows: $rows
 print $data00 $data01 $data02 $data03
 print $data10 $data11 $data12 $data13
@@ -98,18 +98,18 @@ endi
 if $data90 != 10 then
   return -1
 endi
-if $data01 != 7 then
+if $data01 != 0 then
   return -1
 endi
-if $data11 != 6 then
+if $data11 != 1 then
   return -1
 endi
-if $data91 != 3 then
+if $data91 != 9 then
   return -1
 endi
 
-print ==== select first(ts),c1 from group_tb0 group by c1;
-sql select first(ts),c1 from group_tb0 group by c1;
+print ==== select first(ts),c1 from group_tb0 group by c1 order by c1;
+sql select first(ts),c1 from group_tb0 group by c1 order by c1;
 print rows: $rows
 print $data00 $data01 $data02 $data03
 print $data10 $data11 $data12 $data13
@@ -120,16 +120,16 @@ if $row != 10 then
   return -1
 endi
 
-if $data00 != @22-01-01 00:00:00.007@ then
+if $data00 != @22-01-01 00:00:00.000@ then
   return -1
 endi
-if $data01 != 7 then
+if $data01 != 0 then
   return -1
 endi
-if $data90 != @22-01-01 00:00:00.003@ then
+if $data90 != @22-01-01 00:00:00.009@ then
   return -1
 endi
-if $data91 != 3 then
+if $data91 != 9 then
   return -1
 endi
 

From 8c62d5f9fa027c28382ec84641fdd37101cc38d2 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 18:34:26 +0800
Subject: [PATCH 16/23] fix(query): fix bug in tsdbread.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c        |  9 +++--
 source/libs/executor/src/scanoperator.c       | 13 ++++----
 source/libs/executor/src/timewindowoperator.c | 33 +++++++++++--------
 3 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 27d99bd42d..7c39e378cb 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -1621,7 +1621,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
   int64_t st = taosGetTimestampUs();
   int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader);
 
-  blockDataUpdateTsWindow(pBlock, 0);
+  blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotIds[0]);
   pBlock->info.id.uid = pBlockScanInfo->uid;
 
   setComposedBlockFlag(pReader, true);
@@ -2493,7 +2493,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
 
 _end:
   pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
-  blockDataUpdateTsWindow(pResBlock, 0);
+  blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotIds[0]);
 
   setComposedBlockFlag(pReader, true);
   double el = (taosGetTimestampUs() - st) / 1000.0;
@@ -3534,7 +3534,6 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
 int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow,
                              STableBlockScanInfo* pScanInfo) {
   int32_t outputRowIndex = pBlock->info.rows;
-  int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
   int64_t uid = pScanInfo->uid;
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
@@ -3549,7 +3548,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
     i += 1;
   }
 
-  while (i < numOfCols && j < pSchema->numOfCols) {
+  while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) {
     col_id_t colId = pSupInfo->colIds[i];
 
     if (colId == pSchema->columns[j].colId) {
@@ -3570,7 +3569,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
   }
 
   // set null value since current column does not exist in the "pSchema"
-  while (i < numOfCols) {
+  while (i < pSupInfo->numOfCols) {
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
     colDataAppendNULL(pColInfoData, outputRowIndex);
     i += 1;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 1729c9f1e8..9fc3575491 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -2513,10 +2513,8 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
   SExecTaskInfo*                  pTaskInfo = pOperator->pTaskInfo;
   int32_t                         readIdx = source->readerIdx;
   SSDataBlock*                    pBlock = source->inputBlock;
-  STableMergeScanInfo*            pTableScanInfo = pOperator->info;
 
-  SQueryTableDataCond* pQueryCond = taosArrayGet(pTableScanInfo->queryConds, readIdx);
-  blockDataCleanup(pBlock);
+  SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx);
 
   int64_t st = taosGetTimestampUs();
   void*   p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex);
@@ -2534,12 +2532,11 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
     }
 
     // process this data block based on the probabilities
-    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
+    bool processThisBlock = processBlockWithProbability(&pInfo->sample);
     if (!processThisBlock) {
       continue;
     }
 
-    blockDataCleanup(pBlock);
     if (pQueryCond->order == TSDB_ORDER_ASC) {
       pQueryCond->twindows.skey = pBlock->info.window.ekey + 1;
     } else {
@@ -2547,7 +2544,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
     }
 
     uint32_t status = 0;
-    loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status);
+    loadDataBlock(pOperator, &pInfo->base, pBlock, &status);
     //    code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status);
     if (code != TSDB_CODE_SUCCESS) {
       T_LONG_JMP(pTaskInfo->env, code);
@@ -2561,7 +2558,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
     pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
 
     pOperator->resultInfo.totalRows += pBlock->info.rows;
-    pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
+    pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
 
     tsdbReaderClose(pInfo->base.dataReader);
     pInfo->base.dataReader = NULL;
@@ -2641,6 +2638,8 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
     param.readerIdx = i;
     param.pOperator = pOperator;
     param.inputBlock = createOneDataBlock(pInfo->pResBlock, false);
+    blockDataEnsureCapacity(param.inputBlock, pOperator->resultInfo.capacity);
+
     taosArrayPush(pInfo->sortSourceParams, &param);
 
     SQueryTableDataCond cond;
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 5f077c98d7..52f06ffab4 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -1659,22 +1659,11 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
   // the primary timestamp column
   bool needed = false;
 
-  for (int32_t i = 0; i < numOfCols; ++i) {
+  for(int32_t i = 0; i < numOfCols; ++i) {
     SExprInfo* pExpr = pCtx[i].pExpr;
-
     if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
-      SFunctParam* pParam = &pExpr->base.pParam[0];
-
-      SColumn c = *pParam->pCol;
-      taosArrayPush(pInfo->pInterpCols, &c);
       needed = true;
-
-      SGroupKeys key = {0};
-      key.bytes = c.bytes;
-      key.type = c.type;
-      key.isNull = false;
-      key.pData = taosMemoryCalloc(1, c.bytes);
-      taosArrayPush(pInfo->pPrevValues, &key);
+      break;
     }
   }
 
@@ -1699,6 +1688,24 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
     }
   }
 
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SExprInfo* pExpr = pCtx[i].pExpr;
+
+    if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
+      SFunctParam* pParam = &pExpr->base.pParam[0];
+
+      SColumn c = *pParam->pCol;
+      taosArrayPush(pInfo->pInterpCols, &c);
+
+      SGroupKeys key = {0};
+      key.bytes = c.bytes;
+      key.type = c.type;
+      key.isNull = false;
+      key.pData = taosMemoryCalloc(1, c.bytes);
+      taosArrayPush(pInfo->pPrevValues, &key);
+    }
+  }
+
   return needed;
 }
 

From 740daefdc54a49a0451566a28db77b6cee9098f8 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 18:39:15 +0800
Subject: [PATCH 17/23] fix(query): fix a typo.

---
 source/libs/executor/src/scanoperator.c | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 9fc3575491..87aacfdd34 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -996,15 +996,13 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
   if (hasBlock) {
     SDataBlockInfo* pBInfo = &pBlock->info;
 
-    int32_t rows = 0;
-    tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window);
+//    int32_t rows = 0;
+//    tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window);
 
     SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL);
-    blockDataEnsureCapacity(pBlock, rows);
-    pBlock->info.rows = rows;
 
-    relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true);
-    doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, rows);
+//    relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true);
+    doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
 
     pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->id.uid);
   }

From efcc3b06bb3af6791d670922c290ec5de297d3e9 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 20:01:09 +0800
Subject: [PATCH 18/23] fix(query): fix memory leak.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 1 +
 source/libs/executor/src/executil.c    | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 7c39e378cb..f86540e3e2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -3870,6 +3870,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
 
   taosMemoryFreeClear(pSupInfo->plist);
   taosMemoryFree(pSupInfo->colIds);
+  taosMemoryFree(pSupInfo->slotIds);
 
   taosArrayDestroy(pSupInfo->pColAgg);
   for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) {
diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c
index 3f62191ca1..6c219f8bd4 100644
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@@ -1637,7 +1637,10 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
   return TSDB_CODE_SUCCESS;
 }
 
-void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFreeClear(pCond->colList); }
+void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) {
+  taosMemoryFreeClear(pCond->colList);
+  taosMemoryFreeClear(pCond->pSlotList);
+}
 
 int32_t convertFillType(int32_t mode) {
   int32_t type = TSDB_FILL_NONE;

From 24614d03bd2ef21b454d8b9c558fb29977418960 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 21:35:01 +0800
Subject: [PATCH 19/23] fix(query): set correct sma retrieve procedure.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c  | 42 ++++++++++++++-----------
 source/libs/executor/src/scanoperator.c |  2 ++
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index f86540e3e2..24e9181ae6 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -84,7 +84,7 @@ typedef struct SIOCostSummary {
 typedef struct SBlockLoadSuppInfo {
   SArray*          pColAgg;
   SColumnDataAgg   tsColAgg;
-  SColumnDataAgg** plist;
+  SColumnDataAgg** plist;     // todo remove this
   int16_t*         colIds;    // column ids for loading file block data
   int16_t*         slotIds;   // the ordinal index in the destination SSDataBlock
   int32_t          numOfCols;
@@ -4041,12 +4041,11 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
   }
 
   SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter);
-
-  SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
-  //  int64_t   stime = taosGetTimestampUs();
-
   SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
 
+  ASSERT(pReader->pResBlock->info.id.uid == pFBlock->uid);
+
+  SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
   if (tDataBlkHasSma(pBlock)) {
     code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg);
     if (code != TSDB_CODE_SUCCESS) {
@@ -4071,7 +4070,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
   pSup->plist[0] = pTsAgg;
 
   // update the number of NULL data rows
-  size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock);
+  size_t numOfCols = pSup->numOfCols;
 
   int32_t i = 0, j = 0;
   size_t  size = taosArrayGetSize(pSup->pColAgg);
@@ -4095,43 +4094,48 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
   }
 #else
 
+  SSDataBlock* pResBlock = pReader->pResBlock;
+  if (pResBlock->pBlockAgg == NULL) {
+    size_t num = taosArrayGetSize(pResBlock->pDataBlock);
+    pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg));
+  }
+
   // fill the all null data column
-  SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg));
+//  SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg));
 
   while (j < numOfCols && i < size) {
     SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
     if (pAgg->colId == pSup->colIds[j]) {
-      taosArrayPush(pNewAggList, pAgg);
+      pResBlock->pBlockAgg[pSup->slotIds[j]] = pAgg;
       i += 1;
       j += 1;
     } else if (pAgg->colId < pSup->colIds[j]) {
       i += 1;
     } else if (pSup->colIds[j] < pAgg->colId) {
       if (pSup->colIds[j] == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        taosArrayPush(pNewAggList, &pSup->tsColAgg);
+        pResBlock->pBlockAgg[pSup->slotIds[j]] = &pSup->tsColAgg;
       } else {
       // all date in this block are null
         SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow};
-        taosArrayPush(pNewAggList, &nullColAgg);
+        taosArrayPush(pSup->pColAgg, &nullColAgg);
+
+        pResBlock->pBlockAgg[pSup->slotIds[j]] = taosArrayGetLast(pSup->pColAgg);
       }
       j += 1;
     }
   }
 
-  taosArrayClear(pSup->pColAgg);
-  taosArrayAddAll(pSup->pColAgg, pNewAggList);
+//  taosArrayClear(pSup->pColAgg);
 
-  size_t num = taosArrayGetSize(pSup->pColAgg);
-  for(int32_t k = 0; k < num; ++k) {
-    pSup->plist[k] = taosArrayGet(pSup->pColAgg, k);
-  }
-
-  taosArrayDestroy(pNewAggList);
+//  size_t num = taosArrayGetSize(pSup->pColAgg);
+//  for(int32_t k = 0; k < num; ++k) {
+//    pSup->plist[k] = taosArrayGet(pSup->pColAgg, k);
+//  }
 
 #endif
 
   pReader->cost.smaDataLoad += 1;
-  *pBlockStatis = pSup->plist;
+//  *pBlockStatis = pSup->plist;
 
   tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr);
   return code;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 87aacfdd34..b0104c6025 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -236,6 +236,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
     return false;
   }
 
+#if 0
   //  if (allColumnsHaveAgg == true) {
   int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
 
@@ -256,6 +257,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
 
     pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
   }
+#endif
 
   return true;
 }

From 1faf616f1d089ce592c8dcd56326d7140c75aea0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 1 Dec 2022 22:17:53 +0800
Subject: [PATCH 20/23] fix(query): fix null ptr access in evaluation block
 distribution.

---
 source/libs/executor/src/sysscanoperator.c | 37 ++++++++++++----------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c
index 7ac6cee2af..96f21babe6 100644
--- a/source/libs/executor/src/sysscanoperator.c
+++ b/source/libs/executor/src/sysscanoperator.c
@@ -1874,27 +1874,30 @@ static void destroyBlockDistScanOperatorInfo(void* param) {
 }
 
 static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) {
-    memset(pCond, 0, sizeof(SQueryTableDataCond));
+  memset(pCond, 0, sizeof(SQueryTableDataCond));
 
-    pCond->order = TSDB_ORDER_ASC;
-    pCond->numOfCols = 1;
-    pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo));
-    if (pCond->colList == NULL) {
-        terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
-        return terrno;
-    }
+  pCond->order = TSDB_ORDER_ASC;
+  pCond->numOfCols = 1;
+  pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo));
+  pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t));
+  if (pCond->colList == NULL || pCond->pSlotList == NULL) {
+    terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    return terrno;
+  }
 
-    pCond->colList->colId = 1;
-    pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP;
-    pCond->colList->bytes = sizeof(TSKEY);
+  pCond->colList->colId = 1;
+  pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP;
+  pCond->colList->bytes = sizeof(TSKEY);
 
-    pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
-    pCond->suid = uid;
-    pCond->type = TIMEWINDOW_RANGE_CONTAINED;
-    pCond->startVersion = -1;
-    pCond->endVersion = -1;
+  pCond->pSlotList[0] = 0;
 
-    return TSDB_CODE_SUCCESS;
+  pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
+  pCond->suid = uid;
+  pCond->type = TIMEWINDOW_RANGE_CONTAINED;
+  pCond->startVersion = -1;
+  pCond->endVersion = -1;
+
+  return TSDB_CODE_SUCCESS;
 }
 
 SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode,

From 83782fb27bf679b44353511e0eb70393e4193902 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 2 Dec 2022 10:07:48 +0800
Subject: [PATCH 21/23] fix(query): fix invalid tag fill and clear nullbitmp

---
 include/common/tdatablock.h                |  6 ++---
 source/common/src/tdatablock.c             |  8 +++----
 source/libs/executor/src/tfill.c           | 28 +++++++++++++++++-----
 tests/script/tsim/parser/lastrow_query.sim |  4 ++--
 tests/script/tsim/valgrind/checkUdf.sim    |  3 ++-
 5 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h
index 4208d85648..e1d3b01611 100644
--- a/include/common/tdatablock.h
+++ b/include/common/tdatablock.h
@@ -41,9 +41,9 @@ typedef struct SBlockOrderInfo {
     BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
   } while (0)
 
-#define colDataClearNull_f(bm_, r_)                  \
-  do {                                                \
-    BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \
+#define colDataClearNull_f(bm_, r_)                             \
+  do {                                                          \
+    BMCharPos(bm_, r_) &= ((char)(~(1u << (7u - BitPos(r_))))); \
   } while (0)
 
 #define colDataIsNull_var(pColumnInfoData, row)  (pColumnInfoData->varmeta.offset[row] == -1)
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index c300c3e6f0..e9be89a68b 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1679,15 +1679,15 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_
     pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
     memset(&pColInfoData->varmeta.offset[n], 0, total - n);
   } else {  // reset the bitmap value
-    int32_t stopIndex = BitmapLen(n) * 8;
-    for(int32_t i = n + 1; i < stopIndex; ++i) {
+    /*int32_t stopIndex = BitmapLen(n) * 8;
+    for(int32_t i = n; i < stopIndex; ++i) {
       colDataClearNull_f(pColInfoData->nullbitmap, i);
     }
 
     int32_t remain = BitmapLen(total) - BitmapLen(n);
     if (remain > 0) {
-      memset(pColInfoData->nullbitmap, 0, remain);
-    }
+      memset(pColInfoData->nullbitmap+BitmapLen(n), 0, remain);
+    }*/
   }
 }
 
diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c
index 42b1d058b5..4c8967744e 100644
--- a/source/libs/executor/src/tfill.c
+++ b/source/libs/executor/src/tfill.c
@@ -60,8 +60,18 @@ static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowInd
     if (pCol->notFillCol) {
       bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstColInfo, rowIndex);
       if (!filled) {
-        SArray*     p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal;
-        SGroupKeys* pKey = taosArrayGet(p, i);
+        SRowVal* p = NULL;
+        if (FILL_IS_ASC_FILL(pFillInfo)) {
+          if (pFillInfo->prev.key != 0) {
+            p = &pFillInfo->prev;  // prev has been set value
+          } else { // otherwise, use the value in the next row
+            p = &pFillInfo->next;
+          }
+        } else {
+          p = &pFillInfo->next;
+        }
+
+        SGroupKeys* pKey = taosArrayGet(p->pRowVal, i);
         doSetVal(pDstColInfo, rowIndex, pKey);
       }
     } else {
@@ -261,7 +271,10 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) {
 
 static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull);
 
-static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray* pRow) {
+static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal) {
+  SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, pFillInfo->srcTsSlotId);
+  pRowVal->key = ((int64_t*)pTsCol->pData)[rowIndex];
+
   for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
     int32_t type = pFillInfo->pFillCol[i].pExpr->pExpr->nodeType;
     if (type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) {
@@ -272,7 +285,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray
       bool  isNull = colDataIsNull_s(pSrcCol, rowIndex);
       char* p = colDataGetData(pSrcCol, rowIndex);
 
-      saveColData(pRow, i, p, isNull);
+      saveColData(pRowVal->pRowVal, i, p, isNull);
     } else {
       ASSERT(0);
     }
@@ -296,7 +309,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t
 
     // set the next value for interpolation
     if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) {
-      copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next.pRowVal);
+      copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, &pFillInfo->next);
     }
 
     if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) &&
@@ -318,7 +331,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t
 
       if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) {
         int32_t nextRowIndex = pFillInfo->index + 1;
-        copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, pFillInfo->next.pRowVal);
+        copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next);
       }
 
       // copy rows to dst buffer
@@ -334,6 +347,9 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t
         if (!colDataIsNull_s(pSrc, pFillInfo->index)) {
           colDataAppend(pDst, index, src, false);
           saveColData(pFillInfo->prev.pRowVal, i, src, false);
+          if (pFillInfo->srcTsSlotId == dstSlotId) {
+            pFillInfo->prev.key = *(int64_t*)src;
+          }
         } else {  // the value is null
           if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
             colDataAppend(pDst, index, (const char*)&pFillInfo->currentKey, false);
diff --git a/tests/script/tsim/parser/lastrow_query.sim b/tests/script/tsim/parser/lastrow_query.sim
index 5f557fd7bd..9ca9d407b8 100644
--- a/tests/script/tsim/parser/lastrow_query.sim
+++ b/tests/script/tsim/parser/lastrow_query.sim
@@ -70,10 +70,10 @@ sql select _wstart, t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00
 if $row != 2 then
   return -1
 endi
-if $data01 != NULL then
+if $data01 != 8 then
   return -1
 endi
-if $data02 != NULL then
+if $data02 != 8 then
   return -1
 endi
 if  $data03 != NULL then
diff --git a/tests/script/tsim/valgrind/checkUdf.sim b/tests/script/tsim/valgrind/checkUdf.sim
index 594d649714..caf316bd86 100644
--- a/tests/script/tsim/valgrind/checkUdf.sim
+++ b/tests/script/tsim/valgrind/checkUdf.sim
@@ -121,12 +121,13 @@ if $data01 != 152.420471066 then
   return -1
 endi
 
-sql select udf2(f2) from udf.t2 group by 1-udf1(f1);
+sql select udf2(f2) from udf.t2 group by 1-udf1(f1) order by 1-udf1(f1)
 print $rows , $data00 , $data10
 if $rows != 2 then
   return -1
 endi
 if $data00 != 2.000000000 then
+  print expect 2.000000000 , actual: $data00
   return -1
 endi
 if $data10 != 12.083045974 then

From 42a512c39fa8a456496d1b154dc5be22b1bbb1bf Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 2 Dec 2022 15:52:32 +0800
Subject: [PATCH 22/23] refactor: do some internal refactor.

---
 source/dnode/vnode/inc/vnode.h          |  22 +--
 source/dnode/vnode/src/tsdb/tsdbRead.c  | 176 +++++++++++-------------
 source/libs/executor/src/executor.c     |  18 ++-
 source/libs/executor/src/scanoperator.c |  37 ++---
 tests/system-test/2-query/unique.py     |   6 +-
 5 files changed, 116 insertions(+), 143 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 53b8eadb33..64c15c6c0e 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -161,17 +161,17 @@ int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t n
 int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
                        SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr);
 
-void     tsdbReaderClose(STsdbReader *pReader);
-bool     tsdbNextDataBlock(STsdbReader *pReader);
-void     tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow);
-int32_t  tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
-SArray  *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
-int32_t  tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
-int32_t  tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
-int64_t  tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
-void    *tsdbGetIdx(SMeta *pMeta);
-void    *tsdbGetIvtIdx(SMeta *pMeta);
-uint64_t getReaderMaxVersion(STsdbReader *pReader);
+void         tsdbReaderClose(STsdbReader *pReader);
+bool         tsdbNextDataBlock(STsdbReader *pReader);
+void         tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow);
+int32_t      tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockSMA, bool *allHave);
+SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
+int32_t      tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
+int32_t      tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
+int64_t      tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
+void        *tsdbGetIdx(SMeta *pMeta);
+void        *tsdbGetIvtIdx(SMeta *pMeta);
+uint64_t     getReaderMaxVersion(STsdbReader *pReader);
 
 int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
                                 uint64_t suid, void **pReader);
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 24e9181ae6..859ae026a2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -81,12 +81,15 @@ typedef struct SIOCostSummary {
   double  createScanInfoList;
 } SIOCostSummary;
 
+typedef struct SColInfo {
+  int16_t colId;
+  int16_t slotId;
+} SColInfo;
+
 typedef struct SBlockLoadSuppInfo {
   SArray*          pColAgg;
   SColumnDataAgg   tsColAgg;
-  SColumnDataAgg** plist;     // todo remove this
-  int16_t*         colIds;    // column ids for loading file block data
-  int16_t*         slotIds;   // the ordinal index in the destination SSDataBlock
+  SColInfo*        colInfo;    // column ids for loading file block data
   int32_t          numOfCols;
   char**           buildBuf;  // build string tmp buffer, todo remove it later after all string format being updated.
   bool             smaValid;  // the sma on all queried columns are activated
@@ -159,6 +162,7 @@ struct STsdbReader {
   STsdb*             pTsdb;
   uint64_t           suid;
   int16_t            order;
+  bool               freeBlock;
   STimeWindow        window;  // the primary query time window that applies to all queries
   SSDataBlock*       pResBlock;
   int32_t            capacity;
@@ -218,22 +222,21 @@ static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWi
 static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) {
   pSupInfo->smaValid = true;
   pSupInfo->numOfCols = numOfCols;
-  pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t));
-  pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES);
-  pSupInfo->slotIds = taosMemoryMalloc(numOfCols * sizeof(int16_t));
-  if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL || pSupInfo->slotIds == NULL) {
-    taosMemoryFree(pSupInfo->colIds);
-    taosMemoryFree(pSupInfo->buildBuf);
-    taosMemoryFree(pSupInfo->slotIds);
+  pSupInfo->colInfo = taosMemoryMalloc(numOfCols * (sizeof(SColInfo) + POINTER_BYTES));
+  if (pSupInfo->colInfo == NULL) {
+    taosMemoryFree(pSupInfo->colInfo);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  pSupInfo->buildBuf = (char**)((char*)pSupInfo->colInfo + (sizeof(SColInfo) * numOfCols));
   for (int32_t i = 0; i < numOfCols; ++i) {
-    pSupInfo->colIds[i] = pCols[i].colId;
-    pSupInfo->slotIds[i] = pSlotIdList[i];
+    pSupInfo->colInfo[i].colId = pCols[i].colId;
+    pSupInfo->colInfo[i].slotId = pSlotIdList[i];
 
     if (IS_VAR_DATA_TYPE(pCols[i].type)) {
       pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes);
+    } else {
+      pSupInfo->buildBuf[i] = NULL;
     }
   }
 
@@ -245,7 +248,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
 
   while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) {
     STColumn* pTCol = &pSchema->columns[i];
-    if (pTCol->colId == pSupInfo->colIds[j]) {
+    if (pTCol->colId == pSupInfo->colInfo[j].colId) {
       if (!IS_BSMA_ON(pTCol)) {
         pSupInfo->smaValid = false;
         return;
@@ -253,7 +256,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
 
       i += 1;
       j += 1;
-    } else if (pTCol->colId < pSupInfo->colIds[j]) {
+    } else if (pTCol->colId < pSupInfo->colInfo[j].colId) {
       // do nothing
       i += 1;
     } else {
@@ -455,7 +458,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
   if (pLReader->pInfo == NULL) {
     // here we ignore the first column, which is always be the primary timestamp column
     pLReader->pInfo =
-        tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1);
+        tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1);
     if (pLReader->pInfo == NULL) {
       tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr);
       return terrno;
@@ -594,14 +597,22 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
   pReader->blockInfoBuf.numPerBucket = 1000;  // 1000 tables per bucket
   ASSERT(pCond->numOfCols > 0);
 
+  if (pReader->pResBlock == NULL) {
+    pReader->freeBlock = true;
+    pReader->pResBlock = createResBlock(pCond, pReader->capacity);
+    if (pReader->pResBlock == NULL) {
+      code = terrno;
+      goto _end;
+    }
+  }
+
   // todo refactor.
   limitOutputBufferSize(pCond, &pReader->capacity);
 
   // allocate buffer in order to load data blocks from file
   SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
   pSup->pColAgg = taosArrayInit(pCond->numOfCols, sizeof(SColumnDataAgg));
-  pSup->plist = taosMemoryCalloc(pCond->numOfCols, POINTER_BYTES);
-  if (pSup->pColAgg == NULL || pSup->plist == NULL) {
+  if (pSup->pColAgg == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _end;
   }
@@ -1083,8 +1094,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
   int32_t i = 0;
   int32_t rowIndex = 0;
 
-  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
-  if (pSupInfo->colIds[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
+  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+  if (pSupInfo->colInfo[i].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
     copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);
     i += 1;
   }
@@ -1095,10 +1106,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
     rowIndex = 0;
 
     SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);
-    if (pData->cid < pSupInfo->colIds[i]) {
+    if (pData->cid < pSupInfo->colInfo[i].colId) {
       colIndex += 1;
-    } else if (pData->cid == pSupInfo->colIds[i]) {
-      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+    } else if (pData->cid == pSupInfo->colInfo[i].colId) {
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
 
       if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {
         colDataAppendNNULL(pColData, 0, dumpedRows);
@@ -1116,7 +1127,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
       colIndex += 1;
       i += 1;
     } else {  // the specified column does not exist in file block, fill with null data
-      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
       colDataAppendNNULL(pColData, 0, dumpedRows);
       i += 1;
     }
@@ -1124,7 +1135,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
 
   // fill the mis-matched columns with null value
   while (i < numOfOutputCols) {
-    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
     colDataAppendNNULL(pColData, 0, dumpedRows);
     i += 1;
   }
@@ -1162,7 +1173,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
   tBlockDataReset(pBlockData);
   TABLEID tid = {.suid = pReader->suid, .uid = uid};
   int32_t code =
-      tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1);
+      tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -1621,7 +1632,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
   int64_t st = taosGetTimestampUs();
   int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader);
 
-  blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotIds[0]);
+  blockDataUpdateTsWindow(pBlock, pReader->suppInfo.colInfo[0].slotId);
   pBlock->info.id.uid = pBlockScanInfo->uid;
 
   setComposedBlockFlag(pReader, true);
@@ -2493,7 +2504,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
 
 _end:
   pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
-  blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotIds[0]);
+  blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.colInfo[0].slotId);
 
   setComposedBlockFlag(pReader, true);
   double el = (taosGetTimestampUs() - st) / 1000.0;
@@ -3542,24 +3553,24 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
   SColVal colVal = {0};
   int32_t i = 0, j = 0;
 
-  if (pSupInfo->colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
-    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+  if (pSupInfo->colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
     ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts;
     i += 1;
   }
 
   while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) {
-    col_id_t colId = pSupInfo->colIds[i];
+    col_id_t colId = pSupInfo->colInfo[i].colId;
 
     if (colId == pSchema->columns[j].colId) {
-      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
 
       tTSRowGetVal(pTSRow, pSchema, j, &colVal);
       doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo);
       i += 1;
       j += 1;
     } else if (colId < pSchema->columns[j].colId) {
-      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
 
       colDataAppendNULL(pColInfoData, outputRowIndex);
       i += 1;
@@ -3570,7 +3581,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
 
   // set null value since current column does not exist in the "pSchema"
   while (i < pSupInfo->numOfCols) {
-    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotIds[i]);
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
     colDataAppendNULL(pColInfoData, outputRowIndex);
     i += 1;
   }
@@ -3586,8 +3597,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   int32_t outputRowIndex = pResBlock->info.rows;
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
-  if (pReader->suppInfo.colIds[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
-    SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+  if (pReader->suppInfo.colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
     ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex];
     i += 1;
   }
@@ -3598,13 +3609,13 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
 
   while (i < numOfOutputCols && j < numOfInputCols) {
     SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j);
-    if (pData->cid < pSupInfo->colIds[i]) {
+    if (pData->cid < pSupInfo->colInfo[i].colId) {
       j += 1;
       continue;
     }
 
-    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
-    if (pData->cid == pSupInfo->colIds[i]) {
+    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+    if (pData->cid == pSupInfo->colInfo[i].colId) {
       tColDataGetValue(pData, rowIndex, &cv);
       doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
       j += 1;
@@ -3617,7 +3628,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   }
 
   while (i < numOfOutputCols) {
-    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotIds[i]);
+    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
     colDataAppendNULL(pCol, outputRowIndex);
     i += 1;
   }
@@ -3723,7 +3734,14 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
     pCond->twindows.ekey -= 1;
   }
 
-  int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, pResBlock->info.capacity, pResBlock, idstr);
+  int32_t capacity = 0;
+  if (pResBlock == NULL) {
+    capacity = 4096;
+  } else {
+    capacity = pResBlock->info.capacity;
+  }
+
+  int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr);
   if (code != TSDB_CODE_SUCCESS) {
     goto _err;
   }
@@ -3868,10 +3886,6 @@ void tsdbReaderClose(STsdbReader* pReader) {
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
 
-  taosMemoryFreeClear(pSupInfo->plist);
-  taosMemoryFree(pSupInfo->colIds);
-  taosMemoryFree(pSupInfo->slotIds);
-
   taosArrayDestroy(pSupInfo->pColAgg);
   for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) {
     if (pSupInfo->buildBuf[i] != NULL) {
@@ -3879,9 +3893,12 @@ void tsdbReaderClose(STsdbReader* pReader) {
     }
   }
 
-  taosMemoryFree(pSupInfo->buildBuf);
-  tBlockDataDestroy(&pReader->status.fileBlockData, true);
+  if (pReader->freeBlock) {
+    pReader->pResBlock = blockDataDestroy(pReader->pResBlock);
+  }
 
+  taosMemoryFree(pSupInfo->colInfo);
+  tBlockDataDestroy(&pReader->status.fileBlockData, true);
   cleanupDataBlockIterator(&pReader->status.blockIter);
 
   size_t numOfTables = taosHashGetSize(pReader->status.pTableMap);
@@ -4025,18 +4042,18 @@ void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64
   }
 }
 
-int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) {
+int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg ***pBlockSMA, bool* allHave) {
   int32_t code = 0;
   *allHave = false;
 
   if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {
-    *pBlockStatis = NULL;
+    *pBlockSMA = NULL;
     return TSDB_CODE_SUCCESS;
   }
 
   // there is no statistics data for composed block
   if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) {
-    *pBlockStatis = NULL;
+    *pBlockSMA = NULL;
     return TSDB_CODE_SUCCESS;
   }
 
@@ -4054,7 +4071,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
       return code;
     }
   } else {
-    *pBlockStatis = NULL;
+    *pBlockSMA = NULL;
     return TSDB_CODE_SUCCESS;
   }
 
@@ -4067,32 +4084,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
   pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
   pTsAgg->min = pReader->pResBlock->info.window.skey;
   pTsAgg->max = pReader->pResBlock->info.window.ekey;
-  pSup->plist[0] = pTsAgg;
 
   // update the number of NULL data rows
   size_t numOfCols = pSup->numOfCols;
 
   int32_t i = 0, j = 0;
   size_t  size = taosArrayGetSize(pSup->pColAgg);
-#if 0
-  while (j < numOfCols && i < size) {
-    SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
-    if (pAgg->colId == pSup->colIds[j]) {
-      if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) {
-        pSup->plist[j] = pAgg;
-      } else {
-        *allHave = false;
-        break;
-      }
-      i += 1;
-      j += 1;
-    } else if (pAgg->colId < pSup->colIds[j]) {
-      i += 1;
-    } else if (pSup->colIds[j] < pAgg->colId) {
-      j += 1;
-    }
-  }
-#else
 
   SSDataBlock* pResBlock = pReader->pResBlock;
   if (pResBlock->pBlockAgg == NULL) {
@@ -4100,52 +4097,40 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
     pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg));
   }
 
-  // fill the all null data column
-//  SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg));
-
   while (j < numOfCols && i < size) {
     SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
-    if (pAgg->colId == pSup->colIds[j]) {
-      pResBlock->pBlockAgg[pSup->slotIds[j]] = pAgg;
+    if (pAgg->colId == pSup->colInfo[j].colId) {
+      pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = pAgg;
       i += 1;
       j += 1;
-    } else if (pAgg->colId < pSup->colIds[j]) {
+    } else if (pAgg->colId < pSup->colInfo[j].colId) {
       i += 1;
-    } else if (pSup->colIds[j] < pAgg->colId) {
-      if (pSup->colIds[j] == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        pResBlock->pBlockAgg[pSup->slotIds[j]] = &pSup->tsColAgg;
+    } else if (pSup->colInfo[j].colId < pAgg->colId) {
+      if (pSup->colInfo[j].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+        pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = &pSup->tsColAgg;
       } else {
       // all date in this block are null
-        SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow};
+        SColumnDataAgg nullColAgg = {.colId = pSup->colInfo[j].colId, .numOfNull = pBlock->nRow};
         taosArrayPush(pSup->pColAgg, &nullColAgg);
 
-        pResBlock->pBlockAgg[pSup->slotIds[j]] = taosArrayGetLast(pSup->pColAgg);
+        pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = taosArrayGetLast(pSup->pColAgg);
       }
       j += 1;
     }
   }
 
-//  taosArrayClear(pSup->pColAgg);
-
-//  size_t num = taosArrayGetSize(pSup->pColAgg);
-//  for(int32_t k = 0; k < num; ++k) {
-//    pSup->plist[k] = taosArrayGet(pSup->pColAgg, k);
-//  }
-
-#endif
-
+  *pBlockSMA = pResBlock->pBlockAgg;
   pReader->cost.smaDataLoad += 1;
-//  *pBlockStatis = pSup->plist;
 
   tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr);
   return code;
 }
 
-static SArray* doRetrieveDataBlock(STsdbReader* pReader) {
+static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) {
   SReaderStatus* pStatus = &pReader->status;
 
   if (pStatus->composedDataBlock) {
-    return pReader->pResBlock->pDataBlock;
+    return pReader->pResBlock;
   }
 
   SFileDataBlockInfo*  pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter);
@@ -4166,10 +4151,10 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) {
   }
 
   copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
-  return pReader->pResBlock->pDataBlock;
+  return pReader->pResBlock;
 }
 
-SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
+SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
   if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {
     if (pReader->step == EXTERNAL_ROWS_PREV) {
       return doRetrieveDataBlock(pReader->innerReader[0]);
@@ -4196,7 +4181,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
 
   // allocate buffer in order to load data blocks from file
   memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg));
-  memset(pReader->suppInfo.plist, 0, POINTER_BYTES);
 
   pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID;
   tsdbDataFReaderClose(&pReader->pFileReader);
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index d675ac05d5..df11add919 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -957,21 +957,27 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s
   pCond->order = TSDB_ORDER_ASC;
   pCond->numOfCols = pMtInfo->schema->nCols;
   pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo));
-  if (pCond->colList == NULL) {
+  pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t) * pCond->numOfCols);
+  if (pCond->colList == NULL || pCond->pSlotList == NULL) {
+    taosMemoryFreeClear(pCond->colList);
+    taosMemoryFreeClear(pCond->pSlotList);
     terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
     return terrno;
   }
 
-  pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
+  pCond->twindows = TSWINDOW_INITIALIZER;
   pCond->suid = pMtInfo->suid;
   pCond->type = TIMEWINDOW_RANGE_CONTAINED;
   pCond->startVersion = -1;
   pCond->endVersion = sContext->snapVersion;
 
   for (int32_t i = 0; i < pCond->numOfCols; ++i) {
-    pCond->colList[i].type = pMtInfo->schema->pSchema[i].type;
-    pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes;
-    pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId;
+    SColumnInfo* pColInfo = &pCond->colList[i];
+    pColInfo->type = pMtInfo->schema->pSchema[i].type;
+    pColInfo->bytes = pMtInfo->schema->pSchema[i].bytes;
+    pColInfo->colId = pMtInfo->schema->pSchema[i].colId;
+
+    pCond->pSlotList[i] = i;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -1116,7 +1122,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
     int32_t        size = tableListGetSize(pTaskInfo->pTableInfoList);
     ASSERT(size == 1);
 
-    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->pRes, &pInfo->dataReader, NULL);
+    tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL);
 
     cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
     strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index b0104c6025..f09b7dae58 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -224,10 +224,8 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA
 }
 
 static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
-  bool             allColumnsHaveAgg = true;
-  SColumnDataAgg** pColAgg = NULL;
-
-  int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg);
+  bool    allColumnsHaveAgg = true;
+  int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pBlock->pBlockAgg, &allColumnsHaveAgg);
   if (code != TSDB_CODE_SUCCESS) {
     T_LONG_JMP(pTaskInfo->env, code);
   }
@@ -387,11 +385,12 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
   pCost->totalCheckedRows += pBlock->info.rows;
   pCost->loadBlocks += 1;
 
-  SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
-  if (pCols == NULL) {
+  SSDataBlock* p = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
+  if (p == NULL) {
     return terrno;
   }
 
+  ASSERT(p == pBlock);
   doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
 
   // restore the previous value
@@ -994,19 +993,10 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
     return NULL;
   }
 
-  bool hasBlock = tsdbNextDataBlock(pReader);
-  if (hasBlock) {
-    SDataBlockInfo* pBInfo = &pBlock->info;
-
-//    int32_t rows = 0;
-//    tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window);
-
-    SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL);
-
-//    relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true);
+  if (tsdbNextDataBlock(pReader)) {
+    /*SSDataBlock* p = */tsdbRetrieveDataBlock(pReader, NULL);
     doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
-
-    pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->id.uid);
+    pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
   }
 
   tsdbReaderClose(pReader);
@@ -2030,20 +2020,13 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
 
   qDebug("tmqsnap doRawScan called");
   if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
-    SSDataBlock* pBlock = &pInfo->pRes;
-
     if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) {
       if (isTaskKilled(pTaskInfo)) {
         longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
       }
 
-      int32_t rows = 0;
-      tsdbRetrieveDataBlockInfo(pInfo->dataReader, &rows, &pBlock->info.id.uid, &pBlock->info.window);
-      pBlock->info.rows = rows;
-
-      SArray* pCols = tsdbRetrieveDataBlock(pInfo->dataReader, NULL);
-      pBlock->pDataBlock = pCols;
-      if (pCols == NULL) {
+      SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL);
+      if (pBlock == NULL) {
         longjmp(pTaskInfo->env, terrno);
       }
 
diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py
index 2b0336d2d7..6af9b130ef 100644
--- a/tests/system-test/2-query/unique.py
+++ b/tests/system-test/2-query/unique.py
@@ -429,10 +429,10 @@ class TDTestCase:
         tdSql.checkRows(2)
 
         # nest query
-        tdSql.query(f"select  unique(c1) from (select _rowts , t1 ,c1 , tbname from  {dbname}.stb1 ) ")
+        tdSql.query(f"select  unique(c1) v from (select _rowts , t1 ,c1 , tbname from  {dbname}.stb1 ) order by v")
         tdSql.checkRows(11)
-        tdSql.checkData(0,0,6)
-        tdSql.checkData(10,0,3)
+        tdSql.checkData(1,0,0)
+        tdSql.checkData(10,0,9)
         tdSql.query(f"select  unique(t1) from (select _rowts , t1 , tbname from  {dbname}.stb1 )")
         tdSql.checkRows(2)
         tdSql.checkData(0,0,4)

From 27ed740eb56317a5bcb5dc4328fecca8179a51b4 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 2 Dec 2022 17:52:49 +0800
Subject: [PATCH 23/23] refactor: do some internal refactor.

---
 source/dnode/vnode/src/tsdb/tsdbRead.c | 89 +++++++++++++-------------
 1 file changed, 43 insertions(+), 46 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 859ae026a2..ac003783ae 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -81,15 +81,11 @@ typedef struct SIOCostSummary {
   double  createScanInfoList;
 } SIOCostSummary;
 
-typedef struct SColInfo {
-  int16_t colId;
-  int16_t slotId;
-} SColInfo;
-
 typedef struct SBlockLoadSuppInfo {
   SArray*          pColAgg;
   SColumnDataAgg   tsColAgg;
-  SColInfo*        colInfo;    // column ids for loading file block data
+  int16_t*         colId;
+  int16_t*         slotId;
   int32_t          numOfCols;
   char**           buildBuf;  // build string tmp buffer, todo remove it later after all string format being updated.
   bool             smaValid;  // the sma on all queried columns are activated
@@ -222,16 +218,17 @@ static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWi
 static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, int32_t numOfCols) {
   pSupInfo->smaValid = true;
   pSupInfo->numOfCols = numOfCols;
-  pSupInfo->colInfo = taosMemoryMalloc(numOfCols * (sizeof(SColInfo) + POINTER_BYTES));
-  if (pSupInfo->colInfo == NULL) {
-    taosMemoryFree(pSupInfo->colInfo);
+  pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t)*2 + POINTER_BYTES));
+  if (pSupInfo->colId == NULL) {
+    taosMemoryFree(pSupInfo->colId);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  pSupInfo->buildBuf = (char**)((char*)pSupInfo->colInfo + (sizeof(SColInfo) * numOfCols));
+  pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols));
+  pSupInfo->buildBuf = (char**) ((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols));
   for (int32_t i = 0; i < numOfCols; ++i) {
-    pSupInfo->colInfo[i].colId = pCols[i].colId;
-    pSupInfo->colInfo[i].slotId = pSlotIdList[i];
+    pSupInfo->colId[i] = pCols[i].colId;
+    pSupInfo->slotId[i] = pSlotIdList[i];
 
     if (IS_VAR_DATA_TYPE(pCols[i].type)) {
       pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes);
@@ -248,7 +245,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
 
   while(i < pSchema->numOfCols && j < pSupInfo->numOfCols) {
     STColumn* pTCol = &pSchema->columns[i];
-    if (pTCol->colId == pSupInfo->colInfo[j].colId) {
+    if (pTCol->colId == pSupInfo->colId[j]) {
       if (!IS_BSMA_ON(pTCol)) {
         pSupInfo->smaValid = false;
         return;
@@ -256,7 +253,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
 
       i += 1;
       j += 1;
-    } else if (pTCol->colId < pSupInfo->colInfo[j].colId) {
+    } else if (pTCol->colId < pSupInfo->colId[j]) {
       // do nothing
       i += 1;
     } else {
@@ -458,7 +455,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
   if (pLReader->pInfo == NULL) {
     // here we ignore the first column, which is always be the primary timestamp column
     pLReader->pInfo =
-        tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1);
+        tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1);
     if (pLReader->pInfo == NULL) {
       tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr);
       return terrno;
@@ -1094,8 +1091,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
   int32_t i = 0;
   int32_t rowIndex = 0;
 
-  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
-  if (pSupInfo->colInfo[i].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
+  if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
     copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);
     i += 1;
   }
@@ -1106,10 +1103,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
     rowIndex = 0;
 
     SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);
-    if (pData->cid < pSupInfo->colInfo[i].colId) {
+    if (pData->cid < pSupInfo->colId[i]) {
       colIndex += 1;
-    } else if (pData->cid == pSupInfo->colInfo[i].colId) {
-      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+    } else if (pData->cid == pSupInfo->colId[i]) {
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
 
       if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {
         colDataAppendNNULL(pColData, 0, dumpedRows);
@@ -1127,7 +1124,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
       colIndex += 1;
       i += 1;
     } else {  // the specified column does not exist in file block, fill with null data
-      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
       colDataAppendNNULL(pColData, 0, dumpedRows);
       i += 1;
     }
@@ -1135,7 +1132,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
 
   // fill the mis-matched columns with null value
   while (i < numOfOutputCols) {
-    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
     colDataAppendNNULL(pColData, 0, dumpedRows);
     i += 1;
   }
@@ -1173,7 +1170,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
   tBlockDataReset(pBlockData);
   TABLEID tid = {.suid = pReader->suid, .uid = uid};
   int32_t code =
-      tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colInfo[1].colId, pReader->suppInfo.numOfCols - 1);
+      tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -1632,7 +1629,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
   int64_t st = taosGetTimestampUs();
   int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader);
 
-  blockDataUpdateTsWindow(pBlock, pReader->suppInfo.colInfo[0].slotId);
+  blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotId[0]);
   pBlock->info.id.uid = pBlockScanInfo->uid;
 
   setComposedBlockFlag(pReader, true);
@@ -2504,7 +2501,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
 
 _end:
   pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
-  blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.colInfo[0].slotId);
+  blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]);
 
   setComposedBlockFlag(pReader, true);
   double el = (taosGetTimestampUs() - st) / 1000.0;
@@ -3553,24 +3550,24 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
   SColVal colVal = {0};
   int32_t i = 0, j = 0;
 
-  if (pSupInfo->colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) {
-    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+  if (pSupInfo->colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
     ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts;
     i += 1;
   }
 
   while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) {
-    col_id_t colId = pSupInfo->colInfo[i].colId;
+    col_id_t colId = pSupInfo->colId[i];
 
     if (colId == pSchema->columns[j].colId) {
-      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
 
       tTSRowGetVal(pTSRow, pSchema, j, &colVal);
       doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo);
       i += 1;
       j += 1;
     } else if (colId < pSchema->columns[j].colId) {
-      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
 
       colDataAppendNULL(pColInfoData, outputRowIndex);
       i += 1;
@@ -3581,7 +3578,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
 
   // set null value since current column does not exist in the "pSchema"
   while (i < pSupInfo->numOfCols) {
-    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
     colDataAppendNULL(pColInfoData, outputRowIndex);
     i += 1;
   }
@@ -3597,8 +3594,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   int32_t outputRowIndex = pResBlock->info.rows;
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
-  if (pReader->suppInfo.colInfo[i].colId== PRIMARYKEY_TIMESTAMP_COL_ID) {
-    SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+  if (pReader->suppInfo.colId[i]== PRIMARYKEY_TIMESTAMP_COL_ID) {
+    SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
     ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex];
     i += 1;
   }
@@ -3609,13 +3606,13 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
 
   while (i < numOfOutputCols && j < numOfInputCols) {
     SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j);
-    if (pData->cid < pSupInfo->colInfo[i].colId) {
+    if (pData->cid < pSupInfo->colId[i]) {
       j += 1;
       continue;
     }
 
-    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
-    if (pData->cid == pSupInfo->colInfo[i].colId) {
+    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]);
+    if (pData->cid == pSupInfo->colId[i]) {
       tColDataGetValue(pData, rowIndex, &cv);
       doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
       j += 1;
@@ -3628,7 +3625,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
   }
 
   while (i < numOfOutputCols) {
-    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->colInfo[i].slotId);
+    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
     colDataAppendNULL(pCol, outputRowIndex);
     i += 1;
   }
@@ -3897,7 +3894,7 @@ void tsdbReaderClose(STsdbReader* pReader) {
     pReader->pResBlock = blockDataDestroy(pReader->pResBlock);
   }
 
-  taosMemoryFree(pSupInfo->colInfo);
+  taosMemoryFree(pSupInfo->colId);
   tBlockDataDestroy(&pReader->status.fileBlockData, true);
   cleanupDataBlockIterator(&pReader->status.blockIter);
 
@@ -4099,21 +4096,21 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg ***pBlockS
 
   while (j < numOfCols && i < size) {
     SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
-    if (pAgg->colId == pSup->colInfo[j].colId) {
-      pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = pAgg;
+    if (pAgg->colId == pSup->colId[j]) {
+      pResBlock->pBlockAgg[pSup->slotId[j]] = pAgg;
       i += 1;
       j += 1;
-    } else if (pAgg->colId < pSup->colInfo[j].colId) {
+    } else if (pAgg->colId < pSup->colId[j]) {
       i += 1;
-    } else if (pSup->colInfo[j].colId < pAgg->colId) {
-      if (pSup->colInfo[j].colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = &pSup->tsColAgg;
+    } else if (pSup->colId[j] < pAgg->colId) {
+      if (pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID) {
+        pResBlock->pBlockAgg[pSup->slotId[j]] = &pSup->tsColAgg;
       } else {
       // all date in this block are null
-        SColumnDataAgg nullColAgg = {.colId = pSup->colInfo[j].colId, .numOfNull = pBlock->nRow};
+        SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = pBlock->nRow};
         taosArrayPush(pSup->pColAgg, &nullColAgg);
 
-        pResBlock->pBlockAgg[pSup->colInfo[j].slotId] = taosArrayGetLast(pSup->pColAgg);
+        pResBlock->pBlockAgg[pSup->slotId[j]] = taosArrayGetLast(pSup->pColAgg);
       }
       j += 1;
     }