From 854766d9869639d71434e04cfa63ae49d0e0c0aa Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Fri, 8 Dec 2023 15:57:29 +0800
Subject: [PATCH 001/201] enhance: save block then sort by row id

---
 source/libs/executor/inc/executorInt.h  |  13 ++
 source/libs/executor/src/scanoperator.c | 205 +++++++++++++++++++++++-
 2 files changed, 212 insertions(+), 6 deletions(-)

diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index cba26c46b5..c103d8eee7 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -273,6 +273,17 @@ typedef struct STableScanInfo {
   bool            filesetDelimited;
 } STableScanInfo;
 
+typedef struct STmsSortRowIdInfo {
+  int32_t blkId;
+  int64_t dataFileOffset;
+  TdFilePtr idxFile;
+  char idxPath[PATH_MAX];
+  TdFilePtr dataFile;
+  char dataPath[PATH_MAX];
+  SLRUCache* pBlkInfoCache; // blkId->(offset, len)
+  SLRUCache* pBlkDataCache;  // blkId->SSDataBlock*
+} STmsSortRowIdInfo;
+
 typedef struct STableMergeScanInfo {
   int32_t         tableStartIndex;
   int32_t         tableEndIndex;
@@ -301,6 +312,8 @@ typedef struct STableMergeScanInfo {
   bool             bNewFileset;
   bool             bOnlyRetrieveBlock;
   bool             filesetDelimited;
+  bool             bSortRowId;
+  STmsSortRowIdInfo tmsSortRowIdInfo;
 } STableMergeScanInfo;
 
 typedef struct STagScanFilterContext {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index a92770b1f9..d88433c5aa 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3219,6 +3219,141 @@ _error:
   return NULL;
 }
 
+// ========================= table merge scan
+typedef struct STmsSortBlockInfo {
+  int32_t blkId;
+  int32_t length;
+  int64_t offset;
+} STmsSortBlockInfo;
+
+static int32_t saveSourceBlock(STmsSortRowIdInfo* pSortInfo, const SSDataBlock* pSrcBlock, int32_t *pSzBlk) {
+  int32_t szBlk = blockDataGetSize(pSrcBlock) + sizeof(int32_t) + taosArrayGetSize(pSrcBlock->pDataBlock) * sizeof(int32_t);
+  char* buf = taosMemoryMalloc(szBlk);
+  if (buf == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  blockDataToBuf(buf, pSrcBlock);
+  *pSzBlk = szBlk;
+
+  taosLSeekFile(pSortInfo->dataFile, pSortInfo->dataFileOffset, SEEK_SET);
+  taosWriteFile(pSortInfo->dataFile, buf, szBlk);
+
+  STmsSortBlockInfo info = {.blkId = pSortInfo->blkId
+                            , .offset = pSortInfo->dataFileOffset, .length = szBlk};
+  taosLSeekFile(pSortInfo->idxFile, pSortInfo->blkId*sizeof(STmsSortBlockInfo), SEEK_SET);
+  taosWriteFile(pSortInfo->idxFile, &info, sizeof(info));
+
+  return 0;
+}
+
+static int32_t fillSortInputBlock(const STableMergeScanInfo* pInfo,
+                               const SSDataBlock* pSrcBlock, SSDataBlock* pSortInputBlk) {
+  const STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
+
+  int32_t nRows = pSrcBlock->info.rows;
+  pSortInputBlk->info = pSrcBlock->info;
+  blockDataEnsureCapacity(pSortInputBlk, nRows);
+
+  int32_t tsSlotId = ((SBlockOrderInfo*)taosArrayGet(pInfo->pSortInfo, 0))->slotId;
+  SColumnInfoData* tsCol = taosArrayGet(pSortInputBlk->pDataBlock, 0);
+  SColumnInfoData* pSrcTsCol = taosArrayGet(pSrcBlock->pDataBlock, tsSlotId);
+  colDataAssign(tsCol, pSrcTsCol, nRows, &pSortInputBlk->info);
+
+  SColumnInfoData* blkIdCol = taosArrayGet(pSortInputBlk->pDataBlock, 1);
+  colDataSetNItems(blkIdCol, 0, (char*)&pSortInfo->blkId, nRows, false);
+
+  SColumnInfoData* rowIdxCol = taosArrayGet(pSortInputBlk->pDataBlock, 2);
+  for (int32_t i = 0; i < nRows; ++i) {
+    colDataSetInt32(rowIdxCol, i, &i);
+  }
+  return 0;
+}
+
+static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pSortInputBlk) {
+  //TODO: batch save
+  int32_t code = 0;
+  STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
+  int32_t szBlk = 0;
+  code = saveSourceBlock(pSortInfo, pSrcBlock, &szBlk);
+
+  fillSortInputBlock(pInfo, pSrcBlock, pSortInputBlk);
+
+  ++pSortInfo->blkId;
+  pSortInfo->dataFileOffset = ((pSortInfo->dataFileOffset + szBlk) + 4096) & ~4096;
+
+  return code;
+}
+
+static void deleteBlockInfoCache(const void *key, size_t keyLen, void *value, void *ud) {
+  taosMemoryFree(value);
+}
+
+static void deleteBlockDataCache(const void *key, size_t keyLen, void *value, void *ud) {
+  SSDataBlock* pBlock = value;
+  blockDataDestroy(pBlock);
+}
+
+static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId, SSDataBlock** ppBlock) {
+  STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
+
+  LRUHandle* hBlk = taosLRUCacheLookup(pSortInfo->pBlkDataCache, &blockId, sizeof(blockId));
+  if (hBlk) {
+    SSDataBlock* pBlk = taosLRUCacheValue(pSortInfo->pBlkDataCache, hBlk);
+    *ppBlock = pBlk;
+  } else {
+    STmsSortBlockInfo* blkInfo = NULL;
+    LRUHandle*         hBlkInfo = taosLRUCacheLookup(pSortInfo->pBlkInfoCache, &blockId, sizeof(blockId));
+    if (hBlkInfo) {
+      blkInfo = taosLRUCacheValue(pSortInfo->pBlkInfoCache, hBlkInfo);
+    } else {
+      blkInfo = taosMemoryMalloc(sizeof(STmsSortBlockInfo));
+      taosLSeekFile(pSortInfo->idxFile, blockId * sizeof(STmsSortBlockInfo), SEEK_SET);
+      taosReadFile(pSortInfo->idxFile, &blkInfo, sizeof(blkInfo));
+      ASSERT(blkInfo->blkId == blockId);
+      taosLRUCacheInsert(pSortInfo->pBlkInfoCache, &blockId, sizeof(blockId), blkInfo, 1, deleteBlockInfoCache,
+                         &hBlkInfo, TAOS_LRU_PRIORITY_LOW, NULL);
+    }
+    {
+      taosLSeekFile(pSortInfo->dataFile, blkInfo->offset, SEEK_SET);
+      char* buf = taosMemoryMalloc(blkInfo->length);
+      taosReadFile(pSortInfo->dataFile, buf, blkInfo->length);
+      SSDataBlock* pBlock = createOneDataBlock(pInfo->pReaderBlock, false);
+      blockDataFromBuf(pBlock, buf);
+      *ppBlock = pBlock;
+
+      taosLRUCacheInsert(pSortInfo->pBlkInfoCache, &blockId, sizeof(blockId), pBlock, 1, deleteBlockDataCache,
+                         &hBlk, TAOS_LRU_PRIORITY_LOW, NULL);
+    }
+  }
+  return 0;
+}
+
+void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
+  int32_t blkId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
+  int32_t rowIdx = *(int32_t*)tsortGetValue(pTupleHandle, 2);
+  SSDataBlock* pSrcBlk = NULL;
+  retrieveSourceBlock(pInfo, blkId, &pSrcBlk);
+
+  for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
+    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
+    SColumnInfoData* pSrcColInfo = taosArrayGet(pSrcBlk->pDataBlock, i);
+
+    bool isNull = colDataIsNull_s(pSrcColInfo, rowIdx);
+    if (isNull) {
+      colDataSetNULL(pColInfo, pBlock->info.rows);
+    } else {
+      char* pData = colDataGetData(pSrcColInfo, i);
+      if (pData != NULL) {
+        colDataSetVal(pColInfo, pBlock->info.rows, pData, false);
+      }
+    }
+  }
+
+  pBlock->info.dataLoad = 1;
+  pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag;
+  pBlock->info.rows += 1;
+}
+
 static int32_t tableMergeScanDoSkipTable(STableMergeScanInfo* pInfo, SSDataBlock* pBlock) {
   int64_t nRows = 0;
   void*   pNum = tSimpleHashGet(pInfo->mTableNumRows, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid));
@@ -3308,11 +3443,17 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
     if (pInfo->mergeLimit != -1) {
       tableMergeScanDoSkipTable(pInfo, pBlock);
     }
-
+    SSDataBlock* pSortInputBlk = NULL;
+    if (pInfo->bSortRowId) {
+      pSortInputBlk = createOneDataBlock(pInfo->pSortInputBlock, false);
+      transformIntoSortInputBlock(pInfo, pBlock, pSortInputBlk);
+    } else {
+      pSortInputBlk = pBlock;
+    }
     pOperator->resultInfo.totalRows += pBlock->info.rows;
     pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
     
-    return pBlock;
+    return pSortInputBlk;
   }
 
   return NULL;
@@ -3353,6 +3494,33 @@ void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo*
   return;
 }
 
+int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
+  STmsSortRowIdInfo* pSort = &pInfo->tmsSortRowIdInfo;
+  pSort->blkId = 0;
+  pSort->dataFileOffset = 0;
+  taosGetTmpfilePath(tsTempDir, "tms-block-info", pSort->idxPath);
+  pSort->idxFile = taosOpenFile(pSort->idxPath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
+  taosGetTmpfilePath(tsTempDir, "tms-block-data", pSort->dataPath);
+  pSort->dataFile = taosOpenFile(pSort->dataPath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
+  pSort->pBlkInfoCache = taosLRUCacheInit(2048, 0, 0.5);
+  taosLRUCacheSetStrictCapacity(pSort->pBlkInfoCache, false);
+  pSort->pBlkDataCache = taosLRUCacheInit(2048, 0, 0.5);
+  taosLRUCacheSetStrictCapacity(pSort->pBlkInfoCache, false);
+  return 0;
+}
+
+int32_t stopRowIdSort(STableMergeScanInfo *pInfo) {
+  STmsSortRowIdInfo* pSort = &pInfo->tmsSortRowIdInfo;
+  taosCloseFile(&pSort->idxFile);
+  taosRemoveFile(pSort->idxPath);
+  taosCloseFile(&pSort->dataFile);
+  taosRemoveFile(pSort->dataPath);
+
+  taosLRUCacheCleanup(pSort->pBlkInfoCache);
+  taosLRUCacheCleanup(pSort->pBlkDataCache);
+  return 0;
+}
+
 int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
   STableMergeScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
@@ -3361,6 +3529,7 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
 
   pInfo->bNewFileset = false;
 
+  startRowIdSort(pInfo);
   pInfo->sortBufSize = 2048 * pInfo->bufPageSize;
   int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
   pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
@@ -3399,6 +3568,8 @@ void stopDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
 
   tsortDestroySortHandle(pInfo->pSortHandle);
   pInfo->pSortHandle = NULL;
+
+  stopRowIdSort(pInfo);
 }
 
 int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
@@ -3475,8 +3646,11 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock*
       if (pTupleHandle == NULL) {
         break;
       }
-
-      appendOneRowToDataBlock(pResBlock, pTupleHandle);
+      if (!pInfo->bSortRowId) {
+        appendOneRowToDataBlock(pResBlock, pTupleHandle);
+      } else {
+        appendOneRowIdRowToDataBlock(pInfo, pResBlock, pTupleHandle);
+      }
       if (pResBlock->info.rows >= capacity) {
         break;
       }
@@ -3659,9 +3833,27 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
 
   pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
+  if (!pInfo->bSortRowId) {
+    pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
+    pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
+  } else {
+    SSDataBlock* pSortInput = createDataBlock();
+    SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1);
+    blockDataAppendColInfo(pSortInput, &tsCol);
+    SColumnInfoData blkIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
+    blockDataAppendColInfo(pSortInput, &blkIdCol);
+    SColumnInfoData  rowIdxCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
+    blockDataAppendColInfo(pSortInput, &rowIdxCol);
+    pInfo->pSortInputBlock = pSortInput;
 
-  pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
-  pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
+    SArray*         pList = taosArrayInit(1, sizeof(SBlockOrderInfo));
+    SBlockOrderInfo bi = {0};
+    bi.order = pInfo->base.cond.order;
+    bi.slotId = 0;
+    bi.nullFirst = NULL_ORDER_FIRST;
+    taosArrayPush(pList, &bi);
+    pInfo->pSortInfo = pList;
+  }
   initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo);
   pInfo->mTableNumRows = tSimpleHashInit(1024,
                                          taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT));
@@ -3678,6 +3870,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols);
 
   pInfo->filesetDelimited = pTableScanNode->filesetDelimited;
+
   setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED,
                   pInfo, pTaskInfo);
   pOperator->exprSupp.numOfExprs = numOfCols;

From 83139b8d483b5d64ae4ba61b2ec1c8636c6c2029 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 8 Dec 2023 17:45:54 +0800
Subject: [PATCH 002/201] fix: save work

---
 source/libs/executor/src/scanoperator.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index d88433c5aa..07c6c02ba3 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3251,7 +3251,8 @@ static int32_t fillSortInputBlock(const STableMergeScanInfo* pInfo,
   const STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
 
   int32_t nRows = pSrcBlock->info.rows;
-  pSortInputBlk->info = pSrcBlock->info;
+  pSortInputBlk->info.window = pSrcBlock->info.window;
+  pSortInputBlk->info.id = pSrcBlock->info.id;
   blockDataEnsureCapacity(pSortInputBlk, nRows);
 
   int32_t tsSlotId = ((SBlockOrderInfo*)taosArrayGet(pInfo->pSortInfo, 0))->slotId;
@@ -3266,6 +3267,8 @@ static int32_t fillSortInputBlock(const STableMergeScanInfo* pInfo,
   for (int32_t i = 0; i < nRows; ++i) {
     colDataSetInt32(rowIdxCol, i, &i);
   }
+
+  pSortInputBlk->info.rows = nRows;
   return 0;
 }
 
@@ -3305,15 +3308,17 @@ static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId,
     LRUHandle*         hBlkInfo = taosLRUCacheLookup(pSortInfo->pBlkInfoCache, &blockId, sizeof(blockId));
     if (hBlkInfo) {
       blkInfo = taosLRUCacheValue(pSortInfo->pBlkInfoCache, hBlkInfo);
+      uInfo("found block info: %d for %d, offset: %"PRId64", length: %d", blkInfo->blkId, blockId, blkInfo->offset, blkInfo->length)
     } else {
       blkInfo = taosMemoryMalloc(sizeof(STmsSortBlockInfo));
       taosLSeekFile(pSortInfo->idxFile, blockId * sizeof(STmsSortBlockInfo), SEEK_SET);
-      taosReadFile(pSortInfo->idxFile, &blkInfo, sizeof(blkInfo));
+      taosReadFile(pSortInfo->idxFile, blkInfo, sizeof(STmsSortBlockInfo));
       ASSERT(blkInfo->blkId == blockId);
       taosLRUCacheInsert(pSortInfo->pBlkInfoCache, &blockId, sizeof(blockId), blkInfo, 1, deleteBlockInfoCache,
                          &hBlkInfo, TAOS_LRU_PRIORITY_LOW, NULL);
     }
     {
+      uInfo("retrieve block info: %d, offset: %"PRId64", length: %d", blkInfo->blkId, blkInfo->offset, blkInfo->length)
       taosLSeekFile(pSortInfo->dataFile, blkInfo->offset, SEEK_SET);
       char* buf = taosMemoryMalloc(blkInfo->length);
       taosReadFile(pSortInfo->dataFile, buf, blkInfo->length);
@@ -3443,14 +3448,17 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
     if (pInfo->mergeLimit != -1) {
       tableMergeScanDoSkipTable(pInfo, pBlock);
     }
-    SSDataBlock* pSortInputBlk = NULL;
+
+    pOperator->resultInfo.totalRows += pBlock->info.rows;
+
+    SSDataBlock* pSortInputBlk = pInfo->pSortInputBlock;
     if (pInfo->bSortRowId) {
-      pSortInputBlk = createOneDataBlock(pInfo->pSortInputBlock, false);
+      blockDataCleanup(pSortInputBlk);
       transformIntoSortInputBlock(pInfo, pBlock, pSortInputBlk);
     } else {
       pSortInputBlk = pBlock;
     }
-    pOperator->resultInfo.totalRows += pBlock->info.rows;
+
     pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
     
     return pSortInputBlk;
@@ -3833,6 +3841,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
 
   pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
+  pInfo->bSortRowId = true;
   if (!pInfo->bSortRowId) {
     pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
     pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);

From 6acbbc3cffb93b7dc551a565d488c7338b0f0917 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 8 Dec 2023 21:15:06 +0800
Subject: [PATCH 003/201] fix: typo error

---
 source/libs/executor/src/scanoperator.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 07c6c02ba3..efa054cb9a 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3308,7 +3308,6 @@ static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId,
     LRUHandle*         hBlkInfo = taosLRUCacheLookup(pSortInfo->pBlkInfoCache, &blockId, sizeof(blockId));
     if (hBlkInfo) {
       blkInfo = taosLRUCacheValue(pSortInfo->pBlkInfoCache, hBlkInfo);
-      uInfo("found block info: %d for %d, offset: %"PRId64", length: %d", blkInfo->blkId, blockId, blkInfo->offset, blkInfo->length)
     } else {
       blkInfo = taosMemoryMalloc(sizeof(STmsSortBlockInfo));
       taosLSeekFile(pSortInfo->idxFile, blockId * sizeof(STmsSortBlockInfo), SEEK_SET);
@@ -3318,7 +3317,6 @@ static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId,
                          &hBlkInfo, TAOS_LRU_PRIORITY_LOW, NULL);
     }
     {
-      uInfo("retrieve block info: %d, offset: %"PRId64", length: %d", blkInfo->blkId, blkInfo->offset, blkInfo->length)
       taosLSeekFile(pSortInfo->dataFile, blkInfo->offset, SEEK_SET);
       char* buf = taosMemoryMalloc(blkInfo->length);
       taosReadFile(pSortInfo->dataFile, buf, blkInfo->length);
@@ -3326,7 +3324,7 @@ static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId,
       blockDataFromBuf(pBlock, buf);
       *ppBlock = pBlock;
 
-      taosLRUCacheInsert(pSortInfo->pBlkInfoCache, &blockId, sizeof(blockId), pBlock, 1, deleteBlockDataCache,
+      taosLRUCacheInsert(pSortInfo->pBlkDataCache, &blockId, sizeof(blockId), pBlock, 1, deleteBlockDataCache,
                          &hBlk, TAOS_LRU_PRIORITY_LOW, NULL);
     }
   }

From 205e3ecf271027b458c7a0666be1f53309a8145a Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 8 Dec 2023 21:49:36 +0800
Subject: [PATCH 004/201] fix: fix mem leak of block data buf

---
 source/libs/executor/src/scanoperator.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index efa054cb9a..77a4119be0 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3233,15 +3233,16 @@ static int32_t saveSourceBlock(STmsSortRowIdInfo* pSortInfo, const SSDataBlock*
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   blockDataToBuf(buf, pSrcBlock);
-  *pSzBlk = szBlk;
-
   taosLSeekFile(pSortInfo->dataFile, pSortInfo->dataFileOffset, SEEK_SET);
   taosWriteFile(pSortInfo->dataFile, buf, szBlk);
+  taosMemoryFree(buf);
 
   STmsSortBlockInfo info = {.blkId = pSortInfo->blkId
                             , .offset = pSortInfo->dataFileOffset, .length = szBlk};
   taosLSeekFile(pSortInfo->idxFile, pSortInfo->blkId*sizeof(STmsSortBlockInfo), SEEK_SET);
   taosWriteFile(pSortInfo->idxFile, &info, sizeof(info));
+  
+  *pSzBlk = szBlk;
 
   return 0;
 }
@@ -3322,6 +3323,8 @@ static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId,
       taosReadFile(pSortInfo->dataFile, buf, blkInfo->length);
       SSDataBlock* pBlock = createOneDataBlock(pInfo->pReaderBlock, false);
       blockDataFromBuf(pBlock, buf);
+      taosMemoryFree(buf);
+
       *ppBlock = pBlock;
 
       taosLRUCacheInsert(pSortInfo->pBlkDataCache, &blockId, sizeof(blockId), pBlock, 1, deleteBlockDataCache,

From 92a695cd2f3b68ca51ccb160d7967b5a6f7fd6e3 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 8 Dec 2023 22:12:24 +0800
Subject: [PATCH 005/201] fix: memory leaks

---
 source/libs/executor/src/scanoperator.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 77a4119be0..bea202f96d 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3241,7 +3241,7 @@ static int32_t saveSourceBlock(STmsSortRowIdInfo* pSortInfo, const SSDataBlock*
                             , .offset = pSortInfo->dataFileOffset, .length = szBlk};
   taosLSeekFile(pSortInfo->idxFile, pSortInfo->blkId*sizeof(STmsSortBlockInfo), SEEK_SET);
   taosWriteFile(pSortInfo->idxFile, &info, sizeof(info));
-  
+
   *pSzBlk = szBlk;
 
   return 0;
@@ -3525,7 +3525,9 @@ int32_t stopRowIdSort(STableMergeScanInfo *pInfo) {
   taosCloseFile(&pSort->dataFile);
   taosRemoveFile(pSort->dataPath);
 
+  taosLRUCacheEraseUnrefEntries(pSort->pBlkInfoCache);
   taosLRUCacheCleanup(pSort->pBlkInfoCache);
+  taosLRUCacheEraseUnrefEntries(pSort->pBlkDataCache);
   taosLRUCacheCleanup(pSort->pBlkDataCache);
   return 0;
 }

From 9a9264e03a2e197c67b03fb61fe7b5321eab9fd6 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Sat, 9 Dec 2023 19:00:23 +0800
Subject: [PATCH 006/201] fix: fix memory leak

---
 source/libs/executor/src/scanoperator.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index bea202f96d..a33cc11a6e 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3303,19 +3303,21 @@ static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId,
   LRUHandle* hBlk = taosLRUCacheLookup(pSortInfo->pBlkDataCache, &blockId, sizeof(blockId));
   if (hBlk) {
     SSDataBlock* pBlk = taosLRUCacheValue(pSortInfo->pBlkDataCache, hBlk);
+    taosLRUCacheRelease(pSortInfo->pBlkDataCache, hBlk, false);
     *ppBlock = pBlk;
   } else {
     STmsSortBlockInfo* blkInfo = NULL;
     LRUHandle*         hBlkInfo = taosLRUCacheLookup(pSortInfo->pBlkInfoCache, &blockId, sizeof(blockId));
     if (hBlkInfo) {
       blkInfo = taosLRUCacheValue(pSortInfo->pBlkInfoCache, hBlkInfo);
+      taosLRUCacheRelease(pSortInfo->pBlkInfoCache, hBlkInfo, false);
     } else {
       blkInfo = taosMemoryMalloc(sizeof(STmsSortBlockInfo));
       taosLSeekFile(pSortInfo->idxFile, blockId * sizeof(STmsSortBlockInfo), SEEK_SET);
       taosReadFile(pSortInfo->idxFile, blkInfo, sizeof(STmsSortBlockInfo));
       ASSERT(blkInfo->blkId == blockId);
       taosLRUCacheInsert(pSortInfo->pBlkInfoCache, &blockId, sizeof(blockId), blkInfo, 1, deleteBlockInfoCache,
-                         &hBlkInfo, TAOS_LRU_PRIORITY_LOW, NULL);
+                         NULL, TAOS_LRU_PRIORITY_LOW, NULL);
     }
     {
       taosLSeekFile(pSortInfo->dataFile, blkInfo->offset, SEEK_SET);
@@ -3328,7 +3330,7 @@ static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId,
       *ppBlock = pBlock;
 
       taosLRUCacheInsert(pSortInfo->pBlkDataCache, &blockId, sizeof(blockId), pBlock, 1, deleteBlockDataCache,
-                         &hBlk, TAOS_LRU_PRIORITY_LOW, NULL);
+                         NULL, TAOS_LRU_PRIORITY_LOW, NULL);
     }
   }
   return 0;
@@ -3511,9 +3513,9 @@ int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
   pSort->idxFile = taosOpenFile(pSort->idxPath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
   taosGetTmpfilePath(tsTempDir, "tms-block-data", pSort->dataPath);
   pSort->dataFile = taosOpenFile(pSort->dataPath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
-  pSort->pBlkInfoCache = taosLRUCacheInit(2048, 0, 0.5);
+  pSort->pBlkInfoCache = taosLRUCacheInit(2048, -1, 0.5);
   taosLRUCacheSetStrictCapacity(pSort->pBlkInfoCache, false);
-  pSort->pBlkDataCache = taosLRUCacheInit(2048, 0, 0.5);
+  pSort->pBlkDataCache = taosLRUCacheInit(2048, -1, 0.5);
   taosLRUCacheSetStrictCapacity(pSort->pBlkInfoCache, false);
   return 0;
 }

From d01e82439e2e232998cc98b4ea3efdb0a9654522 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Sun, 10 Dec 2023 19:11:39 +0800
Subject: [PATCH 007/201] feature: replace two lru cache with hash

---
 source/libs/executor/inc/executorInt.h  |  3 +-
 source/libs/executor/src/scanoperator.c | 69 ++++++++-----------------
 2 files changed, 23 insertions(+), 49 deletions(-)

diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index c103d8eee7..9a9338a6c7 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -280,8 +280,7 @@ typedef struct STmsSortRowIdInfo {
   char idxPath[PATH_MAX];
   TdFilePtr dataFile;
   char dataPath[PATH_MAX];
-  SLRUCache* pBlkInfoCache; // blkId->(offset, len)
-  SLRUCache* pBlkDataCache;  // blkId->SSDataBlock*
+  SSHashObj* pBlkDataHash;  // blkId->SSDataBlock*
 } STmsSortRowIdInfo;
 
 typedef struct STableMergeScanInfo {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index a33cc11a6e..469d119523 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3288,50 +3288,28 @@ static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlo
   return code;
 }
 
-static void deleteBlockInfoCache(const void *key, size_t keyLen, void *value, void *ud) {
-  taosMemoryFree(value);
-}
-
-static void deleteBlockDataCache(const void *key, size_t keyLen, void *value, void *ud) {
-  SSDataBlock* pBlock = value;
-  blockDataDestroy(pBlock);
-}
 
 static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId, SSDataBlock** ppBlock) {
   STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
 
-  LRUHandle* hBlk = taosLRUCacheLookup(pSortInfo->pBlkDataCache, &blockId, sizeof(blockId));
-  if (hBlk) {
-    SSDataBlock* pBlk = taosLRUCacheValue(pSortInfo->pBlkDataCache, hBlk);
-    taosLRUCacheRelease(pSortInfo->pBlkDataCache, hBlk, false);
-    *ppBlock = pBlk;
-  } else {
-    STmsSortBlockInfo* blkInfo = NULL;
-    LRUHandle*         hBlkInfo = taosLRUCacheLookup(pSortInfo->pBlkInfoCache, &blockId, sizeof(blockId));
-    if (hBlkInfo) {
-      blkInfo = taosLRUCacheValue(pSortInfo->pBlkInfoCache, hBlkInfo);
-      taosLRUCacheRelease(pSortInfo->pBlkInfoCache, hBlkInfo, false);
-    } else {
-      blkInfo = taosMemoryMalloc(sizeof(STmsSortBlockInfo));
-      taosLSeekFile(pSortInfo->idxFile, blockId * sizeof(STmsSortBlockInfo), SEEK_SET);
-      taosReadFile(pSortInfo->idxFile, blkInfo, sizeof(STmsSortBlockInfo));
-      ASSERT(blkInfo->blkId == blockId);
-      taosLRUCacheInsert(pSortInfo->pBlkInfoCache, &blockId, sizeof(blockId), blkInfo, 1, deleteBlockInfoCache,
-                         NULL, TAOS_LRU_PRIORITY_LOW, NULL);
-    }
-    {
-      taosLSeekFile(pSortInfo->dataFile, blkInfo->offset, SEEK_SET);
-      char* buf = taosMemoryMalloc(blkInfo->length);
-      taosReadFile(pSortInfo->dataFile, buf, blkInfo->length);
-      SSDataBlock* pBlock = createOneDataBlock(pInfo->pReaderBlock, false);
-      blockDataFromBuf(pBlock, buf);
-      taosMemoryFree(buf);
+  void* pBlkHashVal  = tSimpleHashGet(pSortInfo->pBlkDataHash, &blockId, sizeof(blockId));
+  if (pBlkHashVal) {
+    *ppBlock = *(SSDataBlock**)pBlkHashVal;
+  }
+  else {
+    STmsSortBlockInfo blkInfo = {0};
 
-      *ppBlock = pBlock;
+    taosLSeekFile(pSortInfo->idxFile, blockId * sizeof(STmsSortBlockInfo), SEEK_SET);
+    taosReadFile(pSortInfo->idxFile, &blkInfo, sizeof(STmsSortBlockInfo));
+    taosLSeekFile(pSortInfo->dataFile, blkInfo.offset, SEEK_SET);
+    char* buf = taosMemoryMalloc(blkInfo.length);
+    taosReadFile(pSortInfo->dataFile, buf, blkInfo.length);
+    SSDataBlock* pBlock = createOneDataBlock(pInfo->pReaderBlock, false);
+    blockDataFromBuf(pBlock, buf);
+    taosMemoryFree(buf);
 
-      taosLRUCacheInsert(pSortInfo->pBlkDataCache, &blockId, sizeof(blockId), pBlock, 1, deleteBlockDataCache,
-                         NULL, TAOS_LRU_PRIORITY_LOW, NULL);
-    }
+    *ppBlock = pBlock;
+    tSimpleHashPut(pSortInfo->pBlkDataHash, &blockId, sizeof(blockId), &pBlock, sizeof(pBlock));
   }
   return 0;
 }
@@ -3356,7 +3334,10 @@ void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBloc
       }
     }
   }
-
+  if (rowIdx == pSrcBlk->info.rows - 1) {
+    tSimpleHashRemove(pInfo->tmsSortRowIdInfo.pBlkDataHash, &blkId, sizeof(blkId));
+    blockDataDestroy(pSrcBlk);
+  }
   pBlock->info.dataLoad = 1;
   pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag;
   pBlock->info.rows += 1;
@@ -3513,10 +3494,7 @@ int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
   pSort->idxFile = taosOpenFile(pSort->idxPath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
   taosGetTmpfilePath(tsTempDir, "tms-block-data", pSort->dataPath);
   pSort->dataFile = taosOpenFile(pSort->dataPath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
-  pSort->pBlkInfoCache = taosLRUCacheInit(2048, -1, 0.5);
-  taosLRUCacheSetStrictCapacity(pSort->pBlkInfoCache, false);
-  pSort->pBlkDataCache = taosLRUCacheInit(2048, -1, 0.5);
-  taosLRUCacheSetStrictCapacity(pSort->pBlkInfoCache, false);
+  pSort->pBlkDataHash = tSimpleHashInit(2048, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
   return 0;
 }
 
@@ -3527,10 +3505,7 @@ int32_t stopRowIdSort(STableMergeScanInfo *pInfo) {
   taosCloseFile(&pSort->dataFile);
   taosRemoveFile(pSort->dataPath);
 
-  taosLRUCacheEraseUnrefEntries(pSort->pBlkInfoCache);
-  taosLRUCacheCleanup(pSort->pBlkInfoCache);
-  taosLRUCacheEraseUnrefEntries(pSort->pBlkDataCache);
-  taosLRUCacheCleanup(pSort->pBlkDataCache);
+  tSimpleHashCleanup(pSort->pBlkDataHash);
   return 0;
 }
 

From c705a71bd9eaf44feadf8b7137d3afb567315081 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Sun, 10 Dec 2023 22:25:46 +0800
Subject: [PATCH 008/201] feat: use disk based buf for src block storage

---
 source/libs/executor/inc/executorInt.h  |   7 +-
 source/libs/executor/src/scanoperator.c | 109 +++++++++---------------
 2 files changed, 40 insertions(+), 76 deletions(-)

diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index 9a9338a6c7..7a9160b392 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -274,12 +274,7 @@ typedef struct STableScanInfo {
 } STableScanInfo;
 
 typedef struct STmsSortRowIdInfo {
-  int32_t blkId;
-  int64_t dataFileOffset;
-  TdFilePtr idxFile;
-  char idxPath[PATH_MAX];
-  TdFilePtr dataFile;
-  char dataPath[PATH_MAX];
+  SDiskbasedBuf* pExtSrcBlkBuf;
   SSHashObj* pBlkDataHash;  // blkId->SSDataBlock*
 } STmsSortRowIdInfo;
 
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 469d119523..6f2e19e694 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3220,36 +3220,8 @@ _error:
 }
 
 // ========================= table merge scan
-typedef struct STmsSortBlockInfo {
-  int32_t blkId;
-  int32_t length;
-  int64_t offset;
-} STmsSortBlockInfo;
-
-static int32_t saveSourceBlock(STmsSortRowIdInfo* pSortInfo, const SSDataBlock* pSrcBlock, int32_t *pSzBlk) {
-  int32_t szBlk = blockDataGetSize(pSrcBlock) + sizeof(int32_t) + taosArrayGetSize(pSrcBlock->pDataBlock) * sizeof(int32_t);
-  char* buf = taosMemoryMalloc(szBlk);
-  if (buf == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  blockDataToBuf(buf, pSrcBlock);
-  taosLSeekFile(pSortInfo->dataFile, pSortInfo->dataFileOffset, SEEK_SET);
-  taosWriteFile(pSortInfo->dataFile, buf, szBlk);
-  taosMemoryFree(buf);
-
-  STmsSortBlockInfo info = {.blkId = pSortInfo->blkId
-                            , .offset = pSortInfo->dataFileOffset, .length = szBlk};
-  taosLSeekFile(pSortInfo->idxFile, pSortInfo->blkId*sizeof(STmsSortBlockInfo), SEEK_SET);
-  taosWriteFile(pSortInfo->idxFile, &info, sizeof(info));
-
-  *pSzBlk = szBlk;
-
-  return 0;
-}
-
-static int32_t fillSortInputBlock(const STableMergeScanInfo* pInfo,
-                               const SSDataBlock* pSrcBlock, SSDataBlock* pSortInputBlk) {
-  const STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
+static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pSortInputBlk) {
+  STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
 
   int32_t nRows = pSrcBlock->info.rows;
   pSortInputBlk->info.window = pSrcBlock->info.window;
@@ -3262,32 +3234,41 @@ static int32_t fillSortInputBlock(const STableMergeScanInfo* pInfo,
   colDataAssign(tsCol, pSrcTsCol, nRows, &pSortInputBlk->info);
 
   SColumnInfoData* blkIdCol = taosArrayGet(pSortInputBlk->pDataBlock, 1);
-  colDataSetNItems(blkIdCol, 0, (char*)&pSortInfo->blkId, nRows, false);
-
   SColumnInfoData* rowIdxCol = taosArrayGet(pSortInputBlk->pDataBlock, 2);
-  for (int32_t i = 0; i < nRows; ++i) {
-    colDataSetInt32(rowIdxCol, i, &i);
+
+  int32_t start = 0;
+  while (start < pSrcBlock->info.rows) {
+    int32_t stop = 0;
+    blockDataSplitRows(pSrcBlock, pSrcBlock->info.hasVarCol, start, &stop, pInfo->bufPageSize);
+    SSDataBlock* p = blockDataExtractBlock(pSrcBlock, start, stop-start+1);
+
+    int32_t pageId = -1;
+    void* pPage = getNewBufPage(pSortInfo->pExtSrcBlkBuf, &pageId);
+
+    int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t);
+    ASSERT(size <= getBufPageSize(pSortInfo->pExtSrcBlkBuf));
+
+    blockDataToBuf(pPage, p);
+    
+    setBufPageDirty(pPage, true);
+    releaseBufPage(pSortInfo->pExtSrcBlkBuf, pPage);
+
+    blockDataDestroy(p);
+    uInfo("sort input block pageId %d start %d, stop %d", pageId, start, stop);
+    colDataSetNItems(blkIdCol, start, (char*)&pageId, stop-start+1, false);
+
+    for (int32_t i = start; i <= stop; ++i) {
+      int32_t rowIdx = i - start;
+      colDataSetInt32(rowIdxCol, i, &rowIdx);
+    }
+    start = stop + 1;
   }
 
   pSortInputBlk->info.rows = nRows;
+
   return 0;
 }
 
-static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pSortInputBlk) {
-  //TODO: batch save
-  int32_t code = 0;
-  STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
-  int32_t szBlk = 0;
-  code = saveSourceBlock(pSortInfo, pSrcBlock, &szBlk);
-
-  fillSortInputBlock(pInfo, pSrcBlock, pSortInputBlk);
-
-  ++pSortInfo->blkId;
-  pSortInfo->dataFileOffset = ((pSortInfo->dataFileOffset + szBlk) + 4096) & ~4096;
-
-  return code;
-}
-
 
 static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId, SSDataBlock** ppBlock) {
   STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
@@ -3297,16 +3278,10 @@ static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId,
     *ppBlock = *(SSDataBlock**)pBlkHashVal;
   }
   else {
-    STmsSortBlockInfo blkInfo = {0};
-
-    taosLSeekFile(pSortInfo->idxFile, blockId * sizeof(STmsSortBlockInfo), SEEK_SET);
-    taosReadFile(pSortInfo->idxFile, &blkInfo, sizeof(STmsSortBlockInfo));
-    taosLSeekFile(pSortInfo->dataFile, blkInfo.offset, SEEK_SET);
-    char* buf = taosMemoryMalloc(blkInfo.length);
-    taosReadFile(pSortInfo->dataFile, buf, blkInfo.length);
+    void* pPage = getBufPage(pSortInfo->pExtSrcBlkBuf, blockId);
     SSDataBlock* pBlock = createOneDataBlock(pInfo->pReaderBlock, false);
-    blockDataFromBuf(pBlock, buf);
-    taosMemoryFree(buf);
+    blockDataFromBuf(pBlock, pPage);
+    releaseBufPage(pSortInfo->pExtSrcBlkBuf, pPage);
 
     *ppBlock = pBlock;
     tSimpleHashPut(pSortInfo->pBlkDataHash, &blockId, sizeof(blockId), &pBlock, sizeof(pBlock));
@@ -3318,8 +3293,9 @@ void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBloc
   int32_t blkId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
   int32_t rowIdx = *(int32_t*)tsortGetValue(pTupleHandle, 2);
   SSDataBlock* pSrcBlk = NULL;
+  uInfo("sort tuple blkId %d, row idx %d", blkId, rowIdx);
   retrieveSourceBlock(pInfo, blkId, &pSrcBlk);
-
+  
   for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
     SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
     SColumnInfoData* pSrcColInfo = taosArrayGet(pSrcBlk->pDataBlock, i);
@@ -3328,7 +3304,7 @@ void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBloc
     if (isNull) {
       colDataSetNULL(pColInfo, pBlock->info.rows);
     } else {
-      char* pData = colDataGetData(pSrcColInfo, i);
+      char* pData = colDataGetData(pSrcColInfo, rowIdx);
       if (pData != NULL) {
         colDataSetVal(pColInfo, pBlock->info.rows, pData, false);
       }
@@ -3488,23 +3464,16 @@ void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo*
 
 int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
   STmsSortRowIdInfo* pSort = &pInfo->tmsSortRowIdInfo;
-  pSort->blkId = 0;
-  pSort->dataFileOffset = 0;
-  taosGetTmpfilePath(tsTempDir, "tms-block-info", pSort->idxPath);
-  pSort->idxFile = taosOpenFile(pSort->idxPath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
-  taosGetTmpfilePath(tsTempDir, "tms-block-data", pSort->dataPath);
-  pSort->dataFile = taosOpenFile(pSort->dataPath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
+  createDiskbasedBuf(&pSort->pExtSrcBlkBuf, pInfo->bufPageSize, pInfo->sortBufSize, "tms-ext-src-block", tsTempDir);
+  dBufSetPrintInfo(pSort->pExtSrcBlkBuf);
   pSort->pBlkDataHash = tSimpleHashInit(2048, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
   return 0;
 }
 
 int32_t stopRowIdSort(STableMergeScanInfo *pInfo) {
   STmsSortRowIdInfo* pSort = &pInfo->tmsSortRowIdInfo;
-  taosCloseFile(&pSort->idxFile);
-  taosRemoveFile(pSort->idxPath);
-  taosCloseFile(&pSort->dataFile);
-  taosRemoveFile(pSort->dataPath);
 
+  destroyDiskbasedBuf(pSort->pExtSrcBlkBuf);
   tSimpleHashCleanup(pSort->pBlkDataHash);
   return 0;
 }

From 3b1a1859496741a35bacd49d2cf1edad7fd5d3d9 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Mon, 11 Dec 2023 09:12:16 +0800
Subject: [PATCH 009/201] fix: coldataSetNItems error

---
 source/libs/executor/src/scanoperator.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 6f2e19e694..8914c2aeac 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3254,10 +3254,8 @@ static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlo
     releaseBufPage(pSortInfo->pExtSrcBlkBuf, pPage);
 
     blockDataDestroy(p);
-    uInfo("sort input block pageId %d start %d, stop %d", pageId, start, stop);
-    colDataSetNItems(blkIdCol, start, (char*)&pageId, stop-start+1, false);
-
     for (int32_t i = start; i <= stop; ++i) {
+      colDataSetInt32(blkIdCol, i, &pageId);
       int32_t rowIdx = i - start;
       colDataSetInt32(rowIdxCol, i, &rowIdx);
     }
@@ -3293,7 +3291,6 @@ void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBloc
   int32_t blkId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
   int32_t rowIdx = *(int32_t*)tsortGetValue(pTupleHandle, 2);
   SSDataBlock* pSrcBlk = NULL;
-  uInfo("sort tuple blkId %d, row idx %d", blkId, rowIdx);
   retrieveSourceBlock(pInfo, blkId, &pSrcBlk);
   
   for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
@@ -3411,10 +3408,11 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
 
     pOperator->resultInfo.totalRows += pBlock->info.rows;
 
-    SSDataBlock* pSortInputBlk = pInfo->pSortInputBlock;
+    SSDataBlock* pSortInputBlk = NULL;
     if (pInfo->bSortRowId) {
-      blockDataCleanup(pSortInputBlk);
-      transformIntoSortInputBlock(pInfo, pBlock, pSortInputBlk);
+      blockDataCleanup(pInfo->pSortInputBlock);
+      transformIntoSortInputBlock(pInfo, pBlock, pInfo->pSortInputBlock);
+      pSortInputBlk = pInfo->pSortInputBlock;
     } else {
       pSortInputBlk = pBlock;
     }
@@ -3474,7 +3472,9 @@ int32_t stopRowIdSort(STableMergeScanInfo *pInfo) {
   STmsSortRowIdInfo* pSort = &pInfo->tmsSortRowIdInfo;
 
   destroyDiskbasedBuf(pSort->pExtSrcBlkBuf);
+  pSort->pExtSrcBlkBuf = NULL;
   tSimpleHashCleanup(pSort->pBlkDataHash);
+  pSort->pBlkDataHash = NULL;
   return 0;
 }
 

From 7f93cb9f1a5f0524dcb917c626d119bdc0f528f1 Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Wed, 13 Dec 2023 11:38:14 +0800
Subject: [PATCH 010/201] fix: use pageid, offset, length as row index

---
 source/libs/executor/inc/executorInt.h  |   3 +-
 source/libs/executor/src/scanoperator.c | 197 +++++++++++++++---------
 2 files changed, 124 insertions(+), 76 deletions(-)

diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index 7a9160b392..35f2e28d36 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -274,8 +274,7 @@ typedef struct STableScanInfo {
 } STableScanInfo;
 
 typedef struct STmsSortRowIdInfo {
-  SDiskbasedBuf* pExtSrcBlkBuf;
-  SSHashObj* pBlkDataHash;  // blkId->SSDataBlock*
+  SDiskbasedBuf* pExtSrcRowsBuf;
 } STmsSortRowIdInfo;
 
 typedef struct STableMergeScanInfo {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 8914c2aeac..be5cf298a4 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3220,6 +3220,80 @@ _error:
 }
 
 // ========================= table merge scan
+
+static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
+  SDiskbasedBuf* pResultBuf = pInfo->tmsSortRowIdInfo.pExtSrcRowsBuf;
+  int32_t rowBytes = blockDataGetRowSize(pBlock);
+
+  SFilePage* pFilePage = NULL;
+
+  // in the first scan, new space needed for results
+  int32_t pageId = -1;
+  SArray* list = getDataBufPagesIdList(pResultBuf);
+
+  if (taosArrayGetSize(list) == 0) {
+    pFilePage = getNewBufPage(pResultBuf, &pageId);
+    pFilePage->num = sizeof(SFilePage);
+  } else {
+    SPageInfo* pi = getLastPageInfo(list);
+    pFilePage = getBufPage(pResultBuf, getPageId(pi));
+    if (pFilePage == NULL) {
+      qError("failed to get buffer, code:%s", tstrerror(terrno));
+      return terrno;
+    }
+
+    pageId = getPageId(pi);
+
+    if (pFilePage->num + rowBytes > getBufPageSize(pResultBuf)) {
+      // release current page first, and prepare the next one
+      releaseBufPageInfo(pResultBuf, pi);
+
+      pFilePage = getNewBufPage(pResultBuf, &pageId);
+      if (pFilePage != NULL) {
+        pFilePage->num = sizeof(SFilePage);
+      }
+    }
+  }
+
+  if (pFilePage == NULL) {
+    return -1;
+  }
+  *pPageId = pageId;
+  *pOffset = pFilePage->num;
+  char* buf = (char*)pFilePage + (*pOffset);
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+
+  char* isNull = (char*)buf;
+  char* pStart = (char*)buf + sizeof(int8_t) * numOfCols;
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
+    if (colDataIsNull_s(pCol, rowIdx)) {
+      isNull[i] = 1;
+      continue;
+    }
+
+    isNull[i] = 0;
+    char* pData = colDataGetData(pCol, rowIdx);
+    if (pCol->info.type == TSDB_DATA_TYPE_JSON) {
+      int32_t dataLen = getJsonValueLen(pData);
+      memcpy(pStart, pData, dataLen);
+      pStart += dataLen;
+    } else if (IS_VAR_DATA_TYPE(pCol->info.type)) {
+      varDataCopy(pStart, pData);
+      pStart += varDataTLen(pData);
+    } else {
+      int32_t bytes = pCol->info.bytes;
+      memcpy(pStart, pData, bytes);
+      pStart += bytes;
+    }
+  }
+  *pLength = (int32_t)(pStart - (char*)buf);
+  pFilePage->num += (*pLength);
+  setBufPageDirty(pFilePage, true);
+  releaseBufPage(pResultBuf, pFilePage);
+  return 0;
+}
+
 static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pSortInputBlk) {
   STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
 
@@ -3233,33 +3307,18 @@ static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlo
   SColumnInfoData* pSrcTsCol = taosArrayGet(pSrcBlock->pDataBlock, tsSlotId);
   colDataAssign(tsCol, pSrcTsCol, nRows, &pSortInputBlk->info);
 
-  SColumnInfoData* blkIdCol = taosArrayGet(pSortInputBlk->pDataBlock, 1);
-  SColumnInfoData* rowIdxCol = taosArrayGet(pSortInputBlk->pDataBlock, 2);
-
-  int32_t start = 0;
-  while (start < pSrcBlock->info.rows) {
-    int32_t stop = 0;
-    blockDataSplitRows(pSrcBlock, pSrcBlock->info.hasVarCol, start, &stop, pInfo->bufPageSize);
-    SSDataBlock* p = blockDataExtractBlock(pSrcBlock, start, stop-start+1);
+  SColumnInfoData* pageIdCol = taosArrayGet(pSortInputBlk->pDataBlock, 1);
+  SColumnInfoData* offsetCol = taosArrayGet(pSortInputBlk->pDataBlock, 2);
+  SColumnInfoData* lengthCol = taosArrayGet(pSortInputBlk->pDataBlock, 3);
 
+  for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) {
     int32_t pageId = -1;
-    void* pPage = getNewBufPage(pSortInfo->pExtSrcBlkBuf, &pageId);
-
-    int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t);
-    ASSERT(size <= getBufPageSize(pSortInfo->pExtSrcBlkBuf));
-
-    blockDataToBuf(pPage, p);
-    
-    setBufPageDirty(pPage, true);
-    releaseBufPage(pSortInfo->pExtSrcBlkBuf, pPage);
-
-    blockDataDestroy(p);
-    for (int32_t i = start; i <= stop; ++i) {
-      colDataSetInt32(blkIdCol, i, &pageId);
-      int32_t rowIdx = i - start;
-      colDataSetInt32(rowIdxCol, i, &rowIdx);
-    }
-    start = stop + 1;
+    int32_t offset = -1;
+    int32_t length = -1;
+    saveBlockRowToBuf(pInfo, pSrcBlock, i, &pageId, &offset, &length);
+    colDataSetInt32(pageIdCol, i, &pageId);
+    colDataSetInt32(pageIdCol, i, &offset);
+    colDataSetInt32(pageIdCol, i, &length);
   }
 
   pSortInputBlk->info.rows = nRows;
@@ -3267,50 +3326,38 @@ static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlo
   return 0;
 }
 
+void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
+  int32_t pageId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
+  int32_t offset = *(int32_t*)tsortGetValue(pTupleHandle, 2);
+  int32_t length = *(int32_t*)tsortGetValue(pTupleHandle, 2);
+  void* page = getBufPage(pInfo->tmsSortRowIdInfo.pExtSrcRowsBuf, pageId);
 
-static int32_t retrieveSourceBlock(STableMergeScanInfo* pInfo, int32_t blockId, SSDataBlock** ppBlock) {
   STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
 
-  void* pBlkHashVal  = tSimpleHashGet(pSortInfo->pBlkDataHash, &blockId, sizeof(blockId));
-  if (pBlkHashVal) {
-    *ppBlock = *(SSDataBlock**)pBlkHashVal;
-  }
-  else {
-    void* pPage = getBufPage(pSortInfo->pExtSrcBlkBuf, blockId);
-    SSDataBlock* pBlock = createOneDataBlock(pInfo->pReaderBlock, false);
-    blockDataFromBuf(pBlock, pPage);
-    releaseBufPage(pSortInfo->pExtSrcBlkBuf, pPage);
-
-    *ppBlock = pBlock;
-    tSimpleHashPut(pSortInfo->pBlkDataHash, &blockId, sizeof(blockId), &pBlock, sizeof(pBlock));
-  }
-  return 0;
-}
-
-void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
-  int32_t blkId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
-  int32_t rowIdx = *(int32_t*)tsortGetValue(pTupleHandle, 2);
-  SSDataBlock* pSrcBlk = NULL;
-  retrieveSourceBlock(pInfo, blkId, &pSrcBlk);
-  
-  for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
+  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+  char* buf = (char*)page + offset;
+  char* isNull = (char*)buf;
+  char* pStart = (char*)buf + sizeof(int8_t) * numOfCols;
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
-    SColumnInfoData* pSrcColInfo = taosArrayGet(pSrcBlk->pDataBlock, i);
 
-    bool isNull = colDataIsNull_s(pSrcColInfo, rowIdx);
-    if (isNull) {
-      colDataSetNULL(pColInfo, pBlock->info.rows);
-    } else {
-      char* pData = colDataGetData(pSrcColInfo, rowIdx);
-      if (pData != NULL) {
-        colDataSetVal(pColInfo, pBlock->info.rows, pData, false);
+    if (!isNull[i]) {
+      colDataSetVal(pColInfo, pBlock->info.rows, pStart, false);
+      if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) {
+        int32_t dataLen = getJsonValueLen(pStart);
+        pStart += dataLen;
+      } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
+        pStart += varDataTLen(pStart);
+      } else {
+        int32_t bytes = pColInfo->info.bytes;
+        pStart += bytes;
       }
+    } else {
+      colDataSetNULL(pColInfo, pBlock->info.rows);
     }
   }
-  if (rowIdx == pSrcBlk->info.rows - 1) {
-    tSimpleHashRemove(pInfo->tmsSortRowIdInfo.pBlkDataHash, &blkId, sizeof(blkId));
-    blockDataDestroy(pSrcBlk);
-  }
+  releaseBufPage(pInfo->tmsSortRowIdInfo.pExtSrcRowsBuf, page);
+
   pBlock->info.dataLoad = 1;
   pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag;
   pBlock->info.rows += 1;
@@ -3462,19 +3509,19 @@ void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo*
 
 int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
   STmsSortRowIdInfo* pSort = &pInfo->tmsSortRowIdInfo;
-  createDiskbasedBuf(&pSort->pExtSrcBlkBuf, pInfo->bufPageSize, pInfo->sortBufSize, "tms-ext-src-block", tsTempDir);
-  dBufSetPrintInfo(pSort->pExtSrcBlkBuf);
-  pSort->pBlkDataHash = tSimpleHashInit(2048, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
+  int32_t pageSize = getProperSortPageSize(blockDataGetRowSize(pInfo->pResBlock),
+                                                      taosArrayGetSize(pInfo->pResBlock->pDataBlock));
+  int32_t memSize = pageSize * 2048;
+  createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize, memSize, "tms-ext-src-block", tsTempDir);
+  dBufSetPrintInfo(pSort->pExtSrcRowsBuf);
   return 0;
 }
 
 int32_t stopRowIdSort(STableMergeScanInfo *pInfo) {
   STmsSortRowIdInfo* pSort = &pInfo->tmsSortRowIdInfo;
 
-  destroyDiskbasedBuf(pSort->pExtSrcBlkBuf);
-  pSort->pExtSrcBlkBuf = NULL;
-  tSimpleHashCleanup(pSort->pBlkDataHash);
-  pSort->pBlkDataHash = NULL;
+  destroyDiskbasedBuf(pSort->pExtSrcRowsBuf);
+  pSort->pExtSrcRowsBuf = NULL;
   return 0;
 }
 
@@ -3798,10 +3845,12 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     SSDataBlock* pSortInput = createDataBlock();
     SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1);
     blockDataAppendColInfo(pSortInput, &tsCol);
-    SColumnInfoData blkIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
-    blockDataAppendColInfo(pSortInput, &blkIdCol);
-    SColumnInfoData  rowIdxCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
-    blockDataAppendColInfo(pSortInput, &rowIdxCol);
+    SColumnInfoData pageIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
+    blockDataAppendColInfo(pSortInput, &pageIdCol);
+    SColumnInfoData  offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
+    blockDataAppendColInfo(pSortInput, &offsetCol);
+    SColumnInfoData  lengthCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
+    blockDataAppendColInfo(pSortInput, &lengthCol);
     pInfo->pSortInputBlock = pSortInput;
 
     SArray*         pList = taosArrayInit(1, sizeof(SBlockOrderInfo));
@@ -3823,8 +3872,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   }
   pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false);
 
-  int32_t  rowSize = pInfo->pResBlock->info.rowSize;
-  uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock);
+  int32_t  rowSize = pInfo->pSortInputBlock->info.rowSize;
+  uint32_t nCols = taosArrayGetSize(pInfo->pSortInputBlock->pDataBlock);
   pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols);
 
   pInfo->filesetDelimited = pTableScanNode->filesetDelimited;

From b309c251dab1bac953be4121be78ce5a6141447c Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 13 Dec 2023 13:21:26 +0800
Subject: [PATCH 011/201] fix: pass simple test

---
 source/libs/executor/src/scanoperator.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index be5cf298a4..90ab994e1c 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3317,8 +3317,8 @@ static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlo
     int32_t length = -1;
     saveBlockRowToBuf(pInfo, pSrcBlock, i, &pageId, &offset, &length);
     colDataSetInt32(pageIdCol, i, &pageId);
-    colDataSetInt32(pageIdCol, i, &offset);
-    colDataSetInt32(pageIdCol, i, &length);
+    colDataSetInt32(offsetCol, i, &offset);
+    colDataSetInt32(lengthCol, i, &length);
   }
 
   pSortInputBlk->info.rows = nRows;
@@ -3327,13 +3327,13 @@ static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlo
 }
 
 void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
+  STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
+
   int32_t pageId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
   int32_t offset = *(int32_t*)tsortGetValue(pTupleHandle, 2);
-  int32_t length = *(int32_t*)tsortGetValue(pTupleHandle, 2);
+  int32_t length = *(int32_t*)tsortGetValue(pTupleHandle, 3);
   void* page = getBufPage(pInfo->tmsSortRowIdInfo.pExtSrcRowsBuf, pageId);
 
-  STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
-
   int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   char* buf = (char*)page + offset;
   char* isNull = (char*)buf;
@@ -3849,7 +3849,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     blockDataAppendColInfo(pSortInput, &pageIdCol);
     SColumnInfoData  offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
     blockDataAppendColInfo(pSortInput, &offsetCol);
-    SColumnInfoData  lengthCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
+    SColumnInfoData  lengthCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 4);
     blockDataAppendColInfo(pSortInput, &lengthCol);
     pInfo->pSortInputBlock = pSortInput;
 

From 5be6ed95534e8dc7f7adcccff45ba67ef3e54acc Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Mon, 18 Dec 2023 14:23:30 +0800
Subject: [PATCH 012/201] fix: increase page size and mem size

---
 source/libs/executor/src/scanoperator.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 478504934c..4f26bc8910 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3566,8 +3566,8 @@ int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
   STmsSortRowIdInfo* pSort = &pInfo->tmsSortRowIdInfo;
   int32_t pageSize = getProperSortPageSize(blockDataGetRowSize(pInfo->pResBlock),
                                                       taosArrayGetSize(pInfo->pResBlock->pDataBlock));
-  int32_t memSize = pageSize * 2048;
-  createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize, memSize, "tms-ext-src-block", tsTempDir);
+  int32_t memSize = pageSize * 4 * 8192;
+  createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize * 4, memSize, "tms-ext-src-block", tsTempDir);
   dBufSetPrintInfo(pSort->pExtSrcRowsBuf);
   return 0;
 }

From 1aa94a8b3d77faa6f24694187deef8ef74809bd5 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 19 Dec 2023 14:55:31 +0800
Subject: [PATCH 013/201] fix: ci error

---
 source/libs/executor/inc/executorInt.h  |  1 +
 source/libs/executor/src/scanoperator.c | 22 ++++++++++++++++++----
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index 65444bfedd..e3eeab4e98 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -275,6 +275,7 @@ typedef struct STableScanInfo {
 
 typedef struct STmsSortRowIdInfo {
   SDiskbasedBuf* pExtSrcRowsBuf;
+  int32_t srcTsSlotId;
 } STmsSortRowIdInfo;
 
 typedef struct STableMergeScanInfo {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 4f26bc8910..0c24869921 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3278,8 +3278,13 @@ static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock
       memcpy(pStart, pData, dataLen);
       pStart += dataLen;
     } else if (IS_VAR_DATA_TYPE(pCol->info.type)) {
-      varDataCopy(pStart, pData);
-      pStart += varDataTLen(pData);
+      if (colDataGetLength(pCol, blockDataGetNumOfRows(pBlock)) != 0) {
+        varDataCopy(pStart, pData);
+        pStart += varDataTLen(pData);        
+      } else {
+        *(VarDataLenT*)(pStart) = 0;
+        pStart += VARSTR_HEADER_SIZE;
+      }
     } else {
       int32_t bytes = pCol->info.bytes;
       memcpy(pStart, pData, bytes);
@@ -3301,9 +3306,8 @@ static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlo
   pSortInputBlk->info.id = pSrcBlock->info.id;
   blockDataEnsureCapacity(pSortInputBlk, nRows);
 
-  int32_t tsSlotId = ((SBlockOrderInfo*)taosArrayGet(pInfo->pSortInfo, 0))->slotId;
   SColumnInfoData* tsCol = taosArrayGet(pSortInputBlk->pDataBlock, 0);
-  SColumnInfoData* pSrcTsCol = taosArrayGet(pSrcBlock->pDataBlock, tsSlotId);
+  SColumnInfoData* pSrcTsCol = taosArrayGet(pSrcBlock->pDataBlock, pSortInfo->srcTsSlotId);
   colDataAssign(tsCol, pSrcTsCol, nRows, &pSortInputBlk->info);
 
   SColumnInfoData* pageIdCol = taosArrayGet(pSortInputBlk->pDataBlock, 1);
@@ -3412,6 +3416,7 @@ static void doGetBlockForTableMergeScan(SOperatorInfo* pOperator, bool* pFinishe
 
   uint32_t status = 0;
   code = loadDataBlock(pOperator, &pInfo->base, pBlock, &status);
+
   if (code != TSDB_CODE_SUCCESS) {
     qInfo("table merge scan load datablock code %d, %s", code, GET_TASKID(pTaskInfo));
     T_LONG_JMP(pTaskInfo->env, code);
@@ -3918,6 +3923,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
     pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
   } else {
+    
     SSDataBlock* pSortInput = createDataBlock();
     SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1);
     blockDataAppendColInfo(pSortInput, &tsCol);
@@ -3929,6 +3935,14 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     blockDataAppendColInfo(pSortInput, &lengthCol);
     pInfo->pSortInputBlock = pSortInput;
 
+    int32_t srcTsSlotId = 0;
+    for (int32_t i = 0; i < taosArrayGetSize(pInfo->base.matchInfo.pList); ++i) {
+      SColMatchItem* colInfo = taosArrayGet(pInfo->base.matchInfo.pList, i);
+      if (colInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+        srcTsSlotId = colInfo->dstSlotId;
+      }
+    }
+    pInfo->tmsSortRowIdInfo.srcTsSlotId = srcTsSlotId;
     SArray*         pList = taosArrayInit(1, sizeof(SBlockOrderInfo));
     SBlockOrderInfo bi = {0};
     bi.order = pInfo->base.cond.order;

From f3847a00c65a7f44a1062a58d81261b63aa60501 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 20 Dec 2023 08:10:45 +0800
Subject: [PATCH 014/201] fix: more space is required for a row

---
 source/libs/executor/src/scanoperator.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 0c24869921..7a778624b2 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3222,7 +3222,7 @@ _error:
 
 static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
   SDiskbasedBuf* pResultBuf = pInfo->tmsSortRowIdInfo.pExtSrcRowsBuf;
-  int32_t rowBytes = blockDataGetRowSize(pBlock);
+  int32_t rowBytes = blockDataGetRowSize(pBlock) + taosArrayGetSize(pBlock->pDataBlock);
 
   SFilePage* pFilePage = NULL;
 

From bde2cf7b34f8121059c0fb1eee7c022dc29d5760 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 20 Dec 2023 09:54:27 +0800
Subject: [PATCH 015/201] enhance: remove length col and refactor

---
 source/libs/executor/inc/executorInt.h  |  2 +-
 source/libs/executor/src/scanoperator.c | 49 ++++++++++++++-----------
 2 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index e3eeab4e98..fa5ad6ff7c 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -312,7 +312,7 @@ typedef struct STableMergeScanInfo {
   bool             rtnNextDurationBlocks;
   int32_t          nextDurationBlocksIdx;
   bool             bSortRowId;
-  STmsSortRowIdInfo tmsSortRowIdInfo;
+  STmsSortRowIdInfo sortRowIdInfo;
 } STableMergeScanInfo;
 
 typedef struct STagScanFilterContext {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 7a778624b2..43cfdedd46 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3221,12 +3221,11 @@ _error:
 // ========================= table merge scan
 
 static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
-  SDiskbasedBuf* pResultBuf = pInfo->tmsSortRowIdInfo.pExtSrcRowsBuf;
+  SDiskbasedBuf* pResultBuf = pInfo->sortRowIdInfo.pExtSrcRowsBuf;
   int32_t rowBytes = blockDataGetRowSize(pBlock) + taosArrayGetSize(pBlock->pDataBlock);
 
   SFilePage* pFilePage = NULL;
 
-  // in the first scan, new space needed for results
   int32_t pageId = -1;
   SArray* list = getDataBufPagesIdList(pResultBuf);
 
@@ -3244,7 +3243,6 @@ static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock
     pageId = getPageId(pi);
 
     if (pFilePage->num + rowBytes > getBufPageSize(pResultBuf)) {
-      // release current page first, and prepare the next one
       releaseBufPageInfo(pResultBuf, pi);
 
       pFilePage = getNewBufPage(pResultBuf, &pageId);
@@ -3255,8 +3253,10 @@ static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock
   }
 
   if (pFilePage == NULL) {
-    return -1;
+    qError("failed to get buffer, code:%s", tstrerror(terrno));
+    return terrno;
   }
+
   *pPageId = pageId;
   *pOffset = pFilePage->num;
   char* buf = (char*)pFilePage + (*pOffset);
@@ -3298,8 +3298,8 @@ static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock
   return 0;
 }
 
-static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pSortInputBlk) {
-  STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
+static int32_t fillSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pSortInputBlk) {
+  STmsSortRowIdInfo* pSortInfo = &pInfo->sortRowIdInfo;
 
   int32_t nRows = pSrcBlock->info.rows;
   pSortInputBlk->info.window = pSrcBlock->info.window;
@@ -3321,7 +3321,6 @@ static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlo
     saveBlockRowToBuf(pInfo, pSrcBlock, i, &pageId, &offset, &length);
     colDataSetInt32(pageIdCol, i, &pageId);
     colDataSetInt32(offsetCol, i, &offset);
-    colDataSetInt32(lengthCol, i, &length);
   }
 
   pSortInputBlk->info.rows = nRows;
@@ -3329,13 +3328,12 @@ static int32_t transformIntoSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlo
   return 0;
 }
 
-void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
-  STmsSortRowIdInfo* pSortInfo = &pInfo->tmsSortRowIdInfo;
+static void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
+  STmsSortRowIdInfo* pSortInfo = &pInfo->sortRowIdInfo;
 
   int32_t pageId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
   int32_t offset = *(int32_t*)tsortGetValue(pTupleHandle, 2);
-  int32_t length = *(int32_t*)tsortGetValue(pTupleHandle, 3);
-  void* page = getBufPage(pInfo->tmsSortRowIdInfo.pExtSrcRowsBuf, pageId);
+  void* page = getBufPage(pInfo->sortRowIdInfo.pExtSrcRowsBuf, pageId);
 
   int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
   char* buf = (char*)page + offset;
@@ -3359,7 +3357,7 @@ void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBloc
       colDataSetNULL(pColInfo, pBlock->info.rows);
     }
   }
-  releaseBufPage(pInfo->tmsSortRowIdInfo.pExtSrcRowsBuf, page);
+  releaseBufPage(pInfo->sortRowIdInfo.pExtSrcRowsBuf, page);
 
   pBlock->info.dataLoad = 1;
   pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag;
@@ -3511,7 +3509,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
     SSDataBlock* pSortInputBlk = NULL;
     if (pInfo->bSortRowId) {
       blockDataCleanup(pInfo->pSortInputBlock);
-      transformIntoSortInputBlock(pInfo, pBlock, pInfo->pSortInputBlock);
+      fillSortInputBlock(pInfo, pBlock, pInfo->pSortInputBlock);
       pSortInputBlk = pInfo->pSortInputBlock;
     } else {
       pSortInputBlk = pBlock;
@@ -3568,17 +3566,17 @@ void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo*
 }
 
 int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
-  STmsSortRowIdInfo* pSort = &pInfo->tmsSortRowIdInfo;
+  STmsSortRowIdInfo* pSort = &pInfo->sortRowIdInfo;
   int32_t pageSize = getProperSortPageSize(blockDataGetRowSize(pInfo->pResBlock),
                                                       taosArrayGetSize(pInfo->pResBlock->pDataBlock));
-  int32_t memSize = pageSize * 4 * 8192;
-  createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize * 4, memSize, "tms-ext-src-block", tsTempDir);
+  int32_t memSize = pageSize * 1024;
+  int32_t code = createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize, memSize, "tms-ext-src-block", tsTempDir);
   dBufSetPrintInfo(pSort->pExtSrcRowsBuf);
-  return 0;
+  return code;
 }
 
 int32_t stopRowIdSort(STableMergeScanInfo *pInfo) {
-  STmsSortRowIdInfo* pSort = &pInfo->tmsSortRowIdInfo;
+  STmsSortRowIdInfo* pSort = &pInfo->sortRowIdInfo;
 
   destroyDiskbasedBuf(pSort->pExtSrcRowsBuf);
   pSort->pExtSrcRowsBuf = NULL;
@@ -3595,7 +3593,11 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
   pInfo->bNewFilesetEvent = false;
   pInfo->bNextDurationBlockEvent = false;
 
-  startRowIdSort(pInfo);
+  code = startRowIdSort(pInfo);
+  if (code != TSDB_CODE_SUCCESS) {
+    T_LONG_JMP(pTaskInfo->env, code);
+  }
+
   pInfo->sortBufSize = 2048 * pInfo->bufPageSize;
   int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
   pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
@@ -3814,6 +3816,11 @@ void destroyTableMergeScanOperatorInfo(void* param) {
   STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param;
   cleanupQueryTableDataCond(&pTableScanInfo->base.cond);
 
+  if (pTableScanInfo->sortRowIdInfo.pExtSrcRowsBuf != NULL) {
+    destroyDiskbasedBuf(pTableScanInfo->sortRowIdInfo.pExtSrcRowsBuf);
+    pTableScanInfo->sortRowIdInfo.pExtSrcRowsBuf = NULL;
+  }
+
   int32_t numOfTable = taosArrayGetSize(pTableScanInfo->sortSourceParams);
 
   pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader);
@@ -3931,8 +3938,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     blockDataAppendColInfo(pSortInput, &pageIdCol);
     SColumnInfoData  offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
     blockDataAppendColInfo(pSortInput, &offsetCol);
-    SColumnInfoData  lengthCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 4);
-    blockDataAppendColInfo(pSortInput, &lengthCol);
     pInfo->pSortInputBlock = pSortInput;
 
     int32_t srcTsSlotId = 0;
@@ -3942,7 +3947,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
         srcTsSlotId = colInfo->dstSlotId;
       }
     }
-    pInfo->tmsSortRowIdInfo.srcTsSlotId = srcTsSlotId;
+    pInfo->sortRowIdInfo.srcTsSlotId = srcTsSlotId;
     SArray*         pList = taosArrayInit(1, sizeof(SBlockOrderInfo));
     SBlockOrderInfo bi = {0};
     bi.order = pInfo->base.cond.order;

From 9ebca3eaff8e49abfaa554284571ea249f50f105 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 20 Dec 2023 10:06:22 +0800
Subject: [PATCH 016/201] fix: access length col removed

---
 source/libs/executor/src/scanoperator.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 43cfdedd46..44182a1e41 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3312,7 +3312,6 @@ static int32_t fillSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlock* pSrcB
 
   SColumnInfoData* pageIdCol = taosArrayGet(pSortInputBlk->pDataBlock, 1);
   SColumnInfoData* offsetCol = taosArrayGet(pSortInputBlk->pDataBlock, 2);
-  SColumnInfoData* lengthCol = taosArrayGet(pSortInputBlk->pDataBlock, 3);
 
   for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) {
     int32_t pageId = -1;

From d6445d2d5b707c3147270256372da6f5b14a37c3 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 20 Dec 2023 16:20:43 +0800
Subject: [PATCH 017/201] enhance: add length verification

---
 source/libs/executor/src/scanoperator.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 44182a1e41..308c318e04 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3222,7 +3222,7 @@ _error:
 
 static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
   SDiskbasedBuf* pResultBuf = pInfo->sortRowIdInfo.pExtSrcRowsBuf;
-  int32_t rowBytes = blockDataGetRowSize(pBlock) + taosArrayGetSize(pBlock->pDataBlock);
+  int32_t rowBytes = blockDataGetRowSize(pBlock) + taosArrayGetSize(pBlock->pDataBlock) + sizeof(int32_t);
 
   SFilePage* pFilePage = NULL;
 
@@ -3291,6 +3291,8 @@ static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock
       pStart += bytes;
     }
   }
+  *(int32_t*)pStart = (char*)pStart - (char*)buf;
+  pStart += sizeof(int32_t);
   *pLength = (int32_t)(pStart - (char*)buf);
   pFilePage->num += (*pLength);
   setBufPageDirty(pFilePage, true);
@@ -3356,6 +3358,11 @@ static void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock
       colDataSetNULL(pColInfo, pBlock->info.rows);
     }
   }
+
+  if (*(int32_t*)pStart != pStart-buf) {
+    qError("table merge scan row buf deserialization. length error %d != %d ", *(int32_t*)pStart, (int32_t)(pStart-buf));
+  };
+
   releaseBufPage(pInfo->sortRowIdInfo.pExtSrcRowsBuf, page);
 
   pBlock->info.dataLoad = 1;

From 084245e6d2ee9fcb643f51e702add85e83da50a3 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 21 Dec 2023 08:05:09 +0800
Subject: [PATCH 018/201] fix: row id sort when no limit and row size is more
 than 256

---
 source/libs/executor/src/scanoperator.c | 83 ++++++++++++++-----------
 1 file changed, 47 insertions(+), 36 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 44182a1e41..d0bde7f219 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3568,8 +3568,9 @@ int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
   STmsSortRowIdInfo* pSort = &pInfo->sortRowIdInfo;
   int32_t pageSize = getProperSortPageSize(blockDataGetRowSize(pInfo->pResBlock),
                                                       taosArrayGetSize(pInfo->pResBlock->pDataBlock));
-  int32_t memSize = pageSize * 1024;
-  int32_t code = createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize, memSize, "tms-ext-src-block", tsTempDir);
+  pageSize *= 2;                                                       
+  int32_t memSize = MIN(pageSize * 2048, 256 * 1024 * 1024);
+  int32_t code = createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize * 2, memSize, "tms-ext-src-block", tsTempDir);
   dBufSetPrintInfo(pSort->pExtSrcRowsBuf);
   return code;
 }
@@ -3863,6 +3864,34 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla
   return TSDB_CODE_SUCCESS;
 }
 
+static void initRowIdSortputBlock(STableMergeScanInfo* pInfo) {
+  SSDataBlock* pSortInput = createDataBlock();
+  SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1);
+  blockDataAppendColInfo(pSortInput, &tsCol);
+  SColumnInfoData pageIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
+  blockDataAppendColInfo(pSortInput, &pageIdCol);
+  SColumnInfoData  offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
+  blockDataAppendColInfo(pSortInput, &offsetCol);
+  pInfo->pSortInputBlock = pSortInput;
+
+  int32_t srcTsSlotId = 0;
+  for (int32_t i = 0; i < taosArrayGetSize(pInfo->base.matchInfo.pList); ++i) {
+    SColMatchItem* colInfo = taosArrayGet(pInfo->base.matchInfo.pList, i);
+    if (colInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+      srcTsSlotId = colInfo->dstSlotId;
+    }
+  }
+  pInfo->sortRowIdInfo.srcTsSlotId = srcTsSlotId;
+  SArray*         pList = taosArrayInit(1, sizeof(SBlockOrderInfo));
+  SBlockOrderInfo bi = {0};
+  bi.order = pInfo->base.cond.order;
+  bi.slotId = 0;
+  bi.nullFirst = NULL_ORDER_FIRST;
+  taosArrayPush(pList, &bi);
+  pInfo->pSortInfo = pList;
+  return;  
+}
+
 SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
                                                 STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
   STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo));
@@ -3922,39 +3951,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   initResultSizeInfo(&pOperator->resultInfo, 1024);
   pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
   blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
+  pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false);
 
-  pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
-  pInfo->bSortRowId = true;
-  if (!pInfo->bSortRowId) {
-    pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
-    pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
-  } else {
-    
-    SSDataBlock* pSortInput = createDataBlock();
-    SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1);
-    blockDataAppendColInfo(pSortInput, &tsCol);
-    SColumnInfoData pageIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
-    blockDataAppendColInfo(pSortInput, &pageIdCol);
-    SColumnInfoData  offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
-    blockDataAppendColInfo(pSortInput, &offsetCol);
-    pInfo->pSortInputBlock = pSortInput;
-
-    int32_t srcTsSlotId = 0;
-    for (int32_t i = 0; i < taosArrayGetSize(pInfo->base.matchInfo.pList); ++i) {
-      SColMatchItem* colInfo = taosArrayGet(pInfo->base.matchInfo.pList, i);
-      if (colInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        srcTsSlotId = colInfo->dstSlotId;
-      }
-    }
-    pInfo->sortRowIdInfo.srcTsSlotId = srcTsSlotId;
-    SArray*         pList = taosArrayInit(1, sizeof(SBlockOrderInfo));
-    SBlockOrderInfo bi = {0};
-    bi.order = pInfo->base.cond.order;
-    bi.slotId = 0;
-    bi.nullFirst = NULL_ORDER_FIRST;
-    taosArrayPush(pList, &bi);
-    pInfo->pSortInfo = pList;
-  }
   initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo);
   pInfo->mTableNumRows = tSimpleHashInit(1024,
                                          taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT));
@@ -3964,9 +3962,22 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     pInfo->mergeLimit = pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset;
     pInfo->mSkipTables = NULL;
   }
-  pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false);
 
-  int32_t  rowSize = pInfo->pSortInputBlock->info.rowSize;
+  if (!hasLimit && blockDataGetRowSize(pInfo->pResBlock) >= 256) {
+    pInfo->bSortRowId = true;
+  } else {
+    pInfo->bSortRowId = false;
+  }
+
+  if (!pInfo->bSortRowId) {
+    pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
+    pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
+  } else {
+    initRowIdSortputBlock(pInfo);
+  }
+
+  pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
+  int32_t  rowSize = blockDataGetRowSize(pInfo->pSortInputBlock);
   uint32_t nCols = taosArrayGetSize(pInfo->pSortInputBlock->pDataBlock);
   pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols);
   if (!tsExperimental) {

From 70419dcbc65f0be7fab754b63de0bc862d207fee Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 21 Dec 2023 08:22:22 +0800
Subject: [PATCH 019/201] fix: change memory size for row storage in memory
 pages

---
 source/libs/executor/src/scanoperator.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 39572648e9..b567d46a4a 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3575,8 +3575,9 @@ int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
   STmsSortRowIdInfo* pSort = &pInfo->sortRowIdInfo;
   int32_t pageSize = getProperSortPageSize(blockDataGetRowSize(pInfo->pResBlock),
                                                       taosArrayGetSize(pInfo->pResBlock->pDataBlock));
-  pageSize *= 2;                                                       
-  int32_t memSize = MIN(pageSize * 2048, 256 * 1024 * 1024);
+  pageSize *= 2;
+  int numOfTables = pInfo->tableEndIndex - pInfo->tableStartIndex + 1;                                                       
+  int32_t memSize = MIN(pageSize * numOfTables, 512 * 1024 * 1024);
   int32_t code = createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize * 2, memSize, "tms-ext-src-block", tsTempDir);
   dBufSetPrintInfo(pSort->pExtSrcRowsBuf);
   return code;

From 7a237504f95ec2950fa200cd83e9a0888aa7f721 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 21 Dec 2023 09:36:11 +0800
Subject: [PATCH 020/201] fix: change pagesize of row storage disk based buf

---
 source/libs/executor/src/scanoperator.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index b567d46a4a..2b95fa8e4d 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3578,7 +3578,7 @@ int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
   pageSize *= 2;
   int numOfTables = pInfo->tableEndIndex - pInfo->tableStartIndex + 1;                                                       
   int32_t memSize = MIN(pageSize * numOfTables, 512 * 1024 * 1024);
-  int32_t code = createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize * 2, memSize, "tms-ext-src-block", tsTempDir);
+  int32_t code = createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize, memSize, "tms-ext-src-block", tsTempDir);
   dBufSetPrintInfo(pSort->pExtSrcRowsBuf);
   return code;
 }

From 1725986d9fd3103e2051f5748207da2356684579 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 21 Dec 2023 11:45:04 +0800
Subject: [PATCH 021/201] fix: columns pre-allocated has no data and offsets

---
 source/libs/executor/src/scanoperator.c | 45 +++++++++++++++++++------
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 2b95fa8e4d..a44108dc50 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3220,10 +3220,7 @@ _error:
 
 // ========================= table merge scan
 
-static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
-  SDiskbasedBuf* pResultBuf = pInfo->sortRowIdInfo.pExtSrcRowsBuf;
-  int32_t rowBytes = blockDataGetRowSize(pBlock) + taosArrayGetSize(pBlock->pDataBlock) + sizeof(int32_t);
-
+static int32_t getPageFromExtSrcRowsBuf(SDiskbasedBuf* pResultBuf, int32_t rowBytes, int32_t* pPageId, SFilePage** ppFilePage) {
   SFilePage* pFilePage = NULL;
 
   int32_t pageId = -1;
@@ -3258,8 +3255,11 @@ static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock
   }
 
   *pPageId = pageId;
-  *pOffset = pFilePage->num;
-  char* buf = (char*)pFilePage + (*pOffset);
+  *ppFilePage = pFilePage;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) {
   size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
 
   char* isNull = (char*)buf;
@@ -3274,14 +3274,21 @@ static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock
     isNull[i] = 0;
     char* pData = colDataGetData(pCol, rowIdx);
     if (pCol->info.type == TSDB_DATA_TYPE_JSON) {
-      int32_t dataLen = getJsonValueLen(pData);
-      memcpy(pStart, pData, dataLen);
-      pStart += dataLen;
+      if (colDataGetLength(pCol, blockDataGetNumOfRows(pBlock)) != 0) {
+        int32_t dataLen = getJsonValueLen(pData);
+        memcpy(pStart, pData, dataLen);
+        pStart += dataLen;
+      } else {
+        // the column that is pre-allocated, has no data and has offset
+        *pStart = 0;
+        pStart += 1;
+      }
     } else if (IS_VAR_DATA_TYPE(pCol->info.type)) {
       if (colDataGetLength(pCol, blockDataGetNumOfRows(pBlock)) != 0) {
         varDataCopy(pStart, pData);
         pStart += varDataTLen(pData);        
       } else {
+        // the column that is pre-allocated, has no data and has offset
         *(VarDataLenT*)(pStart) = 0;
         pStart += VARSTR_HEADER_SIZE;
       }
@@ -3293,7 +3300,23 @@ static int32_t saveBlockRowToBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock
   }
   *(int32_t*)pStart = (char*)pStart - (char*)buf;
   pStart += sizeof(int32_t);
-  *pLength = (int32_t)(pStart - (char*)buf);
+  return (int32_t)(pStart - (char*)buf);
+}
+
+static int32_t saveBlockRowToExtRowsBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
+  SDiskbasedBuf* pResultBuf = pInfo->sortRowIdInfo.pExtSrcRowsBuf;
+  int32_t rowBytes = blockDataGetRowSize(pBlock) + taosArrayGetSize(pBlock->pDataBlock) + sizeof(int32_t);
+  int32_t pageId = -1;
+  SFilePage* pFilePage = NULL;
+  int32_t code = getPageFromExtSrcRowsBuf(pResultBuf, rowBytes, &pageId, &pFilePage);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
+  *pPageId = pageId;
+  *pOffset = pFilePage->num;
+  *pLength = blockRowToBuf(pBlock, rowIdx, (char*)pFilePage + (*pOffset));
+
   pFilePage->num += (*pLength);
   setBufPageDirty(pFilePage, true);
   releaseBufPage(pResultBuf, pFilePage);
@@ -3319,7 +3342,7 @@ static int32_t fillSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlock* pSrcB
     int32_t pageId = -1;
     int32_t offset = -1;
     int32_t length = -1;
-    saveBlockRowToBuf(pInfo, pSrcBlock, i, &pageId, &offset, &length);
+    saveBlockRowToExtRowsBuf(pInfo, pSrcBlock, i, &pageId, &offset, &length);
     colDataSetInt32(pageIdCol, i, &pageId);
     colDataSetInt32(offsetCol, i, &offset);
   }

From d2b0ef75a5798bc2bdbf7ea5fcdb107438e72131 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 21 Dec 2023 13:10:37 +0800
Subject: [PATCH 022/201] fix: windows compilation

---
 source/libs/executor/src/scanoperator.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index a44108dc50..d24b01db2c 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3600,7 +3600,7 @@ int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
                                                       taosArrayGetSize(pInfo->pResBlock->pDataBlock));
   pageSize *= 2;
   int numOfTables = pInfo->tableEndIndex - pInfo->tableStartIndex + 1;                                                       
-  int32_t memSize = MIN(pageSize * numOfTables, 512 * 1024 * 1024);
+  int32_t memSize = TMIN(pageSize * numOfTables, 512 * 1024 * 1024);
   int32_t code = createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize, memSize, "tms-ext-src-block", tsTempDir);
   dBufSetPrintInfo(pSort->pExtSrcRowsBuf);
   return code;

From a802d35af49f5ee7c5646607a4659b787f942dcd Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 21 Dec 2023 16:36:56 +0800
Subject: [PATCH 023/201] fix: change the pre-allocated column check

---
 source/libs/executor/src/scanoperator.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index d24b01db2c..b29d707cc9 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3274,21 +3274,21 @@ static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) {
     isNull[i] = 0;
     char* pData = colDataGetData(pCol, rowIdx);
     if (pCol->info.type == TSDB_DATA_TYPE_JSON) {
-      if (colDataGetLength(pCol, blockDataGetNumOfRows(pBlock)) != 0) {
+      if (pCol->pData) {
         int32_t dataLen = getJsonValueLen(pData);
         memcpy(pStart, pData, dataLen);
         pStart += dataLen;
       } else {
-        // the column that is pre-allocated, has no data and has offset
+        // the column that is pre-allocated has no data and has offset
         *pStart = 0;
         pStart += 1;
       }
     } else if (IS_VAR_DATA_TYPE(pCol->info.type)) {
-      if (colDataGetLength(pCol, blockDataGetNumOfRows(pBlock)) != 0) {
+      if (pCol->pData) {
         varDataCopy(pStart, pData);
         pStart += varDataTLen(pData);        
       } else {
-        // the column that is pre-allocated, has no data and has offset
+        // the column that is pre-allocated has no data and has offset
         *(VarDataLenT*)(pStart) = 0;
         pStart += VARSTR_HEADER_SIZE;
       }

From fbf67e42943913787b525efcbe3edf020217899e Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Wed, 3 Jan 2024 14:03:22 +0800
Subject: [PATCH 024/201] fix: save data when creating intial sources

---
 source/libs/executor/inc/executorInt.h  |   6 -
 source/libs/executor/inc/tsort.h        |   3 +
 source/libs/executor/src/scanoperator.c | 270 ++---------------------
 source/libs/executor/src/tsort.c        | 277 ++++++++++++++++++++++--
 4 files changed, 276 insertions(+), 280 deletions(-)

diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h
index 1ab27e42a0..5b014bef63 100644
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@@ -273,11 +273,6 @@ typedef struct STableScanInfo {
   bool            filesetDelimited;
 } STableScanInfo;
 
-typedef struct STmsSortRowIdInfo {
-  SDiskbasedBuf* pExtSrcRowsBuf;
-  int32_t srcTsSlotId;
-} STmsSortRowIdInfo;
-
 typedef struct STableMergeScanInfo {
   int32_t         tableStartIndex;
   int32_t         tableEndIndex;
@@ -312,7 +307,6 @@ typedef struct STableMergeScanInfo {
   bool             rtnNextDurationBlocks;
   int32_t          nextDurationBlocksIdx;
   bool             bSortRowId;
-  STmsSortRowIdInfo sortRowIdInfo;
 } STableMergeScanInfo;
 
 typedef struct STagScanFilterContext {
diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h
index 365acf2bff..1ed4d7baa5 100644
--- a/source/libs/executor/inc/tsort.h
+++ b/source/libs/executor/inc/tsort.h
@@ -194,6 +194,9 @@ void tsortSetClosed(SSortHandle* pHandle);
 void tsortSetSingleTableMerge(SSortHandle* pHandle);
 void tsortSetAbortCheckFn(SSortHandle* pHandle, bool (*checkFn)(void* param), void* param);
 
+int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsPageSize, int32_t extRowsSize);
+
+void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHandle* pTupleHandle);
 /**
  * @brief comp the tuple with keyBuf, if not equal, new keys will be built in keyBuf, newLen will be stored in keyLen
  * @param [in] pSortCols cols to comp and build
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index f690edcf07..36e810bd23 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3220,179 +3220,6 @@ _error:
 
 // ========================= table merge scan
 
-static int32_t getPageFromExtSrcRowsBuf(SDiskbasedBuf* pResultBuf, int32_t rowBytes, int32_t* pPageId, SFilePage** ppFilePage) {
-  SFilePage* pFilePage = NULL;
-
-  int32_t pageId = -1;
-  SArray* list = getDataBufPagesIdList(pResultBuf);
-
-  if (taosArrayGetSize(list) == 0) {
-    pFilePage = getNewBufPage(pResultBuf, &pageId);
-    pFilePage->num = sizeof(SFilePage);
-  } else {
-    SPageInfo* pi = getLastPageInfo(list);
-    pFilePage = getBufPage(pResultBuf, getPageId(pi));
-    if (pFilePage == NULL) {
-      qError("failed to get buffer, code:%s", tstrerror(terrno));
-      return terrno;
-    }
-
-    pageId = getPageId(pi);
-
-    if (pFilePage->num + rowBytes > getBufPageSize(pResultBuf)) {
-      releaseBufPageInfo(pResultBuf, pi);
-
-      pFilePage = getNewBufPage(pResultBuf, &pageId);
-      if (pFilePage != NULL) {
-        pFilePage->num = sizeof(SFilePage);
-      }
-    }
-  }
-
-  if (pFilePage == NULL) {
-    qError("failed to get buffer, code:%s", tstrerror(terrno));
-    return terrno;
-  }
-
-  *pPageId = pageId;
-  *ppFilePage = pFilePage;
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) {
-  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-
-  char* isNull = (char*)buf;
-  char* pStart = (char*)buf + sizeof(int8_t) * numOfCols;
-  for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
-    if (colDataIsNull_s(pCol, rowIdx)) {
-      isNull[i] = 1;
-      continue;
-    }
-
-    isNull[i] = 0;
-    char* pData = colDataGetData(pCol, rowIdx);
-    if (pCol->info.type == TSDB_DATA_TYPE_JSON) {
-      if (pCol->pData) {
-        int32_t dataLen = getJsonValueLen(pData);
-        memcpy(pStart, pData, dataLen);
-        pStart += dataLen;
-      } else {
-        // the column that is pre-allocated has no data and has offset
-        *pStart = 0;
-        pStart += 1;
-      }
-    } else if (IS_VAR_DATA_TYPE(pCol->info.type)) {
-      if (pCol->pData) {
-        varDataCopy(pStart, pData);
-        pStart += varDataTLen(pData);        
-      } else {
-        // the column that is pre-allocated has no data and has offset
-        *(VarDataLenT*)(pStart) = 0;
-        pStart += VARSTR_HEADER_SIZE;
-      }
-    } else {
-      int32_t bytes = pCol->info.bytes;
-      memcpy(pStart, pData, bytes);
-      pStart += bytes;
-    }
-  }
-  *(int32_t*)pStart = (char*)pStart - (char*)buf;
-  pStart += sizeof(int32_t);
-  return (int32_t)(pStart - (char*)buf);
-}
-
-static int32_t saveBlockRowToExtRowsBuf(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
-  SDiskbasedBuf* pResultBuf = pInfo->sortRowIdInfo.pExtSrcRowsBuf;
-  int32_t rowBytes = blockDataGetRowSize(pBlock) + taosArrayGetSize(pBlock->pDataBlock) + sizeof(int32_t);
-  int32_t pageId = -1;
-  SFilePage* pFilePage = NULL;
-  int32_t code = getPageFromExtSrcRowsBuf(pResultBuf, rowBytes, &pageId, &pFilePage);
-  if (code != TSDB_CODE_SUCCESS) {
-    return code;
-  }
-
-  *pPageId = pageId;
-  *pOffset = pFilePage->num;
-  *pLength = blockRowToBuf(pBlock, rowIdx, (char*)pFilePage + (*pOffset));
-
-  pFilePage->num += (*pLength);
-  setBufPageDirty(pFilePage, true);
-  releaseBufPage(pResultBuf, pFilePage);
-  return 0;
-}
-
-static int32_t fillSortInputBlock(STableMergeScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pSortInputBlk) {
-  STmsSortRowIdInfo* pSortInfo = &pInfo->sortRowIdInfo;
-
-  int32_t nRows = pSrcBlock->info.rows;
-  pSortInputBlk->info.window = pSrcBlock->info.window;
-  pSortInputBlk->info.id = pSrcBlock->info.id;
-  blockDataEnsureCapacity(pSortInputBlk, nRows);
-
-  SColumnInfoData* tsCol = taosArrayGet(pSortInputBlk->pDataBlock, 0);
-  SColumnInfoData* pSrcTsCol = taosArrayGet(pSrcBlock->pDataBlock, pSortInfo->srcTsSlotId);
-  colDataAssign(tsCol, pSrcTsCol, nRows, &pSortInputBlk->info);
-
-  SColumnInfoData* pageIdCol = taosArrayGet(pSortInputBlk->pDataBlock, 1);
-  SColumnInfoData* offsetCol = taosArrayGet(pSortInputBlk->pDataBlock, 2);
-
-  for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) {
-    int32_t pageId = -1;
-    int32_t offset = -1;
-    int32_t length = -1;
-    saveBlockRowToExtRowsBuf(pInfo, pSrcBlock, i, &pageId, &offset, &length);
-    colDataSetInt32(pageIdCol, i, &pageId);
-    colDataSetInt32(offsetCol, i, &offset);
-  }
-
-  pSortInputBlk->info.rows = nRows;
-
-  return 0;
-}
-
-static void appendOneRowIdRowToDataBlock(STableMergeScanInfo* pInfo, SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
-  STmsSortRowIdInfo* pSortInfo = &pInfo->sortRowIdInfo;
-
-  int32_t pageId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
-  int32_t offset = *(int32_t*)tsortGetValue(pTupleHandle, 2);
-  void* page = getBufPage(pInfo->sortRowIdInfo.pExtSrcRowsBuf, pageId);
-
-  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-  char* buf = (char*)page + offset;
-  char* isNull = (char*)buf;
-  char* pStart = (char*)buf + sizeof(int8_t) * numOfCols;
-  for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
-
-    if (!isNull[i]) {
-      colDataSetVal(pColInfo, pBlock->info.rows, pStart, false);
-      if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) {
-        int32_t dataLen = getJsonValueLen(pStart);
-        pStart += dataLen;
-      } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
-        pStart += varDataTLen(pStart);
-      } else {
-        int32_t bytes = pColInfo->info.bytes;
-        pStart += bytes;
-      }
-    } else {
-      colDataSetNULL(pColInfo, pBlock->info.rows);
-    }
-  }
-
-  if (*(int32_t*)pStart != pStart-buf) {
-    qError("table merge scan row buf deserialization. length error %d != %d ", *(int32_t*)pStart, (int32_t)(pStart-buf));
-  };
-
-  releaseBufPage(pInfo->sortRowIdInfo.pExtSrcRowsBuf, page);
-
-  pBlock->info.dataLoad = 1;
-  pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag;
-  pBlock->info.rows += 1;
-}
-
 static int32_t tableMergeScanDoSkipTable(STableMergeScanInfo* pInfo, SSDataBlock* pBlock) {
   int64_t nRows = 0;
   void*   pNum = tSimpleHashGet(pInfo->mTableNumRows, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid));
@@ -3535,18 +3362,9 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
 
     pOperator->resultInfo.totalRows += pBlock->info.rows;
 
-    SSDataBlock* pSortInputBlk = NULL;
-    if (pInfo->bSortRowId) {
-      blockDataCleanup(pInfo->pSortInputBlock);
-      fillSortInputBlock(pInfo, pBlock, pInfo->pSortInputBlock);
-      pSortInputBlk = pInfo->pSortInputBlock;
-    } else {
-      pSortInputBlk = pBlock;
-    }
-
     pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
     
-    return pSortInputBlk;
+    return pBlock;
   }
 
   return NULL;
@@ -3594,26 +3412,6 @@ void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo*
   return;
 }
 
-int32_t startRowIdSort(STableMergeScanInfo *pInfo) {
-  STmsSortRowIdInfo* pSort = &pInfo->sortRowIdInfo;
-  int32_t pageSize = getProperSortPageSize(blockDataGetRowSize(pInfo->pResBlock),
-                                                      taosArrayGetSize(pInfo->pResBlock->pDataBlock));
-  pageSize *= 2;
-  int numOfTables = pInfo->tableEndIndex - pInfo->tableStartIndex + 1;                                                       
-  int32_t memSize = TMIN(pageSize * numOfTables, 512 * 1024 * 1024);
-  int32_t code = createDiskbasedBuf(&pSort->pExtSrcRowsBuf, pageSize, memSize, "tms-ext-src-block", tsTempDir);
-  dBufSetPrintInfo(pSort->pExtSrcRowsBuf);
-  return code;
-}
-
-int32_t stopRowIdSort(STableMergeScanInfo *pInfo) {
-  STmsSortRowIdInfo* pSort = &pInfo->sortRowIdInfo;
-
-  destroyDiskbasedBuf(pSort->pExtSrcRowsBuf);
-  pSort->pExtSrcRowsBuf = NULL;
-  return 0;
-}
-
 int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
   STableMergeScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
@@ -3624,16 +3422,20 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
   pInfo->bNewFilesetEvent = false;
   pInfo->bNextDurationBlockEvent = false;
 
-  code = startRowIdSort(pInfo);
-  if (code != TSDB_CODE_SUCCESS) {
-    T_LONG_JMP(pTaskInfo->env, code);
-  }
-
   pInfo->sortBufSize = 2048 * pInfo->bufPageSize;
   int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
-  pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
-                                             pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
 
+  if (pInfo->bSortRowId) {
+    pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
+                                               pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
+    int32_t memSize = 512 * 1024 * 1024;
+    int32_t rowBytes = blockDataGetRowSize(pInfo->pResBlock) + taosArrayGetSize(pInfo->pResBlock->pDataBlock) + sizeof(int32_t);
+    int32_t pageSize = TMAX(memSize/numOfTable, rowBytes);
+    tsortSetSortByRowId(pInfo->pSortHandle, pageSize, memSize);
+  } else {
+    pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
+                                               pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
+  }
   tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit);
   tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo);
 
@@ -3670,7 +3472,6 @@ void stopDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
   tsortDestroySortHandle(pInfo->pSortHandle);
   pInfo->pSortHandle = NULL;
 
-  stopRowIdSort(pInfo);
 }
 
 int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
@@ -3757,11 +3558,7 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock*
       if (pTupleHandle == NULL) {
         break;
       }
-      if (!pInfo->bSortRowId) {
-        appendOneRowToDataBlock(pResBlock, pTupleHandle);
-      } else {
-        appendOneRowIdRowToDataBlock(pInfo, pResBlock, pTupleHandle);
-      }
+      tsortAppendTupleToBlock(pInfo->pSortHandle, pResBlock, pTupleHandle);
       if (pResBlock->info.rows >= capacity) {
         break;
       }
@@ -3847,11 +3644,6 @@ void destroyTableMergeScanOperatorInfo(void* param) {
   STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param;
   cleanupQueryTableDataCond(&pTableScanInfo->base.cond);
 
-  if (pTableScanInfo->sortRowIdInfo.pExtSrcRowsBuf != NULL) {
-    destroyDiskbasedBuf(pTableScanInfo->sortRowIdInfo.pExtSrcRowsBuf);
-    pTableScanInfo->sortRowIdInfo.pExtSrcRowsBuf = NULL;
-  }
-
   int32_t numOfTable = taosArrayGetSize(pTableScanInfo->sortSourceParams);
 
   pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader);
@@ -3895,34 +3687,6 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla
   return TSDB_CODE_SUCCESS;
 }
 
-static void initRowIdSortputBlock(STableMergeScanInfo* pInfo) {
-  SSDataBlock* pSortInput = createDataBlock();
-  SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1);
-  blockDataAppendColInfo(pSortInput, &tsCol);
-  SColumnInfoData pageIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
-  blockDataAppendColInfo(pSortInput, &pageIdCol);
-  SColumnInfoData  offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
-  blockDataAppendColInfo(pSortInput, &offsetCol);
-  pInfo->pSortInputBlock = pSortInput;
-
-  int32_t srcTsSlotId = 0;
-  for (int32_t i = 0; i < taosArrayGetSize(pInfo->base.matchInfo.pList); ++i) {
-    SColMatchItem* colInfo = taosArrayGet(pInfo->base.matchInfo.pList, i);
-    if (colInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-      srcTsSlotId = colInfo->dstSlotId;
-    }
-  }
-  pInfo->sortRowIdInfo.srcTsSlotId = srcTsSlotId;
-  SArray*         pList = taosArrayInit(1, sizeof(SBlockOrderInfo));
-  SBlockOrderInfo bi = {0};
-  bi.order = pInfo->base.cond.order;
-  bi.slotId = 0;
-  bi.nullFirst = NULL_ORDER_FIRST;
-  taosArrayPush(pList, &bi);
-  pInfo->pSortInfo = pList;
-  return;  
-}
-
 SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
                                                 STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
   STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo));
@@ -4000,12 +3764,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     pInfo->bSortRowId = false;
   }
 
-  if (!pInfo->bSortRowId) {
-    pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
-    pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
-  } else {
-    initRowIdSortputBlock(pInfo);
-  }
+  pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
+  pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
 
   pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
   int32_t  rowSize = blockDataGetRowSize(pInfo->pSortInputBlock);
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 205cd7d3ef..0b3a7c180e 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -75,6 +75,13 @@ struct SSortHandle {
 
   bool (*abortCheckFn)(void* param);
   void* abortCheckParam;
+
+  bool           bSortByRowId;
+  SDiskbasedBuf* pExtRowsBuf;
+  int32_t        extRowsPageSize;
+  int32_t        extRowsMemSize;
+  int32_t        srcTsSlotId;
+  SBlockOrderInfo extRowsOrderInfo;
 };
 
 void tsortSetSingleTableMerge(SSortHandle* pHandle) {
@@ -199,7 +206,7 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page
   pSortHandle->type = type;
   pSortHandle->pageSize = pageSize;
   pSortHandle->numOfPages = numOfPages;
-  pSortHandle->pSortInfo = pSortInfo;
+  pSortHandle->pSortInfo = taosArrayDup(pSortInfo, NULL);
   pSortHandle->loops = 0;
 
   pSortHandle->pqMaxTupleLength = pqMaxTupleLength;
@@ -303,6 +310,10 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) {
   
   taosArrayDestroy(pSortHandle->pOrderedSource);
   taosMemoryFreeClear(pSortHandle);
+  if (pSortHandle->pExtRowsBuf != NULL) {
+    destroyDiskbasedBuf(pSortHandle->pExtRowsBuf);
+  }
+  taosArrayDestroy(pSortHandle->pSortInfo);
 }
 
 int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) {
@@ -848,6 +859,228 @@ static int32_t createPageBuf(SSortHandle* pHandle) {
   return 0;
 }
 
+void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
+  if (pHandle->bSortByRowId) {
+    int32_t pageId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
+    int32_t offset = *(int32_t*)tsortGetValue(pTupleHandle, 2);
+    void*   page = getBufPage(pHandle->pExtRowsBuf, pageId);
+
+    int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+    char*   buf = (char*)page + offset;
+    char*   isNull = (char*)buf;
+    char*   pStart = (char*)buf + sizeof(int8_t) * numOfCols;
+    for (int32_t i = 0; i < numOfCols; ++i) {
+      SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
+
+      if (!isNull[i]) {
+        colDataSetVal(pColInfo, pBlock->info.rows, pStart, false);
+        if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) {
+          int32_t dataLen = getJsonValueLen(pStart);
+          pStart += dataLen;
+        } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
+          pStart += varDataTLen(pStart);
+        } else {
+          int32_t bytes = pColInfo->info.bytes;
+          pStart += bytes;
+        }
+      } else {
+        colDataSetNULL(pColInfo, pBlock->info.rows);
+      }
+    }
+
+    if (*(int32_t*)pStart != pStart - buf) {
+      qError("table merge scan row buf deserialization. length error %d != %d ", *(int32_t*)pStart,
+             (int32_t)(pStart - buf));
+    };
+
+    releaseBufPage(pHandle->pExtRowsBuf, page);
+
+    pBlock->info.dataLoad = 1;
+    pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag;
+    pBlock->info.rows += 1;
+  } else {
+    for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
+      SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
+      bool             isNull = tsortIsNullVal(pTupleHandle, i);
+      if (isNull) {
+        colDataSetNULL(pColInfo, pBlock->info.rows);
+      } else {
+        char* pData = tsortGetValue(pTupleHandle, i);
+        if (pData != NULL) {
+          colDataSetVal(pColInfo, pBlock->info.rows, pData, false);
+        }
+      }
+    }
+
+    pBlock->info.dataLoad = 1;
+    pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag;
+    pBlock->info.rows += 1;
+  }
+}
+
+static int32_t getPageFromExtSrcRowsBuf(SDiskbasedBuf* pResultBuf, int32_t rowBytes, int32_t* pPageId, SFilePage** ppFilePage) {
+  SFilePage* pFilePage = NULL;
+
+  int32_t pageId = -1;
+  SArray* list = getDataBufPagesIdList(pResultBuf);
+
+  if (taosArrayGetSize(list) == 0) {
+    pFilePage = getNewBufPage(pResultBuf, &pageId);
+    pFilePage->num = sizeof(SFilePage);
+  } else {
+    SPageInfo* pi = getLastPageInfo(list);
+    pFilePage = getBufPage(pResultBuf, getPageId(pi));
+    if (pFilePage == NULL) {
+      qError("failed to get buffer, code:%s", tstrerror(terrno));
+      return terrno;
+    }
+
+    pageId = getPageId(pi);
+
+    if (pFilePage->num + rowBytes > getBufPageSize(pResultBuf)) {
+      releaseBufPageInfo(pResultBuf, pi);
+
+      pFilePage = getNewBufPage(pResultBuf, &pageId);
+      if (pFilePage != NULL) {
+        pFilePage->num = sizeof(SFilePage);
+      }
+    }
+  }
+
+  if (pFilePage == NULL) {
+    qError("failed to get buffer, code:%s", tstrerror(terrno));
+    return terrno;
+  }
+
+  *pPageId = pageId;
+  *ppFilePage = pFilePage;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) {
+  size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+
+  char* isNull = (char*)buf;
+  char* pStart = (char*)buf + sizeof(int8_t) * numOfCols;
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
+    if (colDataIsNull_s(pCol, rowIdx)) {
+      isNull[i] = 1;
+      continue;
+    }
+
+    isNull[i] = 0;
+    char* pData = colDataGetData(pCol, rowIdx);
+    if (pCol->info.type == TSDB_DATA_TYPE_JSON) {
+      if (pCol->pData) {
+        int32_t dataLen = getJsonValueLen(pData);
+        memcpy(pStart, pData, dataLen);
+        pStart += dataLen;
+      } else {
+        // the column that is pre-allocated has no data and has offset
+        *pStart = 0;
+        pStart += 1;
+      }
+    } else if (IS_VAR_DATA_TYPE(pCol->info.type)) {
+      if (pCol->pData) {
+        varDataCopy(pStart, pData);
+        pStart += varDataTLen(pData);
+      } else {
+        // the column that is pre-allocated has no data and has offset
+        *(VarDataLenT*)(pStart) = 0;
+        pStart += VARSTR_HEADER_SIZE;
+      }
+    } else {
+      int32_t bytes = pCol->info.bytes;
+      memcpy(pStart, pData, bytes);
+      pStart += bytes;
+    }
+  }
+  *(int32_t*)pStart = (char*)pStart - (char*)buf;
+  pStart += sizeof(int32_t);
+  return (int32_t)(pStart - (char*)buf);
+}
+
+static int32_t saveBlockRowToExtRowsBuf(SSortHandle* pHandle, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
+  SDiskbasedBuf* pResultBuf = pHandle->pExtRowsBuf;
+  int32_t rowBytes = blockDataGetRowSize(pBlock) + taosArrayGetSize(pBlock->pDataBlock) + sizeof(int32_t);
+  int32_t pageId = -1;
+  SFilePage* pFilePage = NULL;
+  int32_t code = getPageFromExtSrcRowsBuf(pResultBuf, rowBytes, &pageId, &pFilePage);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
+  *pPageId = pageId;
+  *pOffset = pFilePage->num;
+  *pLength = blockRowToBuf(pBlock, rowIdx, (char*)pFilePage + (*pOffset));
+
+  pFilePage->num += (*pLength);
+  setBufPageDirty(pFilePage, true);
+  releaseBufPage(pResultBuf, pFilePage);
+  return 0;
+}
+
+
+static void appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSource, int32_t* rowIndex) {
+  int32_t pageId = -1;
+  int32_t offset = -1;
+  int32_t length = -1;
+  saveBlockRowToExtRowsBuf(pHandle, pSource, *rowIndex, &pageId, &offset, &length);
+
+  SSDataBlock* pBlock = pHandle->pDataBlock;
+  SColumnInfoData* pSrcTsCol = taosArrayGet(pSource->pDataBlock, pHandle->extRowsOrderInfo.slotId);
+  SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, 0);
+  char* pData = colDataGetData(pSrcTsCol, *rowIndex);
+  colDataSetVal(pTsCol, pBlock->info.rows, pData, false);
+
+  SColumnInfoData* pPageIdCol = taosArrayGet(pBlock->pDataBlock, 1);
+  colDataSetInt32(pPageIdCol, pBlock->info.rows, &pageId);
+
+  SColumnInfoData* pOffsetCol = taosArrayGet(pBlock->pDataBlock, 2);
+  colDataSetInt32(pOffsetCol, pBlock->info.rows, &offset);
+
+  pBlock->info.rows += 1;
+  *rowIndex += 1;
+}
+
+static void initRowIdSort(SSortHandle* pHandle) {
+  blockDataDestroy(pHandle->pDataBlock);
+
+  SSDataBlock* pSortInput = createDataBlock();
+  SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1);
+  blockDataAppendColInfo(pSortInput, &tsCol);
+  SColumnInfoData pageIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
+  blockDataAppendColInfo(pSortInput, &pageIdCol);
+  SColumnInfoData  offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
+  blockDataAppendColInfo(pSortInput, &offsetCol);
+  pHandle->pDataBlock = pSortInput;
+
+  SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);
+  SBlockOrderInfo bi = {0};
+  bi.order = pOrder->order;
+  bi.slotId = 0;
+  bi.nullFirst = NULL_ORDER_FIRST;
+
+  SArray*         aOrder = taosArrayInit(1, sizeof(SBlockOrderInfo));
+  taosArrayPush(aOrder, &bi);
+
+  taosArrayDestroy(pHandle->pSortInfo);
+  pHandle->pSortInfo = aOrder;
+  return;
+}
+
+int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsPageSize, int32_t extRowsMemSize) {
+  int32_t code = createDiskbasedBuf(&pHandle->pExtRowsBuf, extRowsPageSize, extRowsMemSize, "sort-ext-rows", tsTempDir);
+  pHandle->extRowsPageSize = extRowsPageSize;
+  pHandle->extRowsMemSize = extRowsMemSize;
+  SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);
+  pHandle->extRowsOrderInfo = *pOrder;
+  initRowIdSort(pHandle);
+  pHandle->bSortByRowId = true;
+  return code;
+}
+
 typedef struct SBlkMergeSupport {
   int64_t** aTs;
   int32_t* aRowIdx;
@@ -919,7 +1152,7 @@ static int32_t getPageBufIncForRow(SSDataBlock* blk, int32_t row, int32_t rowIdx
   return sz;
 }
 
-static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockOrderInfo* order, SArray* aExtSrc) {
+static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray* aExtSrc) {
   int32_t code = TSDB_CODE_SUCCESS;
   int pgHeaderSz = sizeof(int32_t) + sizeof(int32_t) * taosArrayGetSize(pHandle->pDataBlock->pDataBlock);
   int32_t rowCap = blockDataGetCapacityInRow(pHandle->pDataBlock, pHandle->pageSize, pgHeaderSz);
@@ -927,13 +1160,15 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO
   blockDataCleanup(pHandle->pDataBlock);
   int32_t numBlks = taosArrayGetSize(aBlk);
 
+  SBlockOrderInfo* pOrigBlockOrder = (!pHandle->bSortByRowId) ? taosArrayGet(pHandle->pSortInfo, 0) : &pHandle->extRowsOrderInfo;
+  SBlockOrderInfo* pHandleBlockOrder = taosArrayGet(pHandle->pSortInfo, 0);
   SBlkMergeSupport sup;
   sup.aRowIdx = taosMemoryCalloc(numBlks, sizeof(int32_t));
   sup.aTs = taosMemoryCalloc(numBlks, sizeof(int64_t*));
-  sup.order = order->order;
+  sup.order = pOrigBlockOrder->order;
   for (int i = 0; i < numBlks; ++i) {
     SSDataBlock* blk = taosArrayGetP(aBlk, i);
-    SColumnInfoData* col = taosArrayGet(blk->pDataBlock, order->slotId);
+    SColumnInfoData* col = taosArrayGet(blk->pDataBlock, pOrigBlockOrder->slotId);
     sup.aTs[i] = (int64_t*)col->pData;
     sup.aRowIdx[i] = 0;
   }
@@ -958,8 +1193,8 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO
   int32_t nMergedRows = 0;
   bool mergeLimitReached = false;
   size_t blkPgSz = pgHeaderSz;
-  int64_t lastPageBufTs = (order->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN;
-  int64_t currTs = (order->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN;
+  int64_t lastPageBufTs = (pHandleBlockOrder->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN;
+  int64_t currTs = (pHandleBlockOrder->order == TSDB_ORDER_ASC) ? INT64_MAX : INT64_MIN;
   while (nRows < totalRows) {
     int32_t minIdx = tMergeTreeGetChosenIndex(pTree);
     SSDataBlock* minBlk = taosArrayGetP(aBlk, minIdx);
@@ -967,7 +1202,7 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO
     int32_t bufInc = getPageBufIncForRow(minBlk, minRow, pHandle->pDataBlock->info.rows);
 
     if (blkPgSz <= pHandle->pageSize && blkPgSz + bufInc > pHandle->pageSize) {
-        SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId);
+        SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, pHandleBlockOrder->slotId);
         lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1];
         appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId);
         nMergedRows += pHandle->pDataBlock->info.rows;
@@ -977,15 +1212,19 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO
         
         if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) {
           mergeLimitReached = true;
-          if ((lastPageBufTs < pHandle->currMergeLimitTs && order->order == TSDB_ORDER_ASC) ||
-              (lastPageBufTs > pHandle->currMergeLimitTs && order->order == TSDB_ORDER_DESC)) {
+          if ((lastPageBufTs < pHandle->currMergeLimitTs && pHandleBlockOrder->order == TSDB_ORDER_ASC) ||
+              (lastPageBufTs > pHandle->currMergeLimitTs && pHandleBlockOrder->order == TSDB_ORDER_DESC)) {
                 pHandle->currMergeLimitTs = lastPageBufTs;
           }
           break;
         }        
     }
     blockDataEnsureCapacity(pHandle->pDataBlock, pHandle->pDataBlock->info.rows + 1);
-    appendOneRowToDataBlock(pHandle->pDataBlock, minBlk, &minRow);
+    if (!pHandle->bSortByRowId) {
+        appendOneRowToDataBlock(pHandle->pDataBlock, minBlk, &minRow);
+    } else {
+        appendToRowIndexDataBlock(pHandle, minBlk, &minRow);
+    }
     blkPgSz += bufInc;
 
     ++nRows;
@@ -999,14 +1238,14 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO
   }
   if (pHandle->pDataBlock->info.rows > 0) {
     if (!mergeLimitReached) {
-      SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, order->slotId);
+      SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, pHandleBlockOrder->slotId);
       lastPageBufTs = ((int64_t*)tsCol->pData)[pHandle->pDataBlock->info.rows - 1];
       appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId);
       nMergedRows += pHandle->pDataBlock->info.rows;
       if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) {
           mergeLimitReached = true;
-          if ((lastPageBufTs < pHandle->currMergeLimitTs && order->order == TSDB_ORDER_ASC) ||
-              (lastPageBufTs > pHandle->currMergeLimitTs && order->order == TSDB_ORDER_DESC)) {
+          if ((lastPageBufTs < pHandle->currMergeLimitTs && pHandleBlockOrder->order == TSDB_ORDER_ASC) ||
+              (lastPageBufTs > pHandle->currMergeLimitTs && pHandleBlockOrder->order == TSDB_ORDER_DESC)) {
                 pHandle->currMergeLimitTs = lastPageBufTs;
           }
       }
@@ -1025,7 +1264,6 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO
 }
 
 static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
-  SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);
   size_t           nSrc = taosArrayGetSize(pHandle->pOrderedSource);
   SArray*          aExtSrc = taosArrayInit(nSrc, POINTER_BYTES);
 
@@ -1040,7 +1278,8 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   SSortSource* pSrc = taosArrayGetP(pHandle->pOrderedSource, 0);
   int32_t      szSort = 0;
 
-  if (pOrder->order == TSDB_ORDER_ASC) {
+  SBlockOrderInfo* pOrigOrder = (!pHandle->bSortByRowId) ? taosArrayGet(pHandle->pSortInfo, 0) : &pHandle->extRowsOrderInfo;
+  if (pOrigOrder->order == TSDB_ORDER_ASC) {
     pHandle->currMergeLimitTs = INT64_MAX;
   } else {
     pHandle->currMergeLimitTs = INT64_MIN;
@@ -1051,10 +1290,10 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   while (1) {
     SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param);
     if (pBlk != NULL) {
-      SColumnInfoData* tsCol = taosArrayGet(pBlk->pDataBlock, pOrder->slotId);
+      SColumnInfoData* tsCol = taosArrayGet(pBlk->pDataBlock, pOrigOrder->slotId);
       int64_t firstRowTs = *(int64_t*)tsCol->pData;
-      if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs)  ||
-          (pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) {
+      if ((pOrigOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs)  ||
+          (pOrigOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) {
             continue;
           }
     }
@@ -1076,7 +1315,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
       tSimpleHashClear(mUidBlk);
 
       int64_t p = taosGetTimestampUs();
-      code = sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc);
+      code = sortBlocksToExtSource(pHandle, aBlkSort, aExtSrc);
       if (code != TSDB_CODE_SUCCESS) {
         tSimpleHashCleanup(mUidBlk);
         taosArrayDestroy(aBlkSort);

From c9b2bb714d77470fe81a5ee31fd2a2b0faa0163a Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 3 Jan 2024 14:39:06 +0800
Subject: [PATCH 025/201] fix: bugs during self-test

---
 source/libs/executor/src/tsort.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 0b3a7c180e..714b268008 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -309,11 +309,11 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) {
   qDebug("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr);
   
   taosArrayDestroy(pSortHandle->pOrderedSource);
-  taosMemoryFreeClear(pSortHandle);
   if (pSortHandle->pExtRowsBuf != NULL) {
     destroyDiskbasedBuf(pSortHandle->pExtRowsBuf);
   }
-  taosArrayDestroy(pSortHandle->pSortInfo);
+  taosArrayDestroy(pSortHandle->pSortInfo);  
+  taosMemoryFreeClear(pSortHandle);
 }
 
 int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) {
@@ -1072,6 +1072,7 @@ static void initRowIdSort(SSortHandle* pHandle) {
 
 int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsPageSize, int32_t extRowsMemSize) {
   int32_t code = createDiskbasedBuf(&pHandle->pExtRowsBuf, extRowsPageSize, extRowsMemSize, "sort-ext-rows", tsTempDir);
+  dBufSetPrintInfo(pHandle->pExtRowsBuf);
   pHandle->extRowsPageSize = extRowsPageSize;
   pHandle->extRowsMemSize = extRowsMemSize;
   SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);
@@ -1267,7 +1268,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   size_t           nSrc = taosArrayGetSize(pHandle->pOrderedSource);
   SArray*          aExtSrc = taosArrayInit(nSrc, POINTER_BYTES);
 
-  size_t maxBufSize = pHandle->numOfPages * pHandle->pageSize;
+  size_t maxBufSize = (pHandle->bSortByRowId) ? pHandle->extRowsMemSize : (pHandle->numOfPages * pHandle->pageSize);
 
   int32_t code = createPageBuf(pHandle);
   if (code != TSDB_CODE_SUCCESS) {

From fb732be16dc9b48db8a969f9a04b194ed212d546 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 3 Jan 2024 15:09:24 +0800
Subject: [PATCH 026/201] fix: change pageSize and buf size

---
 source/libs/executor/src/tsort.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 714b268008..59146f85a5 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1045,7 +1045,6 @@ static void appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSource
 }
 
 static void initRowIdSort(SSortHandle* pHandle) {
-  blockDataDestroy(pHandle->pDataBlock);
 
   SSDataBlock* pSortInput = createDataBlock();
   SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1);
@@ -1054,7 +1053,14 @@ static void initRowIdSort(SSortHandle* pHandle) {
   blockDataAppendColInfo(pSortInput, &pageIdCol);
   SColumnInfoData  offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
   blockDataAppendColInfo(pSortInput, &offsetCol);
+
+  blockDataDestroy(pHandle->pDataBlock);
   pHandle->pDataBlock = pSortInput;
+  
+  int32_t  rowSize = blockDataGetRowSize(pHandle->pDataBlock);
+  size_t nCols = taosArrayGetSize(pHandle->pDataBlock->pDataBlock);
+  pHandle->pageSize = getProperSortPageSize(rowSize, nCols);
+  pHandle->numOfPages = 2048;
 
   SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);
   SBlockOrderInfo bi = {0};

From 06fd7ee96ff9e868f700127a76eba4fddf4b1dec Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 3 Jan 2024 15:34:38 +0800
Subject: [PATCH 027/201] fix: use the correct block to calc page size

---
 source/libs/executor/src/tsort.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 59146f85a5..21c9aeeff3 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1056,7 +1056,7 @@ static void initRowIdSort(SSortHandle* pHandle) {
 
   blockDataDestroy(pHandle->pDataBlock);
   pHandle->pDataBlock = pSortInput;
-  
+
   int32_t  rowSize = blockDataGetRowSize(pHandle->pDataBlock);
   size_t nCols = taosArrayGetSize(pHandle->pDataBlock->pDataBlock);
   pHandle->pageSize = getProperSortPageSize(rowSize, nCols);
@@ -1206,7 +1206,8 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray*
     int32_t minIdx = tMergeTreeGetChosenIndex(pTree);
     SSDataBlock* minBlk = taosArrayGetP(aBlk, minIdx);
     int32_t minRow = sup.aRowIdx[minIdx];
-    int32_t bufInc = getPageBufIncForRow(minBlk, minRow, pHandle->pDataBlock->info.rows);
+    SSDataBlock* incBlock = (pHandle->bSortByRowId) ? pHandle->pDataBlock : minBlk;
+    int32_t bufInc = getPageBufIncForRow(incBlock, minRow, pHandle->pDataBlock->info.rows);
 
     if (blkPgSz <= pHandle->pageSize && blkPgSz + bufInc > pHandle->pageSize) {
         SColumnInfoData* tsCol = taosArrayGet(pHandle->pDataBlock->pDataBlock, pHandleBlockOrder->slotId);
@@ -1215,7 +1216,8 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray*
         nMergedRows += pHandle->pDataBlock->info.rows;
         blockDataCleanup(pHandle->pDataBlock);
         blkPgSz = pgHeaderSz;
-        bufInc = getPageBufIncForRow(minBlk, minRow, 0);
+        incBlock = (pHandle->bSortByRowId) ? pHandle->pDataBlock : minBlk;
+        bufInc = getPageBufIncForRow(incBlock, minRow, 0);
         
         if ((pHandle->mergeLimit != -1) && (nMergedRows >= pHandle->mergeLimit)) {
           mergeLimitReached = true;

From 0f9be1033ac8fcbaafc4b854cfa3785a77f22b52 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 5 Jan 2024 19:22:17 +0800
Subject: [PATCH 028/201] fix: use page size of sort

---
 source/libs/executor/src/scanoperator.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 36e810bd23..b3e6c64953 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3429,9 +3429,9 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
     pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
                                                pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
     int32_t memSize = 512 * 1024 * 1024;
-    int32_t rowBytes = blockDataGetRowSize(pInfo->pResBlock) + taosArrayGetSize(pInfo->pResBlock->pDataBlock) + sizeof(int32_t);
-    int32_t pageSize = TMAX(memSize/numOfTable, rowBytes);
-    tsortSetSortByRowId(pInfo->pSortHandle, pageSize, memSize);
+    // int32_t rowBytes = blockDataGetRowSize(pInfo->pResBlock) + taosArrayGetSize(pInfo->pResBlock->pDataBlock) + sizeof(int32_t);
+    // int32_t pageSize = TMAX(memSize/numOfTable, rowBytes);
+    tsortSetSortByRowId(pInfo->pSortHandle, pInfo->bufPageSize, memSize);
   } else {
     pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
                                                pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);

From 7c58a9a280a1df92ab1361b814d1a049554c09f7 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Mon, 8 Jan 2024 08:21:27 +0800
Subject: [PATCH 029/201] fix: disable sort by row id when num of tables is 1

---
 source/libs/executor/src/scanoperator.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index b3e6c64953..76d63e8896 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3425,7 +3425,7 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
   pInfo->sortBufSize = 2048 * pInfo->bufPageSize;
   int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
 
-  if (pInfo->bSortRowId) {
+  if (pInfo->bSortRowId && numOfTable != 1) {
     pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
                                                pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
     int32_t memSize = 512 * 1024 * 1024;

From b27bf263af954db61f835a998a3620f3fe284a15 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 30 Jan 2024 10:26:24 +0800
Subject: [PATCH 030/201] fix: calcualte row bytes once

---
 source/libs/executor/src/tsort.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 45dfa2e026..4e5cc37d99 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -78,6 +78,7 @@ struct SSortHandle {
 
   bool           bSortByRowId;
   SDiskbasedBuf* pExtRowsBuf;
+  int32_t        extRowBytes;
   int32_t        extRowsPageSize;
   int32_t        extRowsMemSize;
   int32_t        srcTsSlotId;
@@ -1006,7 +1007,7 @@ static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) {
 
 static int32_t saveBlockRowToExtRowsBuf(SSortHandle* pHandle, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
   SDiskbasedBuf* pResultBuf = pHandle->pExtRowsBuf;
-  int32_t rowBytes = blockDataGetRowSize(pBlock) + taosArrayGetSize(pBlock->pDataBlock) + sizeof(int32_t);
+  int32_t rowBytes = pHandle->extRowBytes;
   int32_t pageId = -1;
   SFilePage* pFilePage = NULL;
   int32_t code = getPageFromExtSrcRowsBuf(pResultBuf, rowBytes, &pageId, &pFilePage);
@@ -1082,6 +1083,7 @@ static void initRowIdSort(SSortHandle* pHandle) {
 int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsPageSize, int32_t extRowsMemSize) {
   int32_t code = createDiskbasedBuf(&pHandle->pExtRowsBuf, extRowsPageSize, extRowsMemSize, "sort-ext-rows", tsTempDir);
   dBufSetPrintInfo(pHandle->pExtRowsBuf);
+  pHandle->extRowBytes = blockDataGetRowSize(pHandle->pDataBlock) + taosArrayGetSize(pHandle->pDataBlock->pDataBlock) + sizeof(int32_t);
   pHandle->extRowsPageSize = extRowsPageSize;
   pHandle->extRowsMemSize = extRowsMemSize;
   SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);

From 3facbd43dbf961b673b804d9e8132615424c5dbd Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 31 Jan 2024 13:10:42 +0800
Subject: [PATCH 031/201] feat: first commit to optimize out tpagedbuf

---
 source/libs/executor/src/tsort.c | 64 ++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 4e5cc37d99..2a2d8cf2fc 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -32,6 +32,34 @@ struct STupleHandle {
   int32_t      rowIndex;
 };
 
+typedef struct SSortMemPageEntry {
+  int32_t pageId;
+  bool  active;
+
+  void* data;
+
+  struct SSortMemPageEntry* next;
+  struct SSortMemPageEntry* prev;
+
+} SSortMemPageEntry;
+
+typedef struct SSortMemFile {
+  int32_t pageSize;
+  int32_t cacheSize;
+  char* writeBuf;
+  
+  int32_t currPageId;
+  int32_t currPageOffset;
+  bool bDirty;
+
+  int32_t totalMemPages;
+  SSortMemPageEntry* memPages;
+  int32_t numMemPages;
+  SSHashObj* mActivePages;
+
+  TdFilePtr pBufFile;
+} SSortMemFile;
+
 struct SSortHandle {
   int32_t        type;
   int32_t        pageSize;
@@ -78,6 +106,7 @@ struct SSortHandle {
 
   bool           bSortByRowId;
   SDiskbasedBuf* pExtRowsBuf;
+  SSortMemFile* pExtRowsMemFile;
   int32_t        extRowBytes;
   int32_t        extRowsPageSize;
   int32_t        extRowsMemSize;
@@ -1025,6 +1054,41 @@ static int32_t saveBlockRowToExtRowsBuf(SSortHandle* pHandle, SSDataBlock* pBloc
   return 0;
 }
 
+// pageId * pageSize == pageStartOffset in file. write in pages
+// when pass the page boundaries, the page is move to the front(old). 
+// find hash from pageid to page entry. if the page can not be found, 
+//   1) nuse inactive pages, 2) then new pages if not exceeding mem limit, 3) then active pages
+// new pages is added or moved to the back.
+
+static int32_t createSortMemFile(SSortHandle* pHandle) {
+  if (pHandle->pExtRowsMemFile != NULL) {
+    return TSDB_CODE_SUCCESS;
+  }
+  SSortMemFile* pMemFile = taosMemoryCalloc(1, sizeof(SSortMemFile));
+  pMemFile->pBufFile =
+      taosOpenFile(pBuf->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);  
+  return TSDB_CODE_SUCCESS;
+} 
+
+static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
+  SDiskbasedBuf* pResultBuf = pHandle->pExtRowsBuf;
+  int32_t rowBytes = pHandle->extRowBytes;
+  int32_t pageId = -1;
+  SFilePage* pFilePage = NULL;
+  int32_t code = getPageFromExtSrcRowsBuf(pResultBuf, rowBytes, &pageId, &pFilePage);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
+  *pPageId = pageId;
+  *pOffset = pFilePage->num;
+  *pLength = blockRowToBuf(pBlock, rowIdx, (char*)pFilePage + (*pOffset));
+
+  pFilePage->num += (*pLength);
+  setBufPageDirty(pFilePage, true);
+  releaseBufPage(pResultBuf, pFilePage);
+  return 0;
+}
 
 static void appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSource, int32_t* rowIndex) {
   int32_t pageId = -1;

From 539736cfc935c9bfbdf573ccfba01d1f9510c005 Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Sun, 4 Feb 2024 16:30:34 +0800
Subject: [PATCH 032/201] fix: first version of mem file=

---
 source/libs/executor/src/tsort.c | 238 +++++++++++++++++++------------
 1 file changed, 150 insertions(+), 88 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 2a2d8cf2fc..c01612675a 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -46,18 +46,20 @@ typedef struct SSortMemPageEntry {
 typedef struct SSortMemFile {
   int32_t pageSize;
   int32_t cacheSize;
-  char* writeBuf;
+  char*   pageBuf;
   
   int32_t currPageId;
   int32_t currPageOffset;
   bool bDirty;
 
   int32_t totalMemPages;
-  SSortMemPageEntry* memPages;
+  SSortMemPageEntry* pagesHead;
+  SSortMemPageEntry* pagesTail;
   int32_t numMemPages;
   SSHashObj* mActivePages;
 
-  TdFilePtr pBufFile;
+  TdFilePtr pTdFile;
+  char memFilePath[PATH_MAX];
 } SSortMemFile;
 
 struct SSortHandle {
@@ -105,7 +107,6 @@ struct SSortHandle {
   void* abortCheckParam;
 
   bool           bSortByRowId;
-  SDiskbasedBuf* pExtRowsBuf;
   SSortMemFile* pExtRowsMemFile;
   int32_t        extRowBytes;
   int32_t        extRowsPageSize;
@@ -117,6 +118,11 @@ struct SSortHandle {
   void* mergeLimitReachedParam;
 };
 
+static int32_t destroySortMemFile(SSortHandle* pHandle);
+static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char** ppPage);
+static void    setExtMemFilePageUnused(SSortMemFile* pMemFile, int32_t pageId);
+static int32_t saveLastPageToExtRowsMemFile(SSortHandle* pHandle);
+
 void tsortSetSingleTableMerge(SSortHandle* pHandle) {
   pHandle->singleTableMerge = true;
 }
@@ -342,8 +348,8 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) {
   qDebug("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr);
   
   taosArrayDestroy(pSortHandle->pOrderedSource);
-  if (pSortHandle->pExtRowsBuf != NULL) {
-    destroyDiskbasedBuf(pSortHandle->pExtRowsBuf);
+  if (pSortHandle->pExtRowsMemFile != NULL) {
+    destroySortMemFile(pSortHandle);
   }
   taosArrayDestroy(pSortHandle->pSortInfo);  
   taosMemoryFreeClear(pSortHandle);
@@ -896,7 +902,9 @@ void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHa
   if (pHandle->bSortByRowId) {
     int32_t pageId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
     int32_t offset = *(int32_t*)tsortGetValue(pTupleHandle, 2);
-    void*   page = getBufPage(pHandle->pExtRowsBuf, pageId);
+
+    char* page = NULL;
+    getPageFromExtMemFile(pHandle, pageId, &page);
 
     int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
     char*   buf = (char*)page + offset;
@@ -926,11 +934,13 @@ void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHa
              (int32_t)(pStart - buf));
     };
 
-    releaseBufPage(pHandle->pExtRowsBuf, page);
-
     pBlock->info.dataLoad = 1;
     pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag;
     pBlock->info.rows += 1;
+
+    if (offset + pHandle->extRowBytes >= pHandle->pExtRowsMemFile->pageSize) {
+      setExtMemFilePageUnused(pHandle->pExtRowsMemFile, pageId);
+    }
   } else {
     for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
       SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
@@ -951,45 +961,6 @@ void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHa
   }
 }
 
-static int32_t getPageFromExtSrcRowsBuf(SDiskbasedBuf* pResultBuf, int32_t rowBytes, int32_t* pPageId, SFilePage** ppFilePage) {
-  SFilePage* pFilePage = NULL;
-
-  int32_t pageId = -1;
-  SArray* list = getDataBufPagesIdList(pResultBuf);
-
-  if (taosArrayGetSize(list) == 0) {
-    pFilePage = getNewBufPage(pResultBuf, &pageId);
-    pFilePage->num = sizeof(SFilePage);
-  } else {
-    SPageInfo* pi = getLastPageInfo(list);
-    pFilePage = getBufPage(pResultBuf, getPageId(pi));
-    if (pFilePage == NULL) {
-      qError("failed to get buffer, code:%s", tstrerror(terrno));
-      return terrno;
-    }
-
-    pageId = getPageId(pi);
-
-    if (pFilePage->num + rowBytes > getBufPageSize(pResultBuf)) {
-      releaseBufPageInfo(pResultBuf, pi);
-
-      pFilePage = getNewBufPage(pResultBuf, &pageId);
-      if (pFilePage != NULL) {
-        pFilePage->num = sizeof(SFilePage);
-      }
-    }
-  }
-
-  if (pFilePage == NULL) {
-    qError("failed to get buffer, code:%s", tstrerror(terrno));
-    return terrno;
-  }
-
-  *pPageId = pageId;
-  *ppFilePage = pFilePage;
-  return TSDB_CODE_SUCCESS;
-}
-
 static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) {
   size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
 
@@ -1034,67 +1005,156 @@ static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) {
   return (int32_t)(pStart - (char*)buf);
 }
 
-static int32_t saveBlockRowToExtRowsBuf(SSortHandle* pHandle, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
-  SDiskbasedBuf* pResultBuf = pHandle->pExtRowsBuf;
-  int32_t rowBytes = pHandle->extRowBytes;
-  int32_t pageId = -1;
-  SFilePage* pFilePage = NULL;
-  int32_t code = getPageFromExtSrcRowsBuf(pResultBuf, rowBytes, &pageId, &pFilePage);
-  if (code != TSDB_CODE_SUCCESS) {
-    return code;
-  }
-
-  *pPageId = pageId;
-  *pOffset = pFilePage->num;
-  *pLength = blockRowToBuf(pBlock, rowIdx, (char*)pFilePage + (*pOffset));
-
-  pFilePage->num += (*pLength);
-  setBufPageDirty(pFilePage, true);
-  releaseBufPage(pResultBuf, pFilePage);
-  return 0;
-}
-
 // pageId * pageSize == pageStartOffset in file. write in pages
 // when pass the page boundaries, the page is move to the front(old). 
 // find hash from pageid to page entry. if the page can not be found, 
-//   1) nuse inactive pages, 2) then new pages if not exceeding mem limit, 3) then active pages
+//   1) unused inactive pages, 2) then new pages if not exceeding mem limit, 3) then active pages
 // new pages is added or moved to the back.
 
+static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char** ppPage) {
+  SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
+  SSortMemPageEntry** ppPageEntry = tSimpleHashGet(pMemFile->mActivePages, &pageId, sizeof(pageId));
+  if (ppPageEntry) {
+    *ppPage = (char*)((*ppPageEntry)->data);
+  } else {
+    SSortMemPageEntry* pEntry = pMemFile->pagesHead->next;
+    if (pEntry && !pEntry->active || pMemFile->numMemPages >= pMemFile->totalMemPages) {
+      if (pEntry->active) {
+        tSimpleHashRemove(pMemFile->mActivePages, &pEntry->pageId, sizeof(pEntry->pageId));
+      }
+      pEntry->prev->next = pEntry->next;
+      pEntry->next->prev = pEntry->prev;
+      taosLSeekFile(pMemFile->pTdFile, pageId * pMemFile->pageSize, SEEK_SET);
+      taosReadFile(pMemFile->pTdFile, pEntry->data, pMemFile->pageSize);
+      pEntry->active = false;
+    } else if (pMemFile->numMemPages < pMemFile->totalMemPages) {
+      pEntry = taosMemoryCalloc(1, sizeof(SSortMemPageEntry));
+      pEntry->data = taosMemoryMalloc(pMemFile->pageSize);
+      ++pMemFile->numMemPages;
+    }
+    {
+      SSortMemPageEntry* tail = pMemFile->pagesTail;
+      tail->next = pEntry;
+      pEntry->next = NULL;
+      pEntry->prev = tail;
+      pEntry->active = true;
+      pMemFile->pagesTail = pEntry;
+      tSimpleHashPut(pMemFile->mActivePages, &pageId, sizeof(pageId), &pEntry, POINTER_BYTES);
+      *ppPage = pEntry->data;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static void setExtMemFilePageUnused(SSortMemFile* pMemFile, int32_t pageId) {
+  SSortMemPageEntry** ppPageEntry = tSimpleHashGet(pMemFile->mActivePages, &pageId, sizeof(pageId));
+  SSortMemPageEntry* pEntry = *ppPageEntry;
+  if (pEntry == pMemFile->pagesTail) {
+    pMemFile->pagesTail = pEntry->prev;
+  }
+
+  pEntry->prev->next = pEntry->next;
+  pEntry->next->prev = pEntry->prev;
+
+  SSortMemPageEntry* first = pMemFile->pagesHead->next;
+  SSortMemPageEntry* head = pMemFile->pagesHead;
+  head->next = pEntry;
+  pEntry->next = first;
+  first->prev = pEntry;
+  pEntry->prev = head;
+
+  pEntry->active = false;
+  tSimpleHashRemove(pMemFile->mActivePages, &pageId, sizeof(pageId));
+  return;
+}
+
 static int32_t createSortMemFile(SSortHandle* pHandle) {
   if (pHandle->pExtRowsMemFile != NULL) {
     return TSDB_CODE_SUCCESS;
   }
   SSortMemFile* pMemFile = taosMemoryCalloc(1, sizeof(SSortMemFile));
-  pMemFile->pBufFile =
-      taosOpenFile(pBuf->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);  
+
+  taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
+  pMemFile->pTdFile =
+      taosOpenFile(pMemFile->memFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
+  pMemFile->currPageId = -1;
+  pMemFile->currPageOffset = -1;
+
+  pMemFile->pageSize = pHandle->extRowsPageSize;
+  pMemFile->cacheSize = pHandle->extRowsMemSize;
+  pMemFile->pageBuf = taosMemoryMalloc(pMemFile->pageSize);
+  pMemFile->bDirty = false;
+
+  pMemFile->mActivePages = tSimpleHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
+  pMemFile->pagesHead = taosMemoryCalloc(1, sizeof(SSortMemPageEntry));
+  pMemFile->pagesTail = pMemFile->pagesHead;
+
+  pMemFile->totalMemPages = pMemFile->cacheSize / pMemFile->pageSize;
+  pMemFile->numMemPages = 0;
+
+  pHandle->pExtRowsMemFile = pMemFile;
   return TSDB_CODE_SUCCESS;
 } 
 
+static int32_t destroySortMemFile(SSortHandle* pHandle) {
+  if (pHandle->pExtRowsMemFile == NULL) return TSDB_CODE_SUCCESS;
+
+  SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
+  SSortMemPageEntry* pEntry = pMemFile->pagesHead;
+  while (pEntry != NULL) {
+    if (pEntry->data) {
+      taosMemoryFree(pEntry->data);
+    }
+    SSortMemPageEntry* pCurr = pEntry;
+    pEntry = pEntry->next;
+    taosMemoryFree(pCurr);
+  }
+  tSimpleHashCleanup(pMemFile->mActivePages);
+  taosMemoryFree(pMemFile->pageBuf);
+  taosCloseFile(pMemFile->pTdFile);
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
-  SDiskbasedBuf* pResultBuf = pHandle->pExtRowsBuf;
-  int32_t rowBytes = pHandle->extRowBytes;
-  int32_t pageId = -1;
-  SFilePage* pFilePage = NULL;
-  int32_t code = getPageFromExtSrcRowsBuf(pResultBuf, rowBytes, &pageId, &pFilePage);
-  if (code != TSDB_CODE_SUCCESS) {
-    return code;
+  SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
+  if (pMemFile->currPageId == -1) {
+    pMemFile->currPageId = 0;
+    pMemFile->currPageOffset = 0;
+  } else {
+    if (pMemFile->currPageOffset + pHandle->extRowBytes >= pMemFile->pageSize) {
+      taosLSeekFile(pMemFile->pTdFile, pMemFile->currPageId * pMemFile->pageSize, SEEK_SET);
+      taosWriteFile(pMemFile->pTdFile, pMemFile->pageBuf, pMemFile->currPageOffset + 1);
+
+      ++pMemFile->currPageId;
+      pMemFile->currPageOffset = 0;
+    }
   }
 
-  *pPageId = pageId;
-  *pOffset = pFilePage->num;
-  *pLength = blockRowToBuf(pBlock, rowIdx, (char*)pFilePage + (*pOffset));
+  *pPageId = pMemFile->currPageId;
+  *pOffset = pMemFile->currPageOffset;
+  int32_t blockLen = blockRowToBuf(pBlock, rowIdx, pMemFile->pageBuf + pMemFile->currPageOffset);
+  *pLength = blockLen;
+  pMemFile->currPageOffset += blockLen;
+  pMemFile->bDirty = true;
+  return TSDB_CODE_SUCCESS;
+}
 
-  pFilePage->num += (*pLength);
-  setBufPageDirty(pFilePage, true);
-  releaseBufPage(pResultBuf, pFilePage);
-  return 0;
+static int32_t saveLastPageToExtRowsMemFile(SSortHandle* pHandle) {
+  SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
+  if (!pMemFile->bDirty) {
+    return TSDB_CODE_SUCCESS;
+  }
+  taosLSeekFile(pMemFile->pTdFile, pMemFile->currPageId * pMemFile->pageSize, SEEK_SET);
+  taosWriteFile(pMemFile->pTdFile, pMemFile->pageBuf, pMemFile->currPageOffset + 1);
+  pMemFile->bDirty = false;
+  return TSDB_CODE_SUCCESS;
 }
 
 static void appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSource, int32_t* rowIndex) {
   int32_t pageId = -1;
   int32_t offset = -1;
   int32_t length = -1;
-  saveBlockRowToExtRowsBuf(pHandle, pSource, *rowIndex, &pageId, &offset, &length);
+  saveBlockRowToExtRowsMemFile(pHandle, pSource, *rowIndex, &pageId, &offset, &length);
 
   SSDataBlock* pBlock = pHandle->pDataBlock;
   SColumnInfoData* pSrcTsCol = taosArrayGet(pSource->pDataBlock, pHandle->extRowsOrderInfo.slotId);
@@ -1145,14 +1205,13 @@ static void initRowIdSort(SSortHandle* pHandle) {
 }
 
 int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsPageSize, int32_t extRowsMemSize) {
-  int32_t code = createDiskbasedBuf(&pHandle->pExtRowsBuf, extRowsPageSize, extRowsMemSize, "sort-ext-rows", tsTempDir);
-  dBufSetPrintInfo(pHandle->pExtRowsBuf);
   pHandle->extRowBytes = blockDataGetRowSize(pHandle->pDataBlock) + taosArrayGetSize(pHandle->pDataBlock->pDataBlock) + sizeof(int32_t);
   pHandle->extRowsPageSize = extRowsPageSize;
   pHandle->extRowsMemSize = extRowsMemSize;
   SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);
   pHandle->extRowsOrderInfo = *pOrder;
   initRowIdSort(pHandle);
+  int32_t code = createSortMemFile(pHandle);
   pHandle->bSortByRowId = true;
   return code;
 }
@@ -1346,6 +1405,9 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray*
     }
     blockDataCleanup(pHandle->pDataBlock);
   }
+
+  saveLastPageToExtRowsMemFile(pHandle);
+
   SSDataBlock* pMemSrcBlk = createOneDataBlock(pHandle->pDataBlock, false);
   doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId);
 

From fa55a32e7761a6c277c68dea7e17797ee2bfbf8c Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Sun, 4 Feb 2024 16:53:47 +0800
Subject: [PATCH 033/201] fix: pass compilation

---
 source/libs/executor/src/tsort.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index c01612675a..e633af12ed 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1111,7 +1111,7 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
   }
   tSimpleHashCleanup(pMemFile->mActivePages);
   taosMemoryFree(pMemFile->pageBuf);
-  taosCloseFile(pMemFile->pTdFile);
+  taosCloseFile(&pMemFile->pTdFile);
   return TSDB_CODE_SUCCESS;
 }
 

From 8e0f578dba5984db83c3f7b4155303bc54cdbc4b Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Sun, 4 Feb 2024 16:57:48 +0800
Subject: [PATCH 034/201] fix: memory leak

---
 source/libs/executor/src/tsort.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index e633af12ed..22c3f14b43 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1112,6 +1112,8 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
   tSimpleHashCleanup(pMemFile->mActivePages);
   taosMemoryFree(pMemFile->pageBuf);
   taosCloseFile(&pMemFile->pTdFile);
+  taosMemoryFree(pMemFile);
+  pHandle->pExtRowsMemFile = NULL;
   return TSDB_CODE_SUCCESS;
 }
 

From 23bd2aa525801e2343ce89d36ae470787f249953 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Sun, 4 Feb 2024 17:51:29 +0800
Subject: [PATCH 035/201] fix: first pass

---
 source/libs/executor/src/tsort.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 22c3f14b43..3f30e3fe89 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1024,8 +1024,6 @@ static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char*
       }
       pEntry->prev->next = pEntry->next;
       pEntry->next->prev = pEntry->prev;
-      taosLSeekFile(pMemFile->pTdFile, pageId * pMemFile->pageSize, SEEK_SET);
-      taosReadFile(pMemFile->pTdFile, pEntry->data, pMemFile->pageSize);
       pEntry->active = false;
     } else if (pMemFile->numMemPages < pMemFile->totalMemPages) {
       pEntry = taosMemoryCalloc(1, sizeof(SSortMemPageEntry));
@@ -1033,6 +1031,8 @@ static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char*
       ++pMemFile->numMemPages;
     }
     {
+      taosLSeekFile(pMemFile->pTdFile, pageId * pMemFile->pageSize, SEEK_SET);
+      taosReadFile(pMemFile->pTdFile, pEntry->data, pMemFile->pageSize);      
       SSortMemPageEntry* tail = pMemFile->pagesTail;
       tail->next = pEntry;
       pEntry->next = NULL;

From 18366934c30c5b46651d26ebd97e33da7133597b Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Sun, 4 Feb 2024 20:16:19 +0800
Subject: [PATCH 036/201] fix: use c api

---
 source/libs/executor/src/tsort.c | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 3f30e3fe89..022a420c06 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -13,6 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define ALLOW_FORBID_FUNC
+
 #include "query.h"
 #include "tcommon.h"
 
@@ -58,7 +60,7 @@ typedef struct SSortMemFile {
   int32_t numMemPages;
   SSHashObj* mActivePages;
 
-  TdFilePtr pTdFile;
+  FILE* pTdFile;
   char memFilePath[PATH_MAX];
 } SSortMemFile;
 
@@ -1031,8 +1033,8 @@ static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char*
       ++pMemFile->numMemPages;
     }
     {
-      taosLSeekFile(pMemFile->pTdFile, pageId * pMemFile->pageSize, SEEK_SET);
-      taosReadFile(pMemFile->pTdFile, pEntry->data, pMemFile->pageSize);      
+      fseek(pMemFile->pTdFile, pageId * pMemFile->pageSize, SEEK_SET);
+      fread(pEntry->data, pMemFile->pageSize, 1, pMemFile->pTdFile);      
       SSortMemPageEntry* tail = pMemFile->pagesTail;
       tail->next = pEntry;
       pEntry->next = NULL;
@@ -1076,7 +1078,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
 
   taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
   pMemFile->pTdFile =
-      taosOpenFile(pMemFile->memFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
+      fopen(pMemFile->memFilePath, "wb+");
   pMemFile->currPageId = -1;
   pMemFile->currPageOffset = -1;
 
@@ -1111,7 +1113,8 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
   }
   tSimpleHashCleanup(pMemFile->mActivePages);
   taosMemoryFree(pMemFile->pageBuf);
-  taosCloseFile(&pMemFile->pTdFile);
+  fclose(pMemFile->pTdFile);
+  taosRemoveFile(pMemFile->memFilePath);
   taosMemoryFree(pMemFile);
   pHandle->pExtRowsMemFile = NULL;
   return TSDB_CODE_SUCCESS;
@@ -1124,8 +1127,8 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p
     pMemFile->currPageOffset = 0;
   } else {
     if (pMemFile->currPageOffset + pHandle->extRowBytes >= pMemFile->pageSize) {
-      taosLSeekFile(pMemFile->pTdFile, pMemFile->currPageId * pMemFile->pageSize, SEEK_SET);
-      taosWriteFile(pMemFile->pTdFile, pMemFile->pageBuf, pMemFile->currPageOffset + 1);
+      fseek(pMemFile->pTdFile, pMemFile->currPageId * pMemFile->pageSize, SEEK_SET);
+      fwrite(pMemFile->pageBuf, pMemFile->currPageOffset + 1, 1, pMemFile->pTdFile);
 
       ++pMemFile->currPageId;
       pMemFile->currPageOffset = 0;
@@ -1146,8 +1149,8 @@ static int32_t saveLastPageToExtRowsMemFile(SSortHandle* pHandle) {
   if (!pMemFile->bDirty) {
     return TSDB_CODE_SUCCESS;
   }
-  taosLSeekFile(pMemFile->pTdFile, pMemFile->currPageId * pMemFile->pageSize, SEEK_SET);
-  taosWriteFile(pMemFile->pTdFile, pMemFile->pageBuf, pMemFile->currPageOffset + 1);
+  fseek(pMemFile->pTdFile, pMemFile->currPageId * pMemFile->pageSize, SEEK_SET);
+  fwrite(pMemFile->pageBuf, pMemFile->currPageOffset + 1, 1, pMemFile->pTdFile);
   pMemFile->bDirty = false;
   return TSDB_CODE_SUCCESS;
 }

From 247bfae03997d052d694d63de7e3a9a0df7b7649 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Sun, 4 Feb 2024 21:12:19 +0800
Subject: [PATCH 037/201] fix: use fseeko

---
 source/libs/executor/src/tsort.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 022a420c06..1935bb68bb 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1127,7 +1127,7 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p
     pMemFile->currPageOffset = 0;
   } else {
     if (pMemFile->currPageOffset + pHandle->extRowBytes >= pMemFile->pageSize) {
-      fseek(pMemFile->pTdFile, pMemFile->currPageId * pMemFile->pageSize, SEEK_SET);
+      fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->currPageId) * pMemFile->pageSize, SEEK_SET);
       fwrite(pMemFile->pageBuf, pMemFile->currPageOffset + 1, 1, pMemFile->pTdFile);
 
       ++pMemFile->currPageId;
@@ -1149,7 +1149,7 @@ static int32_t saveLastPageToExtRowsMemFile(SSortHandle* pHandle) {
   if (!pMemFile->bDirty) {
     return TSDB_CODE_SUCCESS;
   }
-  fseek(pMemFile->pTdFile, pMemFile->currPageId * pMemFile->pageSize, SEEK_SET);
+  fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->currPageId) * pMemFile->pageSize, SEEK_SET);
   fwrite(pMemFile->pageBuf, pMemFile->currPageOffset + 1, 1, pMemFile->pTdFile);
   pMemFile->bDirty = false;
   return TSDB_CODE_SUCCESS;

From 83e44f07355e09a17ed5dfa1d68d6483bfcefda8 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Sun, 4 Feb 2024 22:36:25 +0800
Subject: [PATCH 038/201] fix: more big batch writes than page writes

---
 source/libs/executor/src/tsort.c | 35 ++++++++++++++++++++++----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 1935bb68bb..25c69c35c9 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -48,7 +48,10 @@ typedef struct SSortMemPageEntry {
 typedef struct SSortMemFile {
   int32_t pageSize;
   int32_t cacheSize;
-  char*   pageBuf;
+  
+  char*   writePageBuf;
+  int32_t startPageId;
+  int32_t numWritePages;
   
   int32_t currPageId;
   int32_t currPageOffset;
@@ -123,7 +126,7 @@ struct SSortHandle {
 static int32_t destroySortMemFile(SSortHandle* pHandle);
 static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char** ppPage);
 static void    setExtMemFilePageUnused(SSortMemFile* pMemFile, int32_t pageId);
-static int32_t saveLastPageToExtRowsMemFile(SSortHandle* pHandle);
+static int32_t saveDirtyPagesToExtRowsMemFile(SSortHandle* pHandle);
 
 void tsortSetSingleTableMerge(SSortHandle* pHandle) {
   pHandle->singleTableMerge = true;
@@ -1084,7 +1087,8 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
 
   pMemFile->pageSize = pHandle->extRowsPageSize;
   pMemFile->cacheSize = pHandle->extRowsMemSize;
-  pMemFile->pageBuf = taosMemoryMalloc(pMemFile->pageSize);
+  pMemFile->numWritePages = pMemFile->cacheSize/pMemFile->pageSize;
+  pMemFile->writePageBuf = taosMemoryMalloc(pMemFile->pageSize * pMemFile->numWritePages);
   pMemFile->bDirty = false;
 
   pMemFile->mActivePages = tSimpleHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
@@ -1112,7 +1116,7 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
     taosMemoryFree(pCurr);
   }
   tSimpleHashCleanup(pMemFile->mActivePages);
-  taosMemoryFree(pMemFile->pageBuf);
+  taosMemoryFree(pMemFile->writePageBuf);
   fclose(pMemFile->pTdFile);
   taosRemoveFile(pMemFile->memFilePath);
   taosMemoryFree(pMemFile);
@@ -1125,32 +1129,41 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p
   if (pMemFile->currPageId == -1) {
     pMemFile->currPageId = 0;
     pMemFile->currPageOffset = 0;
+    pMemFile->startPageId = 0;
   } else {
     if (pMemFile->currPageOffset + pHandle->extRowBytes >= pMemFile->pageSize) {
-      fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->currPageId) * pMemFile->pageSize, SEEK_SET);
-      fwrite(pMemFile->pageBuf, pMemFile->currPageOffset + 1, 1, pMemFile->pTdFile);
 
       ++pMemFile->currPageId;
       pMemFile->currPageOffset = 0;
+
+      if (pMemFile->currPageId - pMemFile->startPageId >= pMemFile->numWritePages) {
+        fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
+        fwrite(pMemFile->writePageBuf, pMemFile->pageSize * pMemFile->numWritePages, 1, pMemFile->pTdFile);
+
+        pMemFile->startPageId = pMemFile->currPageId;
+      }
     }
   }
 
   *pPageId = pMemFile->currPageId;
   *pOffset = pMemFile->currPageOffset;
-  int32_t blockLen = blockRowToBuf(pBlock, rowIdx, pMemFile->pageBuf + pMemFile->currPageOffset);
+  int32_t offsetPages = (pMemFile->currPageId - pMemFile->startPageId) * pMemFile->pageSize;
+  int32_t blockLen = blockRowToBuf(pBlock, rowIdx, 
+                  pMemFile->writePageBuf + offsetPages + pMemFile->currPageOffset);
   *pLength = blockLen;
   pMemFile->currPageOffset += blockLen;
   pMemFile->bDirty = true;
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t saveLastPageToExtRowsMemFile(SSortHandle* pHandle) {
+static int32_t saveDirtyPagesToExtRowsMemFile(SSortHandle* pHandle) {
   SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
   if (!pMemFile->bDirty) {
     return TSDB_CODE_SUCCESS;
   }
-  fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->currPageId) * pMemFile->pageSize, SEEK_SET);
-  fwrite(pMemFile->pageBuf, pMemFile->currPageOffset + 1, 1, pMemFile->pTdFile);
+  fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
+  int32_t numWriteBytes = pMemFile->pageSize * (pMemFile->currPageId - pMemFile->startPageId) + pMemFile->currPageOffset + 1;
+  fwrite(pMemFile->writePageBuf, numWriteBytes, 1, pMemFile->pTdFile);
   pMemFile->bDirty = false;
   return TSDB_CODE_SUCCESS;
 }
@@ -1411,7 +1424,7 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray*
     blockDataCleanup(pHandle->pDataBlock);
   }
 
-  saveLastPageToExtRowsMemFile(pHandle);
+  saveDirtyPagesToExtRowsMemFile(pHandle);
 
   SSDataBlock* pMemSrcBlk = createOneDataBlock(pHandle->pDataBlock, false);
   doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId);

From 472020a2b968e28ffe14630ae0979ec57f919608 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 18 Feb 2024 14:07:15 +0800
Subject: [PATCH 039/201] enh(stream): handle event async function.

---
 include/libs/stream/tstream.h         |   5 +-
 source/dnode/vnode/src/tq/tq.c        |  67 ++++++++-----
 source/libs/stream/inc/streamsm.h     |  20 ++--
 source/libs/stream/src/streamStart.c  |   2 +-
 source/libs/stream/src/streamTaskSm.c | 139 ++++++++++++++++++++++----
 5 files changed, 176 insertions(+), 57 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 2135bb706b..587e762448 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -799,7 +799,10 @@ int8_t  streamTaskSetSchedStatusInactive(SStreamTask* pTask);
 int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, bool metaLock);
 
 int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
-int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event);
+
+typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
+int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param);
+int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param);
 void    streamTaskRestoreStatus(SStreamTask* pTask);
 
 int32_t streamTaskStop(SStreamTask* pTask);
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index bde6889ecd..940a8e0c49 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -916,6 +916,22 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask
   }
 }
 
+int32_t handleStep2Async(SStreamTask* pStreamTask, void* param) {
+  STQ* pTq = param;
+
+  SStreamMeta* pMeta = pStreamTask->pMeta;
+  STaskId      hId = pStreamTask->hTaskInfo.id;
+  SStreamTask* pTask = streamMetaAcquireTask(pStreamTask->pMeta, hId.streamId, hId.taskId);
+  if (pTask == NULL) {
+    // todo handle error
+  }
+
+  doStartFillhistoryStep2(pTask, pStreamTask, pTq);
+
+  streamMetaReleaseTask(pMeta, pTask);
+  return 0;
+}
+
 // this function should be executed by only one thread, so we set an sentinel to protect this function
 int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
   SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont;
@@ -988,7 +1004,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
       streamReExecScanHistoryFuture(pTask, retInfo.idleTime);
     } else {
       SStreamTaskState* p = streamTaskGetStatus(pTask);
-      ETaskStatus s = p->state;
+      ETaskStatus       s = p->state;
 
       if (s == TASK_STATUS__PAUSE) {
         tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr,
@@ -1006,37 +1022,34 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
   // the following procedure should be executed, no matter status is stop/pause or not
   tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, pTask->execInfo.step1El);
 
-  if (pTask->info.fillHistory) {
-    SStreamTask* pStreamTask = NULL;
+  ASSERT(pTask->info.fillHistory == 1);
 
-    // 1. get the related stream task
-    pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId);
-    if (pStreamTask == NULL) {
-      tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s",
-              pTask->streamTaskId.taskId, pTask->id.idStr);
+  // 1. get the related stream task
+  SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId);
+  if (pStreamTask == NULL) {
+    tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s",
+            pTask->streamTaskId.taskId, pTask->id.idStr);
 
-      tqDebug("s-task:%s fill-history task set status to be dropping", id);
-      streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id);
+    tqDebug("s-task:%s fill-history task set status to be dropping", id);
+    streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id);
 
-      atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
-      streamMetaReleaseTask(pMeta, pTask);
-      return -1;
-    }
-
-    ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE);
-
-    code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
-    if (code == TSDB_CODE_SUCCESS) {
-      doStartFillhistoryStep2(pTask, pStreamTask, pTq);
-    } else {
-      tqError("s-task:%s failed to halt s-task:%s, not launch step2", id, pStreamTask->id.idStr);
-    }
-
-    streamMetaReleaseTask(pMeta, pStreamTask);
-  } else {
-    ASSERT(0);
+    atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
+    streamMetaReleaseTask(pMeta, pTask);
+    return -1;
   }
 
+  ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE);
+
+  //    code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
+  code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, handleStep2Async, pTq);
+//  if (code == TSDB_CODE_SUCCESS) {
+//    doStartFillhistoryStep2(pTask, pStreamTask, pTq);
+//  } else {
+//    tqError("s-task:%s failed to halt s-task:%s, not launch step2", id, pStreamTask->id.idStr);
+//  }
+
+  streamMetaReleaseTask(pMeta, pStreamTask);
+
   atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
   streamMetaReleaseTask(pMeta, pTask);
   return code;
diff --git a/source/libs/stream/inc/streamsm.h b/source/libs/stream/inc/streamsm.h
index 22e1c4497b..47e0ce1b55 100644
--- a/source/libs/stream/inc/streamsm.h
+++ b/source/libs/stream/inc/streamsm.h
@@ -26,21 +26,21 @@ extern "C" {
 typedef int32_t (*__state_trans_fn)(SStreamTask*);
 typedef int32_t (*__state_trans_succ_fn)(SStreamTask*);
 
-typedef struct SAttachedEventInfo {
+typedef struct SFutureHandleEventInfo {
   ETaskStatus      status;  // required status that this event can be handled
   EStreamTaskEvent event;   // the delayed handled event
   void*            pParam;
-  void*            pFn;
-} SAttachedEventInfo;
+  __state_trans_user_fn callBackFn;
+} SFutureHandleEventInfo;
 
 typedef struct STaskStateTrans {
-  bool                  autoInvokeEndFn;
-  SStreamTaskState      state;
-  EStreamTaskEvent      event;
-  SStreamTaskState      next;
-  __state_trans_fn      pAction;
-  __state_trans_succ_fn pSuccAction;
-  SAttachedEventInfo    attachEvent;
+  bool                   autoInvokeEndFn;
+  SStreamTaskState       state;
+  EStreamTaskEvent       event;
+  SStreamTaskState       next;
+  __state_trans_fn       pAction;
+  __state_trans_succ_fn  pSuccAction;
+  SFutureHandleEventInfo attachEvent;
 } STaskStateTrans;
 
 struct SStreamTaskSM {
diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c
index ee98bc801b..dd99f59f91 100644
--- a/source/libs/stream/src/streamStart.c
+++ b/source/libs/stream/src/streamStart.c
@@ -385,7 +385,7 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) {
 
 void doProcessDownstreamReadyRsp(SStreamTask* pTask) {
   EStreamTaskEvent event = (pTask->info.fillHistory == 0) ? TASK_EVENT_INIT : TASK_EVENT_INIT_SCANHIST;
-  streamTaskOnHandleEventSuccess(pTask->status.pSM, event);
+  streamTaskOnHandleEventSuccess(pTask->status.pSM, event, NULL, NULL);
 
   int64_t initTs = pTask->execInfo.init;
   int64_t startTs = pTask->execInfo.start;
diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c
index 83e71c42bc..ecd3fba725 100644
--- a/source/libs/stream/src/streamTaskSm.c
+++ b/source/libs/stream/src/streamTaskSm.c
@@ -63,16 +63,20 @@ static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask);
 
 static STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event,
                                             __state_trans_fn fn, __state_trans_succ_fn succFn,
-                                            SAttachedEventInfo* pEventInfo, bool autoInvoke);
+                                            SFutureHandleEventInfo* pEventInfo, bool autoInvoke);
 
 static int32_t dummyFn(SStreamTask* UNUSED_PARAM(p)) { return TSDB_CODE_SUCCESS; }
 
-static int32_t attachEvent(SStreamTask* pTask, SAttachedEventInfo* pEvtInfo) {
+static int32_t attachNextHandledEvent(SStreamTask* pTask, SFutureHandleEventInfo* pEvtInfo) {
   char* p = streamTaskGetStatus(pTask)->name;
 
   stDebug("s-task:%s status:%s attach event:%s required status:%s, since not allowed to handle it", pTask->id.idStr, p,
           GET_EVT_NAME(pEvtInfo->event), StreamTaskStatusList[pEvtInfo->status].name);
-  taosArrayPush(pTask->status.pSM->pWaitingEventList, pEvtInfo);
+
+  SArray* pList = pTask->status.pSM->pWaitingEventList;
+  taosArrayPush(pList, pEvtInfo);
+
+  stDebug("s-task:%s add into waiting list, total waiting events:%d", pTask->id.idStr, (int32_t)taosArrayGetSize(pList));
   return 0;
 }
 
@@ -170,9 +174,11 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName,
   stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr,
           pEventName, el, pSM->prev.state.name, pSM->current.name);
 
-  SAttachedEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0);
+  ASSERT(taosArrayGetSize(pSM->pWaitingEventList) == 1);
 
-  // OK, let's handle the attached event, since the task has reached the required status now
+  SFutureHandleEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0);
+
+  // OK, let's handle the waiting event, since the task has reached the required status now
   if (pSM->current.state == pEvtInfo->status) {
     stDebug("s-task:%s handle the event:%s in waiting list, state:%s", pTask->id.idStr,
             GET_EVT_NAME(pEvtInfo->event), pSM->current.name);
@@ -189,7 +195,7 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName,
 
     code = pNextTrans->pAction(pSM->pTask);
     if (pNextTrans->autoInvokeEndFn) {
-      return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event);
+      return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event, pEvtInfo->callBackFn, pEvtInfo->pParam);
     } else {
       return code;
     }
@@ -242,7 +248,7 @@ SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) {
   }
 
   pSM->pTask = pTask;
-  pSM->pWaitingEventList = taosArrayInit(4, sizeof(SAttachedEventInfo));
+  pSM->pWaitingEventList = taosArrayInit(4, sizeof(SFutureHandleEventInfo));
   if (pSM->pWaitingEventList == NULL) {
     taosMemoryFree(pSM);
 
@@ -273,7 +279,7 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt
   const char*  id = pTask->id.idStr;
 
   if (pTrans->attachEvent.event != 0) {
-    attachEvent(pTask, &pTrans->attachEvent);
+    attachNextHandledEvent(pTask, &pTrans->attachEvent);
     taosThreadMutexUnlock(&pTask->lock);
 
     while (1) {
@@ -303,7 +309,53 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt
     // todo handle error code;
 
     if (pTrans->autoInvokeEndFn) {
-      streamTaskOnHandleEventSuccess(pSM, event);
+      streamTaskOnHandleEventSuccess(pSM, event, NULL, NULL);
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans, __state_trans_user_fn callbackFn, void* param) {
+  SStreamTask* pTask = pSM->pTask;
+  const char*  id = pTask->id.idStr;
+
+  if (pTrans->attachEvent.event != 0) {
+    SFutureHandleEventInfo info = pTrans->attachEvent;
+    info.pParam = param;
+    info.callBackFn = callbackFn;
+
+    attachNextHandledEvent(pTask, &info);
+    taosThreadMutexUnlock(&pTask->lock);
+
+//    while (1) {
+//      // wait for the task to be here
+//      taosThreadMutexLock(&pTask->lock);
+//      ETaskStatus s = streamTaskGetStatus(pTask)->state;
+//      taosThreadMutexUnlock(&pTask->lock);
+//
+//      if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already
+//        stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event));
+//        return TSDB_CODE_SUCCESS;
+//      } else if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__STOP && s != TASK_STATUS__UNINIT) {
+//        stDebug("s-task:%s not handle event:%s yet, wait for 100ms and recheck", id, GET_EVT_NAME(event));
+//        taosMsleep(100);
+//      } else {
+//        stDebug("s-task:%s is dropped or stopped already, not wait.", id);
+//        return TSDB_CODE_STREAM_INVALID_STATETRANS;
+//      }
+//    }
+
+  } else {  // override current active trans
+    pSM->pActiveTrans = pTrans;
+    pSM->startTs = taosGetTimestampMs();
+    taosThreadMutexUnlock(&pTask->lock);
+
+    int32_t code = pTrans->pAction(pTask);
+    // todo handle error code;
+
+    if (pTrans->autoInvokeEndFn) {
+      streamTaskOnHandleEventSuccess(pSM, event, NULL, NULL);
     }
   }
 
@@ -349,6 +401,46 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
   return code;
 }
 
+int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) {
+  int32_t          code = TSDB_CODE_SUCCESS;
+  SStreamTask*     pTask = pSM->pTask;
+  STaskStateTrans* pTrans = NULL;
+
+  while (1) {
+    taosThreadMutexLock(&pTask->lock);
+
+    if (pSM->pActiveTrans != NULL && pSM->pActiveTrans->autoInvokeEndFn) {
+      EStreamTaskEvent evt = pSM->pActiveTrans->event;
+      taosThreadMutexUnlock(&pTask->lock);
+
+      stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed",
+              pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt));
+      ASSERT(0);
+      taosMsleep(100);
+    } else {
+      // no active event trans exists, handle this event directly
+      pTrans = streamTaskFindTransform(pSM->current.state, event);
+      if (pTrans == NULL) {
+        stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, GET_EVT_NAME(event));
+        taosThreadMutexUnlock(&pTask->lock);
+        return TSDB_CODE_STREAM_INVALID_STATETRANS;
+      }
+
+      if (pSM->pActiveTrans != NULL) {
+        // currently in some state transfer procedure, not auto invoke transfer, quit from this prcedure
+        stDebug("s-task:%s event:%s handle procedure quit, status %s -> %s failed, handle event %s now",
+                pTask->id.idStr, GET_EVT_NAME(pSM->pActiveTrans->event), pSM->current.name,
+                pSM->pActiveTrans->next.name, GET_EVT_NAME(event));
+      }
+
+      code = doHandleEventAsync(pSM, event, pTrans, callbackFn, param);
+      break;
+    }
+  }
+
+  return code;
+}
+
 static void keepPrevInfo(SStreamTaskSM* pSM) {
   STaskStateTrans* pTrans = pSM->pActiveTrans;
 
@@ -356,8 +448,9 @@ static void keepPrevInfo(SStreamTaskSM* pSM) {
   pSM->prev.evt = pTrans->event;
 }
 
-int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event) {
+int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param) {
   SStreamTask* pTask = pSM->pTask;
+  const char* id = pTask->id.idStr;
 
   // do update the task status
   taosThreadMutexLock(&pTask->lock);
@@ -369,16 +462,16 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
            s == TASK_STATUS__UNINIT || s == TASK_STATUS__READY);
 
     // the pSM->prev.evt may be 0, so print string is not appropriate.
-    stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", pTask->id.idStr,
-            GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pSM->prev.evt));
+    stDebug("s-task:%s event:%s handled failed, current status:%s, trigger event:%s", id, GET_EVT_NAME(event),
+            pSM->current.name, GET_EVT_NAME(pSM->prev.evt));
 
     taosThreadMutexUnlock(&pTask->lock);
     return TSDB_CODE_STREAM_INVALID_STATETRANS;
   }
 
   if (pTrans->event != event) {
-    stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", pTask->id.idStr,
-           GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pTrans->event));
+    stWarn("s-task:%s handle event:%s failed, current status:%s, active trans evt:%s", id, GET_EVT_NAME(event),
+           pSM->current.name, GET_EVT_NAME(pTrans->event));
     taosThreadMutexUnlock(&pTask->lock);
     return TSDB_CODE_STREAM_INVALID_STATETRANS;
   }
@@ -388,16 +481,26 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
   pSM->current = pTrans->next;
   pSM->pActiveTrans = NULL;
 
+  // todo remove it
   // on success callback, add into lock if necessary, or maybe we should add an option for this?
   pTrans->pSuccAction(pTask);
 
+  // after handling the callback function assigned by invoker, go on handling the waiting tasks
+  if (callbackFn != NULL) {
+    stDebug("s-task:%s start to handle user-specified callback fn for event:%s", id, GET_EVT_NAME(pTrans->event));
+    callbackFn(pSM->pTask, param);
+
+    stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event));
+  }
+
+  // tasks in waiting list
   if (taosArrayGetSize(pSM->pWaitingEventList) > 0) {
     doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask);
   } else {
     taosThreadMutexUnlock(&pTask->lock);
 
     int64_t el = (taosGetTimestampMs() - pSM->startTs);
-    stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr,
+    stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", id,
             GET_EVT_NAME(pTrans->event), el, pSM->prev.state.name, pSM->current.name);
   }
 
@@ -453,7 +556,7 @@ void streamTaskSetStatusReady(SStreamTask* pTask) {
 }
 
 STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event, __state_trans_fn fn,
-                                     __state_trans_succ_fn succFn, SAttachedEventInfo* pEventInfo, bool autoInvoke) {
+                                     __state_trans_succ_fn succFn, SFutureHandleEventInfo* pEventInfo, bool autoInvoke) {
   STaskStateTrans trans = {0};
   trans.state = StreamTaskStatusList[current];
   trans.next = StreamTaskStatusList[next];
@@ -497,7 +600,7 @@ void doInitStateTransferTable(void) {
   trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true);
   taosArrayPush(streamTaskSMTrans, &trans);
 
-  SAttachedEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT};
+  SFutureHandleEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT};
 
   trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true);
   taosArrayPush(streamTaskSMTrans, &trans);
@@ -518,7 +621,7 @@ void doInitStateTransferTable(void) {
   trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
   taosArrayPush(streamTaskSMTrans, &trans);
 
-  info = (SAttachedEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE};
+  info = (SFutureHandleEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE};
   trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true);
   taosArrayPush(streamTaskSMTrans, &trans);
   trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true);

From dbea34ce0ea9621fc01d08b35dbc266de708a55f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 18 Feb 2024 14:34:42 +0800
Subject: [PATCH 040/201] fix(stream): set the callback function for handle
 event.

---
 source/libs/stream/src/streamTaskSm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c
index ecd3fba725..4ca0040941 100644
--- a/source/libs/stream/src/streamTaskSm.c
+++ b/source/libs/stream/src/streamTaskSm.c
@@ -355,7 +355,7 @@ static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, ST
     // todo handle error code;
 
     if (pTrans->autoInvokeEndFn) {
-      streamTaskOnHandleEventSuccess(pSM, event, NULL, NULL);
+      streamTaskOnHandleEventSuccess(pSM, event, callbackFn, param);
     }
   }
 

From c4f9bee62922e7b738f872cf66e2d9c30d29f939 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 18 Feb 2024 15:46:37 +0800
Subject: [PATCH 041/201] fix(stream): add into buffer pool before start trans,
 to avoid false alarm on orphan task.

---
 source/dnode/mnode/impl/src/mndStream.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 173d28c705..cc6b2eadcb 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -720,6 +720,13 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
+  // add into buffer firstly
+  // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already.
+  taosThreadMutexLock(&execInfo.lock);
+  mDebug("stream tasks register into node list");
+  saveStreamTasksInfo(&streamObj, &execInfo);
+  taosThreadMutexUnlock(&execInfo.lock);
+
   // execute creation
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@@ -729,12 +736,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
 
   mndTransDrop(pTrans);
 
-  taosThreadMutexLock(&execInfo.lock);
-
-  mDebug("stream tasks register into node list");
-  saveStreamTasksInfo(&streamObj, &execInfo);
-  taosThreadMutexUnlock(&execInfo.lock);
-
   SName dbname = {0};
   tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
 

From 245f0ef806493242470db3cbc8346cdc782af3f9 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 18 Feb 2024 17:05:00 +0800
Subject: [PATCH 042/201] fix(stream): fix deadlock.

---
 source/dnode/vnode/src/tq/tqStreamTask.c | 2 +-
 source/libs/stream/src/streamTaskSm.c    | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c
index 280c110711..73508202d9 100644
--- a/source/dnode/vnode/src/tq/tqStreamTask.c
+++ b/source/dnode/vnode/src/tq/tqStreamTask.c
@@ -28,8 +28,8 @@ static int32_t tqScanWalInFuture(STQ* pTq, int32_t numOfTasks, int32_t idleDurat
 
 // extract data blocks(submit/delete) from WAL, and add them into the input queue for all the sources tasks.
 int32_t tqScanWal(STQ* pTq) {
-  int32_t      vgId = TD_VID(pTq->pVnode);
   SStreamMeta* pMeta = pTq->pStreamMeta;
+  int32_t      vgId = pMeta->vgId;
   int64_t      st = taosGetTimestampMs();
 
   tqDebug("vgId:%d continue to check if data in wal are available, scanCounter:%d", vgId, pMeta->scanInfo.scanCounter);
diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c
index 4ca0040941..a44e107851 100644
--- a/source/libs/stream/src/streamTaskSm.c
+++ b/source/libs/stream/src/streamTaskSm.c
@@ -415,7 +415,6 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _
 
       stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed",
               pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt));
-      ASSERT(0);
       taosMsleep(100);
     } else {
       // no active event trans exists, handle this event directly
@@ -485,6 +484,9 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
   // on success callback, add into lock if necessary, or maybe we should add an option for this?
   pTrans->pSuccAction(pTask);
 
+  taosThreadMutexUnlock(&pTask->lock);
+
+  // todo: add parameter to control lock
   // after handling the callback function assigned by invoker, go on handling the waiting tasks
   if (callbackFn != NULL) {
     stDebug("s-task:%s start to handle user-specified callback fn for event:%s", id, GET_EVT_NAME(pTrans->event));
@@ -493,6 +495,8 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even
     stDebug("s-task:%s handle user-specified callback fn for event:%s completed", id, GET_EVT_NAME(pTrans->event));
   }
 
+  taosThreadMutexLock(&pTask->lock);
+
   // tasks in waiting list
   if (taosArrayGetSize(pSM->pWaitingEventList) > 0) {
     doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask);

From d5b316839dae1135d4cc2ff9f0a5015e0d4d1a4e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 19 Feb 2024 09:14:21 +0800
Subject: [PATCH 043/201] refactor: do some internal refactor.

---
 source/libs/stream/src/streamTaskSm.c | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c
index a44e107851..593c2c5754 100644
--- a/source/libs/stream/src/streamTaskSm.c
+++ b/source/libs/stream/src/streamTaskSm.c
@@ -89,11 +89,6 @@ int32_t streamTaskInitStatus(SStreamTask* pTask) {
   return 0;
 }
 
-static int32_t streamTaskDoCheckpoint(SStreamTask* pTask) {
-  stDebug("s-task:%s start to do checkpoint", pTask->id.idStr);
-  return 0;
-}
-
 int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask) {
   if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
     streamTaskSendCheckpointSourceRsp(pTask);
@@ -612,9 +607,9 @@ void doInitStateTransferTable(void) {
   taosArrayPush(streamTaskSMTrans, &trans);
 
   // checkpoint related event
-  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, streamTaskDoCheckpoint, NULL, true);
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL, true);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, streamTaskDoCheckpoint, NULL, true);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL, true);
   taosArrayPush(streamTaskSMTrans, &trans);
   trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL, true);
   taosArrayPush(streamTaskSMTrans, &trans);

From 7e866c55278f86b892f50a906f86734ad992495d Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 19 Feb 2024 11:32:04 +0800
Subject: [PATCH 044/201] refactor: do some internal refactor.

---
 source/dnode/vnode/src/tq/tq.c        |   7 --
 source/libs/stream/src/streamMeta.c   |   9 +-
 source/libs/stream/src/streamTaskSm.c | 113 ++++++++++----------------
 3 files changed, 50 insertions(+), 79 deletions(-)

diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 940a8e0c49..5ac44cbbae 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -1039,14 +1039,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
   }
 
   ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE);
-
-  //    code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
   code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, handleStep2Async, pTq);
-//  if (code == TSDB_CODE_SUCCESS) {
-//    doStartFillhistoryStep2(pTask, pStreamTask, pTq);
-//  } else {
-//    tqError("s-task:%s failed to halt s-task:%s, not launch step2", id, pStreamTask->id.idStr);
-//  }
 
   streamMetaReleaseTask(pMeta, pStreamTask);
 
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index db74ce9897..774a32b265 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -669,6 +669,13 @@ static void doRemoveIdFromList(SStreamMeta* pMeta, int32_t num, SStreamTaskId* i
   }
 }
 
+static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) {
+  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
+    streamTaskSendCheckpointSourceRsp(pTask);
+  }
+  return 0;
+}
+
 int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) {
   SStreamTask* pTask = NULL;
 
@@ -687,7 +694,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t
     }
 
     // handle the dropping event
-    streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_DROPPING);
+    streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL);
   } else {
     stDebug("vgId:%d failed to find the task:0x%x, it may be dropped already", pMeta->vgId, taskId);
     streamMetaWUnLock(pMeta);
diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c
index 593c2c5754..9ca5248157 100644
--- a/source/libs/stream/src/streamTaskSm.c
+++ b/source/libs/stream/src/streamTaskSm.c
@@ -59,15 +59,14 @@ static int32_t streamTaskInitStatus(SStreamTask* pTask);
 static int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask);
 static int32_t initStateTransferTable();
 static void    doInitStateTransferTable(void);
-static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask);
 
 static STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event,
                                             __state_trans_fn fn, __state_trans_succ_fn succFn,
-                                            SFutureHandleEventInfo* pEventInfo, bool autoInvoke);
+                                            SFutureHandleEventInfo* pEventInfo);
 
 static int32_t dummyFn(SStreamTask* UNUSED_PARAM(p)) { return TSDB_CODE_SUCCESS; }
 
-static int32_t attachNextHandledEvent(SStreamTask* pTask, SFutureHandleEventInfo* pEvtInfo) {
+static int32_t attachWaitedEvent(SStreamTask* pTask, SFutureHandleEventInfo* pEvtInfo) {
   char* p = streamTaskGetStatus(pTask)->name;
 
   stDebug("s-task:%s status:%s attach event:%s required status:%s, since not allowed to handle it", pTask->id.idStr, p,
@@ -89,13 +88,6 @@ int32_t streamTaskInitStatus(SStreamTask* pTask) {
   return 0;
 }
 
-int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask) {
-  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-    streamTaskSendCheckpointSourceRsp(pTask);
-  }
-  return 0;
-}
-
 int32_t streamTaskKeepCurrentVerInWal(SStreamTask* pTask) {
   if (!HAS_RELATED_FILLHISTORY_TASK(pTask)) {
     stError("s-task:%s no related fill-history task, since it may have been dropped already", pTask->id.idStr);
@@ -274,7 +266,7 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt
   const char*  id = pTask->id.idStr;
 
   if (pTrans->attachEvent.event != 0) {
-    attachNextHandledEvent(pTask, &pTrans->attachEvent);
+    attachWaitedEvent(pTask, &pTrans->attachEvent);
     taosThreadMutexUnlock(&pTask->lock);
 
     while (1) {
@@ -313,34 +305,13 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt
 
 static int32_t doHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskStateTrans* pTrans, __state_trans_user_fn callbackFn, void* param) {
   SStreamTask* pTask = pSM->pTask;
-  const char*  id = pTask->id.idStr;
-
   if (pTrans->attachEvent.event != 0) {
     SFutureHandleEventInfo info = pTrans->attachEvent;
     info.pParam = param;
     info.callBackFn = callbackFn;
 
-    attachNextHandledEvent(pTask, &info);
+    attachWaitedEvent(pTask, &info);
     taosThreadMutexUnlock(&pTask->lock);
-
-//    while (1) {
-//      // wait for the task to be here
-//      taosThreadMutexLock(&pTask->lock);
-//      ETaskStatus s = streamTaskGetStatus(pTask)->state;
-//      taosThreadMutexUnlock(&pTask->lock);
-//
-//      if ((s == pTrans->next.state) && (pSM->prev.evt == pTrans->event)) {// this event has been handled already
-//        stDebug("s-task:%s attached event:%s handled", id, GET_EVT_NAME(pTrans->event));
-//        return TSDB_CODE_SUCCESS;
-//      } else if (s != TASK_STATUS__DROPPING && s != TASK_STATUS__STOP && s != TASK_STATUS__UNINIT) {
-//        stDebug("s-task:%s not handle event:%s yet, wait for 100ms and recheck", id, GET_EVT_NAME(event));
-//        taosMsleep(100);
-//      } else {
-//        stDebug("s-task:%s is dropped or stopped already, not wait.", id);
-//        return TSDB_CODE_STREAM_INVALID_STATETRANS;
-//      }
-//    }
-
   } else {  // override current active trans
     pSM->pActiveTrans = pTrans;
     pSM->startTs = taosGetTimestampMs();
@@ -415,13 +386,13 @@ int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, _
       // no active event trans exists, handle this event directly
       pTrans = streamTaskFindTransform(pSM->current.state, event);
       if (pTrans == NULL) {
-        stDebug("s-task:%s failed to handle event:%s", pTask->id.idStr, GET_EVT_NAME(event));
+        stDebug("s-task:%s failed to handle event:%s, status:%s", pTask->id.idStr, GET_EVT_NAME(event), pSM->current.name);
         taosThreadMutexUnlock(&pTask->lock);
         return TSDB_CODE_STREAM_INVALID_STATETRANS;
       }
 
       if (pSM->pActiveTrans != NULL) {
-        // currently in some state transfer procedure, not auto invoke transfer, quit from this prcedure
+        // currently in some state transfer procedure, not auto invoke transfer, quit from this procedure
         stDebug("s-task:%s event:%s handle procedure quit, status %s -> %s failed, handle event %s now",
                 pTask->id.idStr, GET_EVT_NAME(pSM->pActiveTrans->event), pSM->current.name,
                 pSM->pActiveTrans->next.name, GET_EVT_NAME(event));
@@ -555,7 +526,7 @@ void streamTaskSetStatusReady(SStreamTask* pTask) {
 }
 
 STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStreamTaskEvent event, __state_trans_fn fn,
-                                     __state_trans_succ_fn succFn, SFutureHandleEventInfo* pEventInfo, bool autoInvoke) {
+                                     __state_trans_succ_fn succFn, SFutureHandleEventInfo* pEventInfo) {
   STaskStateTrans trans = {0};
   trans.state = StreamTaskStatusList[current];
   trans.next = StreamTaskStatusList[next];
@@ -570,7 +541,7 @@ STaskStateTrans createStateTransform(ETaskStatus current, ETaskStatus next, EStr
 
   trans.pAction = (fn != NULL) ? fn : dummyFn;
   trans.pSuccAction = (succFn != NULL) ? succFn : dummyFn;
-  trans.autoInvokeEndFn = autoInvoke;
+  trans.autoInvokeEndFn = (fn == NULL);
   return trans;
 }
 
@@ -584,93 +555,93 @@ void doInitStateTransferTable(void) {
   streamTaskSMTrans = taosArrayInit(8, sizeof(STaskStateTrans));
 
   // initialization event handle
-  STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, false, false);
+  STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, false, false);
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
 
   // scan-history related event
-  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__READY, TASK_EVENT_SCANHIST_DONE, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
 
   // halt stream task, from other task status
-  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true);
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
 
   SFutureHandleEventInfo info = {.status = TASK_STATUS__READY, .event = TASK_EVENT_HALT};
 
-  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info, true);
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, &info);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL, true);
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__HALT, TASK_EVENT_HALT, NULL, streamTaskKeepCurrentVerInWal, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
 
   // checkpoint related event
-  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__CK, TASK_EVENT_GEN_CHECKPOINT, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__READY, TASK_EVENT_CHECKPOINT_DONE, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
 
   // pause & resume related event handle
-  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
 
   info = (SFutureHandleEventInfo){.status = TASK_STATUS__READY, .event = TASK_EVENT_PAUSE};
-  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true);
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info, true);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, &info);
   taosArrayPush(streamTaskSMTrans, &trans);
 
-  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__PAUSE, TASK_EVENT_PAUSE, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_PAUSE, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_PAUSE, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
 
   // resume is completed by restore status of state-machine
 
   // stop related event
-  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__STOP, TASK_EVENT_STOP, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
 
   // dropping related event
-  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__READY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__DROPPING, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__STOP, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__SCAN_HISTORY, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__HALT, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__PAUSE, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
-  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, streamTaskSendTransSuccessMsg, NULL, NULL, true);
+  trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL);
   taosArrayPush(streamTaskSMTrans, &trans);
 }
 //clang-format on
\ No newline at end of file

From d3e8adf2ebc3b1e941f29085f7694c1640a94bb7 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 19 Feb 2024 13:36:57 +0800
Subject: [PATCH 045/201] enh(stream): async handle pause event

---
 source/dnode/vnode/src/tqCommon/tqCommon.c |  2 +-
 source/libs/stream/src/streamExec.c        | 13 ++++++++-----
 source/libs/stream/src/streamTask.c        |  8 ++++++--
 3 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index c4973b7c1e..3cdb3d8ecf 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -837,7 +837,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg){
     pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId);
     if (pHistoryTask == NULL) {
       tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64
-                  ", it may have been dropped already",
+              ", it may have been dropped already",
               pMeta->vgId, pTask->hTaskInfo.id.taskId);
       streamMetaReleaseTask(pMeta, pTask);
 
diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 7fb8095acd..a2cf8bdab5 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -410,6 +410,11 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t haltCallback(SStreamTask* pTask, void* param) {
+  streamTaskOpenAllUpstreamInput(pTask);
+  streamTaskSendCheckpointReq(pTask);
+}
+
 int32_t streamTransferStateToStreamTask(SStreamTask* pTask) {
   int32_t code = TSDB_CODE_SUCCESS;
   SStreamMeta* pMeta = pTask->pMeta;
@@ -419,11 +424,12 @@ int32_t streamTransferStateToStreamTask(SStreamTask* pTask) {
   int32_t level = pTask->info.taskLevel;
   if (level == TASK_LEVEL__AGG || level == TASK_LEVEL__SOURCE) {  // do transfer task operator states.
     code = streamDoTransferStateToStreamTask(pTask);
-  } else { // no state transfer for sink tasks, and drop fill-history task, followed by opening inputQ of sink task.
+  } else {
+    // no state transfer for sink tasks, and drop fill-history task, followed by opening inputQ of sink task.
     SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId);
     if (pStreamTask != NULL) {
       // halt the related stream sink task
-      code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
+      code = streamTaskHandleEventAsync(pStreamTask->status.pSM, TASK_EVENT_HALT, haltCallback, NULL);
       if (code != TSDB_CODE_SUCCESS) {
         stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", pTask->id.idStr,
                 pStreamTask->id.idStr, tstrerror(code));
@@ -432,9 +438,6 @@ int32_t streamTransferStateToStreamTask(SStreamTask* pTask) {
       } else {
         stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr);
       }
-
-      streamTaskOpenAllUpstreamInput(pStreamTask);
-      streamTaskSendCheckpointReq(pStreamTask);
       streamMetaReleaseTask(pMeta, pStreamTask);
     }
   }
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index fef733c9f3..bba49e1226 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -847,8 +847,8 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc)
   pDst->chkpointTransId = pSrc->chkpointTransId;
 }
 
-void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) {
-  streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_PAUSE);
+static int32_t taskPauseCallback(SStreamTask* pTask, void* param) {
+  SStreamMeta* pMeta = pTask->pMeta;
 
   int32_t num = atomic_add_fetch_32(&pMeta->numOfPausedTasks, 1);
   stInfo("vgId:%d s-task:%s pause stream task. pause task num:%d", pMeta->vgId, pTask->id.idStr, num);
@@ -862,6 +862,10 @@ void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) {
   stDebug("vgId:%d s-task:%s set pause flag and pause task", pMeta->vgId, pTask->id.idStr);
 }
 
+void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) {
+  streamTaskHandleEventAsync(pTask->status.pSM, TASK_EVENT_PAUSE, taskPauseCallback, NULL);
+}
+
 void streamTaskResume(SStreamTask* pTask) {
   SStreamTaskState prevState = *streamTaskGetStatus(pTask);
   SStreamMeta* pMeta = pTask->pMeta;

From 730edf24f905d77556c12eb9340894078f3601d2 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 19 Feb 2024 13:38:14 +0800
Subject: [PATCH 046/201] fix(stream): fix syntax error.

---
 source/libs/stream/src/streamTask.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index bba49e1226..5c03aced32 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -860,6 +860,7 @@ static int32_t taskPauseCallback(SStreamTask* pTask, void* param) {
   }
 
   stDebug("vgId:%d s-task:%s set pause flag and pause task", pMeta->vgId, pTask->id.idStr);
+  return TSDB_CODE_SUCCESS;
 }
 
 void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) {

From c5b26b406f6e0eb493951734fee87805c2ef829f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 19 Feb 2024 13:38:56 +0800
Subject: [PATCH 047/201] fix(stream): fix syntax error.

---
 source/libs/stream/src/streamExec.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index a2cf8bdab5..8d33eaae62 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -413,6 +413,7 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
 static int32_t haltCallback(SStreamTask* pTask, void* param) {
   streamTaskOpenAllUpstreamInput(pTask);
   streamTaskSendCheckpointReq(pTask);
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t streamTransferStateToStreamTask(SStreamTask* pTask) {

From bf242a4f64d34987f1e1bf942dd65a5babf8e366 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 19 Feb 2024 15:57:21 +0800
Subject: [PATCH 048/201] fix(stream): handle pause event in waiting list.

---
 include/libs/stream/tstream.h         |  2 +-
 source/dnode/snode/src/snode.c        |  1 +
 source/dnode/vnode/src/tq/tq.c        |  1 +
 source/libs/stream/src/streamTask.c   | 21 +++------
 source/libs/stream/src/streamTaskSm.c | 63 ++++++++++++++++++++-------
 5 files changed, 57 insertions(+), 31 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 587e762448..17b87f2355 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -803,7 +803,7 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
 typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param);
 int32_t streamTaskHandleEventAsync(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param);
 int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event, __state_trans_user_fn callbackFn, void* param);
-void    streamTaskRestoreStatus(SStreamTask* pTask);
+int32_t streamTaskRestoreStatus(SStreamTask* pTask);
 
 int32_t streamTaskStop(SStreamTask* pTask);
 int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp,
diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c
index f173c327c7..3a72146a41 100644
--- a/source/dnode/snode/src/snode.c
+++ b/source/dnode/snode/src/snode.c
@@ -87,6 +87,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
   // checkpoint ver is the kept version, handled data should be the next version.
   if (pTask->chkInfo.checkpointId != 0) {
     pTask->chkInfo.nextProcessVer = pTask->chkInfo.checkpointVer + 1;
+    pChkInfo->processedVer = pChkInfo->checkpointVer;
     sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64,
             pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
   }
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 5ac44cbbae..81d9a9f13f 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -835,6 +835,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
   // checkpoint ver is the kept version, handled data should be the next version.
   if (pChkInfo->checkpointId != 0) {
     pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1;
+    pChkInfo->processedVer = pChkInfo->checkpointVer;
     tqInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr,
            pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
   }
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 5c03aced32..aab2e6ab95 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -636,8 +636,6 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE
       stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpSet:%s", pTask->id.taskId,
               pDispatcher->taskId, nodeId, buf);
     }
-  } else {
-    // do nothing
   }
 }
 
@@ -869,20 +867,15 @@ void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask) {
 
 void streamTaskResume(SStreamTask* pTask) {
   SStreamTaskState prevState = *streamTaskGetStatus(pTask);
+
   SStreamMeta* pMeta = pTask->pMeta;
-
-  if (prevState.state == TASK_STATUS__PAUSE || prevState.state == TASK_STATUS__HALT) {
-    streamTaskRestoreStatus(pTask);
-
-    char* pNew = streamTaskGetStatus(pTask)->name;
-    if (prevState.state == TASK_STATUS__PAUSE) {
-      int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
-      stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num);
-    } else {
-      stInfo("s-task:%s status:%s resume from %s", pTask->id.idStr, pNew, prevState.name);
-    }
+  int32_t      code = streamTaskRestoreStatus(pTask);
+  if (code == TSDB_CODE_SUCCESS) {
+    char*   pNew = streamTaskGetStatus(pTask)->name;
+    int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1);
+    stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num);
   } else {
-    stDebug("s-task:%s status:%s not in pause/halt status, no need to resume", pTask->id.idStr, prevState.name);
+    stInfo("s-task:%s status:%s no need to resume, paused task(s):%d", pTask->id.idStr, prevState.name, pMeta->numOfPausedTasks);
   }
 }
 
diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c
index 9ca5248157..6aa215586a 100644
--- a/source/libs/stream/src/streamTaskSm.c
+++ b/source/libs/stream/src/streamTaskSm.c
@@ -196,30 +196,61 @@ static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName,
   return code;
 }
 
-void streamTaskRestoreStatus(SStreamTask* pTask) {
+static int32_t removeEventInWaitingList(SStreamTask* pTask, EStreamTaskEvent event) {
   SStreamTaskSM* pSM = pTask->status.pSM;
 
+  bool removed = false;
   taosThreadMutexLock(&pTask->lock);
 
-  ASSERT(pSM->pActiveTrans == NULL);
-  ASSERT(pSM->current.state == TASK_STATUS__PAUSE || pSM->current.state == TASK_STATUS__HALT);
+  int32_t num = taosArrayGetSize(pSM->pWaitingEventList);
+  for (int32_t i = 0; i < num; ++i) {
+    SFutureHandleEventInfo* pInfo = taosArrayGet(pSM->pWaitingEventList, i);
+    if (pInfo->event == event) {
+      taosArrayRemove(pSM->pWaitingEventList, i);
+      stDebug("s-task:%s pause event in waiting list not be handled yet, remove it from waiting list, remaining:%d",
+              pTask->id.idStr, pInfo->event);
+      removed = true;
+      break;
+    }
+  }
 
-  SStreamTaskState state = pSM->current;
-  pSM->current = pSM->prev.state;
-
-  pSM->prev.state = state;
-  pSM->prev.evt = 0;
-
-  pSM->startTs = taosGetTimestampMs();
-
-  if (taosArrayGetSize(pSM->pWaitingEventList) > 0) {
-    stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr, pSM->prev.state.name, pSM->current.name);
-    doHandleWaitingEvent(pSM, "restore-pause/halt", pTask);
-  } else {
-    stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name);
+  if (!removed) {
+    stDebug("s-task:%s failed to remove event:%s in waiting list", pTask->id.idStr, StreamTaskEventList[event].name);
   }
 
   taosThreadMutexUnlock(&pTask->lock);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t streamTaskRestoreStatus(SStreamTask* pTask) {
+  SStreamTaskSM* pSM = pTask->status.pSM;
+  int32_t code = 0;
+
+  taosThreadMutexLock(&pTask->lock);
+
+  if (pSM->current.state == TASK_STATUS__PAUSE && pSM->pActiveTrans == NULL) {
+    SStreamTaskState state = pSM->current;
+    pSM->current = pSM->prev.state;
+
+    pSM->prev.state = state;
+    pSM->prev.evt = 0;
+
+    pSM->startTs = taosGetTimestampMs();
+
+    if (taosArrayGetSize(pSM->pWaitingEventList) > 0) {
+      stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr,
+              pSM->prev.state.name, pSM->current.name);
+      doHandleWaitingEvent(pSM, "restore-pause/halt", pTask);
+    } else {
+      stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name);
+    }
+  } else {
+    removeEventInWaitingList(pTask, TASK_EVENT_PAUSE);
+    code = -1;  // failed to restore the status
+  }
+
+  taosThreadMutexUnlock(&pTask->lock);
+  return code;
 }
 
 SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask) {

From 9e1446aa88fbb74340ad6f30ec6b97d990c3fa6e Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 22 Feb 2024 10:03:29 +0800
Subject: [PATCH 049/201] fix: initialize result block

---
 source/libs/executor/src/scanoperator.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 68743a9d7a..496d2a9f43 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4348,6 +4348,9 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     pInfo->mSkipTables = NULL;
   }
 
+  initResultSizeInfo(&pOperator->resultInfo, 1024);
+  pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
+  blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
   if (!hasLimit && blockDataGetRowSize(pInfo->pResBlock) >= 256) {
     pInfo->bSortRowId = true;
   } else {
@@ -4355,9 +4358,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   }
 
   
-  initResultSizeInfo(&pOperator->resultInfo, 1024);
-  pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
-  blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
 
   pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
   pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false);

From b5bc45c73baf12de8586003ce7416fccedb10f86 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 22 Feb 2024 11:21:44 +0800
Subject: [PATCH 050/201] fix: save to ext mem file only when sort by row id

---
 source/libs/executor/src/tsort.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 363125c216..b313e46270 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1423,8 +1423,9 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray*
     }
     blockDataCleanup(pHandle->pDataBlock);
   }
-
-  saveDirtyPagesToExtRowsMemFile(pHandle);
+  if (pHandle->bSortByRowId) {
+    saveDirtyPagesToExtRowsMemFile(pHandle);
+  }
 
   SSDataBlock* pMemSrcBlk = createOneDataBlock(pHandle->pDataBlock, false);
   doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId);

From 3341d02060e221f7b1537a7527daf4f09ac98cee Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Thu, 22 Feb 2024 17:03:17 +0800
Subject: [PATCH 051/201] fix: don't copy all data when use limit

---
 source/common/src/tdatablock.c      | 1 +
 source/libs/executor/src/executor.c | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 9439c172c4..690eda17e1 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1688,6 +1688,7 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) {
 
 static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
+    pColInfoData->varmeta.length = pColInfoData->varmeta.offset[n];
     // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
     memset(&pColInfoData->varmeta.offset[n], 0, total - n);
   }
diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 2534c5e9f0..cbf0d38bd9 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -648,7 +648,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
     ASSERT(p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT);
     taosArrayPush(pResList, &p);
 
-    if (current >= rowsThreshold) {
+    if (current >= rowsThreshold || current >= pRes->info.rows) {
       break;
     }
 

From 0ae2e2905d2f0a15d56da7866ec3167bd2b0050b Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 23 Feb 2024 08:24:16 +0800
Subject: [PATCH 052/201] fix: change fseek to fseeko for 64 bit offset

---
 source/libs/executor/src/tsort.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index b313e46270..68b3bd2c48 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1036,7 +1036,7 @@ static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char*
       ++pMemFile->numMemPages;
     }
     {
-      fseek(pMemFile->pTdFile, pageId * pMemFile->pageSize, SEEK_SET);
+      fseeko(pMemFile->pTdFile, ((int64_t)pageId) * pMemFile->pageSize, SEEK_SET);
       fread(pEntry->data, pMemFile->pageSize, 1, pMemFile->pTdFile);      
       SSortMemPageEntry* tail = pMemFile->pagesTail;
       tail->next = pEntry;

From 3fb6ed8a426ab08f95c06cc2279b4240696e4f07 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 23 Feb 2024 08:35:01 +0800
Subject: [PATCH 053/201] feat: free write buf after reading all blocks

---
 source/libs/executor/src/tsort.c | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 68b3bd2c48..7f44b5e433 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -127,6 +127,7 @@ static int32_t destroySortMemFile(SSortHandle* pHandle);
 static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char** ppPage);
 static void    setExtMemFilePageUnused(SSortMemFile* pMemFile, int32_t pageId);
 static int32_t saveDirtyPagesToExtRowsMemFile(SSortHandle* pHandle);
+static int32_t freeExtRowMemFileWriteBuf(SSortHandle* pHandle);
 
 void tsortSetSingleTableMerge(SSortHandle* pHandle) {
   pHandle->singleTableMerge = true;
@@ -1116,7 +1117,11 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
     taosMemoryFree(pCurr);
   }
   tSimpleHashCleanup(pMemFile->mActivePages);
+  pMemFile->mActivePages = NULL;
+
   taosMemoryFree(pMemFile->writePageBuf);
+  pMemFile->writePageBuf = NULL;
+
   fclose(pMemFile->pTdFile);
   taosRemoveFile(pMemFile->memFilePath);
   taosMemoryFree(pMemFile);
@@ -1168,6 +1173,17 @@ static int32_t saveDirtyPagesToExtRowsMemFile(SSortHandle* pHandle) {
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t freeExtRowMemFileWriteBuf(SSortHandle* pHandle) {
+  SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
+
+  if (pMemFile == NULL) return TSDB_CODE_SUCCESS;
+
+  taosMemoryFree(pMemFile->writePageBuf);
+  pMemFile->writePageBuf = NULL;
+  taosMemoryTrim(0);
+  return TSDB_CODE_SUCCESS;
+}
+
 static void appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSource, int32_t* rowIndex) {
   int32_t pageId = -1;
   int32_t offset = -1;
@@ -1585,6 +1601,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   }
   taosArrayDestroy(aExtSrc);
   tSimpleHashCleanup(mTableNumRows);
+  if (pHandle->bSortByRowId) {
+    freeExtRowMemFileWriteBuf(pHandle);
+  }
   pHandle->type = SORT_SINGLESOURCE_SORT;
   return TSDB_CODE_SUCCESS;
 }

From 754a15cac8d6f42166b0bb41dfbadc116882d23d Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 23 Feb 2024 10:55:19 +0800
Subject: [PATCH 054/201] feat: change from FILE* to TDFilePtr

---
 source/libs/executor/src/tsort.c | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 7f44b5e433..8ed255313e 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -63,7 +63,8 @@ typedef struct SSortMemFile {
   int32_t numMemPages;
   SSHashObj* mActivePages;
 
-  FILE* pTdFile;
+  //FILE* pTdFile;
+  TdFilePtr pTdFile;
   char memFilePath[PATH_MAX];
 } SSortMemFile;
 
@@ -1037,8 +1038,12 @@ static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char*
       ++pMemFile->numMemPages;
     }
     {
-      fseeko(pMemFile->pTdFile, ((int64_t)pageId) * pMemFile->pageSize, SEEK_SET);
-      fread(pEntry->data, pMemFile->pageSize, 1, pMemFile->pTdFile);      
+      // fseeko(pMemFile->pTdFile, ((int64_t)pageId) * pMemFile->pageSize, SEEK_SET);
+      // fread(pEntry->data, pMemFile->pageSize, 1, pMemFile->pTdFile);
+
+      taosLSeekFile(pMemFile->pTdFile, ((int64_t)pageId) * pMemFile->pageSize, SEEK_SET);
+      taosReadFile(pMemFile->pTdFile, pEntry->data, pMemFile->pageSize);
+
       SSortMemPageEntry* tail = pMemFile->pagesTail;
       tail->next = pEntry;
       pEntry->next = NULL;
@@ -1082,7 +1087,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
 
   taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
   pMemFile->pTdFile =
-      fopen(pMemFile->memFilePath, "wb+");
+      taosOpenFile(pMemFile->memFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
   pMemFile->currPageId = -1;
   pMemFile->currPageOffset = -1;
 
@@ -1122,7 +1127,8 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
   taosMemoryFree(pMemFile->writePageBuf);
   pMemFile->writePageBuf = NULL;
 
-  fclose(pMemFile->pTdFile);
+  // fclose(pMemFile->pTdFile);
+  taosCloseFile(&pMemFile->pTdFile);
   taosRemoveFile(pMemFile->memFilePath);
   taosMemoryFree(pMemFile);
   pHandle->pExtRowsMemFile = NULL;
@@ -1142,9 +1148,10 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p
       pMemFile->currPageOffset = 0;
 
       if (pMemFile->currPageId - pMemFile->startPageId >= pMemFile->numWritePages) {
-        fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
-        fwrite(pMemFile->writePageBuf, pMemFile->pageSize * pMemFile->numWritePages, 1, pMemFile->pTdFile);
-
+        // fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
+        // fwrite(pMemFile->writePageBuf, pMemFile->pageSize * pMemFile->numWritePages, 1, pMemFile->pTdFile);
+        taosLSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
+        taosWriteFile(pMemFile->pTdFile, pMemFile->writePageBuf, pMemFile->pageSize * pMemFile->numWritePages);
         pMemFile->startPageId = pMemFile->currPageId;
       }
     }
@@ -1166,9 +1173,11 @@ static int32_t saveDirtyPagesToExtRowsMemFile(SSortHandle* pHandle) {
   if (!pMemFile->bDirty) {
     return TSDB_CODE_SUCCESS;
   }
-  fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
+  // fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
+  taosLSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
   int32_t numWriteBytes = pMemFile->pageSize * (pMemFile->currPageId - pMemFile->startPageId) + pMemFile->currPageOffset + 1;
-  fwrite(pMemFile->writePageBuf, numWriteBytes, 1, pMemFile->pTdFile);
+  // fwrite(pMemFile->writePageBuf, numWriteBytes, 1, pMemFile->pTdFile);
+  taosWriteFile(pMemFile->pTdFile, pMemFile->writePageBuf, numWriteBytes);
   pMemFile->bDirty = false;
   return TSDB_CODE_SUCCESS;
 }

From 830de66111f24b08d687ccc809af8b63bae9cb52 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 23 Feb 2024 13:55:10 +0800
Subject: [PATCH 055/201] fix: compilation error and force sort by row id for
 CI

---
 source/libs/executor/src/scanoperator.c | 2 +-
 source/libs/executor/src/tsort.c        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index b71d63a22d..43a716a079 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4351,7 +4351,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     pInfo->bSortRowId = false;
   }
 
-  
+  pInfo->bSortRowId = true;
 
   pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
   pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false);
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index fbde32a701..e1cd8b822f 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1543,8 +1543,8 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
     if (pBlk != NULL) {
       SColumnInfoData* tsCol = taosArrayGet(pBlk->pDataBlock, pOrigOrder->slotId);
       int64_t firstRowTs = *(int64_t*)tsCol->pData;
-      if ((pOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs)  ||
-          (pOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) {
+      if ((pOrigOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs)  ||
+          (pOrigOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) {
         if (bExtractedBlock) {
           blockDataDestroy(pBlk);
 	      }

From c54e4aeaadc69e0104ce31d091826653a70dc081 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Fri, 23 Feb 2024 09:30:53 +0000
Subject: [PATCH 056/201] add test case

---
 source/libs/index/src/indexComm.c             | 11 ++-
 source/libs/stream/inc/streamBackendRocksdb.h |  7 +-
 source/libs/stream/src/streamMeta.c           | 28 +++---
 source/libs/stream/test/CMakeLists.txt        | 94 ++++++++++++++-----
 source/libs/stream/test/backendTest.cpp       | 38 ++++++++
 source/libs/stream/test/tstreamUpdateTest.cpp | 13 ++-
 6 files changed, 137 insertions(+), 54 deletions(-)
 create mode 100644 source/libs/stream/test/backendTest.cpp

diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c
index 1313221952..b7b9f1cc9f 100644
--- a/source/libs/index/src/indexComm.c
+++ b/source/libs/index/src/indexComm.c
@@ -76,8 +76,8 @@ char* idxInt2str(int64_t val, char* dst, int radix) {
   return dst - 1;
 }
 __compar_fn_t idxGetCompar(int8_t type) {
-  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY ||
-      type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) {
+  if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_NCHAR ||
+      type == TSDB_DATA_TYPE_GEOMETRY) {
     return (__compar_fn_t)strcmp;
   }
   return getComparFunc(type, 0);
@@ -108,8 +108,8 @@ static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) {
   return tCompare(func, QUERY_TERM, a, b, type);
 }
 TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) {
-  if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR ||
-      dtype == TSDB_DATA_TYPE_VARBINARY || dtype == TSDB_DATA_TYPE_GEOMETRY) {
+  if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY ||
+      dtype == TSDB_DATA_TYPE_GEOMETRY) {
     return tDoCompare(func, cmptype, a, b);
   }
 #if 1
@@ -290,6 +290,7 @@ int idxUidCompare(const void* a, const void* b) {
   uint64_t r = *(uint64_t*)b;
   return l - r;
 }
+#ifdef BUILD_NO_CALL
 int32_t idxConvertData(void* src, int8_t type, void** dst) {
   int tlen = -1;
   switch (type) {
@@ -372,6 +373,8 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) {
   // indexMayFillNumbericData(*dst, tlen);
   return tlen;
 }
+#endif
+
 int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) {
   if (src == NULL) {
     *dst = strndup(INDEX_DATA_NULL_STR, (int)strlen(INDEX_DATA_NULL_STR));
diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h
index 03f70604b7..1f8e99cd27 100644
--- a/source/libs/stream/inc/streamBackendRocksdb.h
+++ b/source/libs/stream/inc/streamBackendRocksdb.h
@@ -17,7 +17,7 @@
 #define _STREAM_BACKEDN_ROCKSDB_H_
 
 #include "rocksdb/c.h"
-//#include "streamInt.h"
+// #include "streamInt.h"
 #include "streamState.h"
 #include "tcommon.h"
 
@@ -244,11 +244,6 @@ int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId);
 
 int32_t taskDbBuildSnap(void* arg, SArray* pSnap);
 
-// int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result);
-
-// STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId);
-// void            taskDbDestroy(void* pDb, bool flush);
-
 int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId);
 
 SBkdMgt* bkdMgtCreate(char* path);
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index b35f401cb9..a09b940a19 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -70,7 +70,7 @@ static void streamMetaEnvInit() {
   streamTimerInit();
 }
 
-void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit);}
+void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); }
 
 void streamMetaCleanup() {
   taosCloseRef(streamBackendId);
@@ -1104,14 +1104,14 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) {
         .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputq.queue)),
     };
 
-    entry.inputRate = entry.inputQUsed * 100.0 / (2*STREAM_TASK_QUEUE_CAPACITY_IN_SIZE);
+    entry.inputRate = entry.inputQUsed * 100.0 / (2 * STREAM_TASK_QUEUE_CAPACITY_IN_SIZE);
     if ((*pTask)->info.taskLevel == TASK_LEVEL__SINK) {
       entry.sinkQuota = (*pTask)->outputInfo.pTokenBucket->quotaRate;
       entry.sinkDataSize = SIZE_IN_MiB((*pTask)->execInfo.sink.dataSize);
     }
 
     if ((*pTask)->chkInfo.checkpointingId != 0) {
-      entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId)? 1:0;
+      entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId) ? 1 : 0;
       entry.checkpointId = (*pTask)->chkInfo.checkpointingId;
       entry.chkpointTransId = (*pTask)->chkInfo.transId;
 
@@ -1172,7 +1172,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) {
     stDebug("vgId:%d no tasks and no mnd epset, not send stream hb to mnode", pMeta->vgId);
   }
 
-  _end:
+_end:
   streamMetaClearHbMsg(&hbMsg);
   return TSDB_CODE_SUCCESS;
 }
@@ -1304,28 +1304,28 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) {
 }
 
 void streamMetaRLock(SStreamMeta* pMeta) {
-//  stTrace("vgId:%d meta-rlock", pMeta->vgId);
+  //  stTrace("vgId:%d meta-rlock", pMeta->vgId);
   taosThreadRwlockRdlock(&pMeta->lock);
 }
 
 void streamMetaRUnLock(SStreamMeta* pMeta) {
-//  stTrace("vgId:%d meta-runlock", pMeta->vgId);
+  //  stTrace("vgId:%d meta-runlock", pMeta->vgId);
   int32_t code = taosThreadRwlockUnlock(&pMeta->lock);
   if (code != TSDB_CODE_SUCCESS) {
     stError("vgId:%d meta-runlock failed, code:%d", pMeta->vgId, code);
   } else {
-//    stTrace("vgId:%d meta-runlock completed", pMeta->vgId);
+    //    stTrace("vgId:%d meta-runlock completed", pMeta->vgId);
   }
 }
 
 void streamMetaWLock(SStreamMeta* pMeta) {
-//  stTrace("vgId:%d meta-wlock", pMeta->vgId);
+  //  stTrace("vgId:%d meta-wlock", pMeta->vgId);
   taosThreadRwlockWrlock(&pMeta->lock);
-//  stTrace("vgId:%d meta-wlock completed", pMeta->vgId);
+  //  stTrace("vgId:%d meta-wlock completed", pMeta->vgId);
 }
 
 void streamMetaWUnLock(SStreamMeta* pMeta) {
-//  stTrace("vgId:%d meta-wunlock", pMeta->vgId);
+  //  stTrace("vgId:%d meta-wunlock", pMeta->vgId);
   taosThreadRwlockUnlock(&pMeta->lock);
 }
 
@@ -1395,7 +1395,7 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader)
     pMeta->sendMsgBeforeClosing = true;
   }
 
-  pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER;
+  pMeta->role = (isLeader) ? NODE_ROLE_LEADER : NODE_ROLE_FOLLOWER;
   streamMetaWUnLock(pMeta);
 
   if (isLeader) {
@@ -1531,8 +1531,8 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) {
 
 bool streamMetaAllTasksReady(const SStreamMeta* pMeta) {
   int32_t num = taosArrayGetSize(pMeta->pTaskList);
-  for(int32_t i = 0; i < num; ++i) {
-    STaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i);
+  for (int32_t i = 0; i < num; ++i) {
+    STaskId*      pTaskId = taosArrayGet(pMeta->pTaskList, i);
     SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pTaskId, sizeof(*pTaskId));
     if (ppTask == NULL) {
       continue;
@@ -1633,7 +1633,7 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
     pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0;
 
     stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x (succ:%d) startTs:%" PRId64
-                ", readyTs:%" PRId64 " total elapsed time:%.2fs",
+            ", readyTs:%" PRId64 " total elapsed time:%.2fs",
             pMeta->vgId, numOfTotal, taskId, ready, pStartInfo->startTs, pStartInfo->readyTs,
             pStartInfo->elapsedTime / 1000.0);
 
diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt
index c90e05bcf6..c18372a493 100644
--- a/source/libs/stream/test/CMakeLists.txt
+++ b/source/libs/stream/test/CMakeLists.txt
@@ -1,40 +1,82 @@
 
-MESSAGE(STATUS "build stream unit test")
-
-# GoogleTest requires at least C++11
-SET(CMAKE_CXX_STANDARD 11)
-AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
 # bloomFilterTest
-ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp")
 
-TARGET_LINK_LIBRARIES(streamUpdateTest
-        PUBLIC os util common gtest gtest_main stream executor index
-        )
+#TARGET_LINK_LIBRARIES(streamUpdateTest
+        #PUBLIC os util common gtest gtest_main stream executor index
+        #)
 
-TARGET_INCLUDE_DIRECTORIES(
-  streamUpdateTest
+#TARGET_INCLUDE_DIRECTORIES(
+  #streamUpdateTest
+  #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
+  #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
+#)
+
+#ADD_EXECUTABLE(checkpointTest checkpointTest.cpp)
+#TARGET_LINK_LIBRARIES(
+        #checkpointTest
+        #PUBLIC os common gtest stream executor qcom index transport util
+#)
+
+#TARGET_INCLUDE_DIRECTORIES(
+        #checkpointTest
+        #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
+#)
+
+#add_executable(backendTest "")
+
+#target_sources(backendTest 
+        #PRIVATE
+        #"backendTest.cpp"
+#)
+
+#TARGET_LINK_LIBRARIES(
+        #backendTest
+        #PUBLIC rocksdb
+        #PUBLIC os common gtest stream executor qcom index transport util
+#)
+
+#TARGET_INCLUDE_DIRECTORIES(
+        #backendTest
+        #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
+        #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
+#)
+
+#add_test(
+  #NAME streamUpdateTest
+  #COMMAND streamUpdateTest
+#)
+
+#add_test(
+  #NAME checkpointTest
+  #COMMAND checkpointTest
+#)
+#add_test(
+  #NAME backendTest
+  #COMMAND backendTest
+#)
+
+
+add_executable(backendTest "")
+
+target_sources(backendTest
+  PRIVATE
+  "backendTest.cpp"
+)
+
+target_include_directories(
+  backendTest
   PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
   PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
 )
 
-ADD_EXECUTABLE(checkpointTest checkpointTest.cpp)
-TARGET_LINK_LIBRARIES(
-        checkpointTest
+target_link_libraries(
+        backendTest
+        PUBLIC rocksdb
         PUBLIC os common gtest stream executor qcom index transport util
 )
 
-TARGET_INCLUDE_DIRECTORIES(
-        checkpointTest
-        PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
-)
-
 add_test(
-  NAME streamUpdateTest
-  COMMAND streamUpdateTest
-)
-
-add_test(
-  NAME checkpointTest
-  COMMAND checkpointTest
+  NAME backendTest 
+  COMMAND backendTest 
 )
\ No newline at end of file
diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
new file mode 100644
index 0000000000..5051337aee
--- /dev/null
+++ b/source/libs/stream/test/backendTest.cpp
@@ -0,0 +1,38 @@
+#include <gtest/gtest.h>
+
+#include <taoserror.h>
+#include <tglobal.h>
+#include <iostream>
+#include "streamBackendRocksdb.h"
+
+class BackendEnv : public ::testing::Test {
+ protected:
+  virtual void SetUp() {}
+  virtual void TearDown() {}
+};
+
+void *backendCreate() {
+  const char *streamPath = "/tmp";
+
+  char *absPath = NULL;
+  void *p = NULL;
+  // SBackendWrapper *p = streamBackendInit(streamPath, -1, 2);
+  // p = taskDbOpen((char *)streamPath, (char *)"test", -1);
+  // p = bkdMgtCreate((char *)streamPath);
+
+  ASSERT(p != NULL);
+  return p;
+}
+void backendOpen() {
+  void *p = backendCreate();
+  ASSERT(p != NULL);
+}
+
+TEST_F(BackendEnv, checkOpen) { backendOpen(); }
+TEST_F(BackendEnv, backendOpt) {}
+TEST_F(BackendEnv, backendDestroy) {}
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
\ No newline at end of file
diff --git a/source/libs/stream/test/tstreamUpdateTest.cpp b/source/libs/stream/test/tstreamUpdateTest.cpp
index 1b999e5fb0..59171876ff 100644
--- a/source/libs/stream/test/tstreamUpdateTest.cpp
+++ b/source/libs/stream/test/tstreamUpdateTest.cpp
@@ -14,10 +14,7 @@ class StreamStateEnv : public ::testing::Test {
     streamMetaInit();
     backend = streamBackendInit(path, 0, 0);
   }
-  virtual void TearDown() {
-    streamMetaCleanup();
-    // indexClose(index);
-  }
+  virtual void TearDown() { streamMetaCleanup(); }
 
   const char *path = TD_TMP_DIR_PATH "stream";
   void       *backend;
@@ -50,6 +47,14 @@ bool equalSBF(SScalableBf *left, SScalableBf *right) {
 }
 
 TEST(TD_STREAM_UPDATE_TEST, update) {
+  const char *streamPath = "/tmp";
+
+  char *absPath = NULL;
+  void *p = NULL;
+  // SBackendWrapper *p = streamBackendInit(streamPath, -1, 2);
+  // p = taskDbOpen((char *)streamPath, (char *)"test", -1);
+  p = bkdMgtCreate((char *)streamPath);
+
   // const int64_t interval = 20 * 1000;
   // const int64_t watermark = 10 * 60 * 1000;
   // SUpdateInfo  *pSU = updateInfoInit(interval, TSDB_TIME_PRECISION_MILLI, watermark);

From 5647d990b40caf1c5d70d3f3154b1bf89626c18c Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Sun, 25 Feb 2024 21:37:33 +0800
Subject: [PATCH 057/201] fix: colDataAssign

---
 source/common/src/tdatablock.c | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 690eda17e1..1e566ccfbe 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -452,20 +452,31 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
   }
 
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
+    int32_t newLen = pSource->varmeta.offset[numOfRows - 1];
+    if (newLen != -1) {
+      if (pSource->info.type == TSDB_DATA_TYPE_JSON) {
+        newLen += getJsonValueLen(pSource->pData + newLen);
+      } else {
+        newLen += varDataTLen(pSource->pData + newLen);
+      }
+    } else {
+      newLen = pSource->varmeta.length;
+    }
+
     memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows);
-    if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) {
-      char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length);
+    if (pColumnInfoData->varmeta.allocLen < newLen) {
+      char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newLen);
       if (tmp == NULL) {
         return TSDB_CODE_OUT_OF_MEMORY;
       }
 
       pColumnInfoData->pData = tmp;
-      pColumnInfoData->varmeta.allocLen = pSource->varmeta.length;
+      pColumnInfoData->varmeta.allocLen = newLen;
     }
 
-    pColumnInfoData->varmeta.length = pSource->varmeta.length;
+    pColumnInfoData->varmeta.length = newLen;
     if (pColumnInfoData->pData != NULL && pSource->pData != NULL) {
-      memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length);
+      memcpy(pColumnInfoData->pData, pSource->pData, newLen);
     }
   } else {
     memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows));
@@ -1688,7 +1699,7 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) {
 
 static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
-    pColInfoData->varmeta.length = pColInfoData->varmeta.offset[n];
+    // pColInfoData->varmeta.length = pColInfoData->varmeta.offset[n];
     // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
     memset(&pColInfoData->varmeta.offset[n], 0, total - n);
   }

From 8230b238fb8dbb5f375bd6855d2d58cba235afb4 Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Sun, 25 Feb 2024 22:56:32 +0800
Subject: [PATCH 058/201] fix: keepFirstNRows

---
 source/common/src/tdatablock.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 1e566ccfbe..6b6c3642de 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -452,17 +452,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
   }
 
   if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
-    int32_t newLen = pSource->varmeta.offset[numOfRows - 1];
-    if (newLen != -1) {
-      if (pSource->info.type == TSDB_DATA_TYPE_JSON) {
-        newLen += getJsonValueLen(pSource->pData + newLen);
-      } else {
-        newLen += varDataTLen(pSource->pData + newLen);
-      }
-    } else {
-      newLen = pSource->varmeta.length;
-    }
-
+    int32_t newLen = pSource->varmeta.length;
     memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows);
     if (pColumnInfoData->varmeta.allocLen < newLen) {
       char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newLen);
@@ -1699,7 +1689,17 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) {
 
 static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
-    // pColInfoData->varmeta.length = pColInfoData->varmeta.offset[n];
+    int32_t newLen = pColInfoData->varmeta.offset[n - 1];
+    if (newLen != -1) {
+      if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
+        newLen += getJsonValueLen(pColInfoData->pData + newLen);
+      } else {
+        newLen += varDataTLen(pColInfoData->pData + newLen);
+      }
+    } else {
+      newLen = pColInfoData->varmeta.length;
+    }
+    pColInfoData->varmeta.length = newLen;
     // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
     memset(&pColInfoData->varmeta.offset[n], 0, total - n);
   }

From fc4caa23b5fb1221a83c63e7d7ebe07b911e8af5 Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Mon, 26 Feb 2024 13:52:10 +0800
Subject: [PATCH 059/201] fix: cache

---
 source/libs/executor/src/executor.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index cbf0d38bd9..2534c5e9f0 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -648,7 +648,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
     ASSERT(p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT);
     taosArrayPush(pResList, &p);
 
-    if (current >= rowsThreshold || current >= pRes->info.rows) {
+    if (current >= rowsThreshold) {
       break;
     }
 

From dc6d96da70aceffaa68bb16077da1a52cf4ccf9f Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Mon, 26 Feb 2024 16:45:15 +0800
Subject: [PATCH 060/201] fix: remove taosMemoryTrim when free write buf

---
 source/libs/executor/src/tsort.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index e1cd8b822f..a779a278cd 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1189,7 +1189,6 @@ static int32_t freeExtRowMemFileWriteBuf(SSortHandle* pHandle) {
 
   taosMemoryFree(pMemFile->writePageBuf);
   pMemFile->writePageBuf = NULL;
-  taosMemoryTrim(0);
   return TSDB_CODE_SUCCESS;
 }
 

From e41e10bf35534fb6b6804f8f79ca356f31fc6c3e Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 26 Feb 2024 11:51:46 +0000
Subject: [PATCH 061/201] add  stream backend  test

---
 source/libs/stream/inc/streamBackendRocksdb.h | 11 +++-
 source/libs/stream/test/CMakeLists.txt        | 60 +++++++++++++------
 source/libs/stream/test/backendTest.cpp       | 30 ++++++++--
 source/libs/stream/test/checkpointTest.cpp    | 28 ++++-----
 4 files changed, 89 insertions(+), 40 deletions(-)

diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h
index 1f8e99cd27..6e6f85d8f9 100644
--- a/source/libs/stream/inc/streamBackendRocksdb.h
+++ b/source/libs/stream/inc/streamBackendRocksdb.h
@@ -21,6 +21,10 @@
 #include "streamState.h"
 #include "tcommon.h"
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef struct SCfComparator {
   rocksdb_comparator_t** comp;
   int32_t                numOfComp;
@@ -253,4 +257,9 @@ int32_t  bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname);
 void     bkdMgtDestroy(SBkdMgt* bm);
 
 int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** path, SArray* list);
-#endif
\ No newline at end of file
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt
index c18372a493..c472207b27 100644
--- a/source/libs/stream/test/CMakeLists.txt
+++ b/source/libs/stream/test/CMakeLists.txt
@@ -57,26 +57,48 @@
 #)
 
 
-add_executable(backendTest "")
+#add_executable(backendTest "")
 
-target_sources(backendTest
-  PRIVATE
-  "backendTest.cpp"
-)
+#target_sources(backendTest
+  #PUBLIC 
+  #"backendTest.cpp"
+#)
 
-target_include_directories(
-  backendTest
-  PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
-  PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
-)
+#target_include_directories(
+  #backendTest
+  #PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
+  #PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
+#)
 
-target_link_libraries(
-        backendTest
-        PUBLIC rocksdb
-        PUBLIC os common gtest stream executor qcom index transport util
-)
+#target_link_libraries(
+        #backendTest
+        #PUBLIC rocksdb
+        #PUBLIC os common gtest stream executor qcom index transport util
+#)
 
-add_test(
-  NAME backendTest 
-  COMMAND backendTest 
-)
\ No newline at end of file
+
+MESSAGE(STATUS "build parser unit test")
+
+IF(NOT TD_DARWIN)
+        # GoogleTest requires at least C++11
+        SET(CMAKE_CXX_STANDARD 11)
+        AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
+
+        ADD_EXECUTABLE(backendTest ${SOURCE_LIST})
+        TARGET_LINK_LIBRARIES(
+          backendTest
+          PUBLIC rocksdb
+          PUBLIC os common gtest stream executor qcom index transport util vnode
+        )
+
+        TARGET_INCLUDE_DIRECTORIES(
+                backendTest 
+                PUBLIC "${TD_SOURCE_DIR}/include/libs/stream/"
+                PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc"
+        )
+
+        ADD_TEST(
+          NAME backendTest 
+          COMMAND backendTest 
+        )
+ENDIF ()
\ No newline at end of file
diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index 5051337aee..d70f16b461 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -4,6 +4,11 @@
 #include <tglobal.h>
 #include <iostream>
 #include "streamBackendRocksdb.h"
+#include "streamSnapshot.h"
+#include "streamState.h"
+#include "tstream.h"
+#include "tstreamFileState.h"
+#include "tstreamUpdate.h"
 
 class BackendEnv : public ::testing::Test {
  protected:
@@ -13,22 +18,35 @@ class BackendEnv : public ::testing::Test {
 
 void *backendCreate() {
   const char *streamPath = "/tmp";
+  void       *p = NULL;
 
-  char *absPath = NULL;
-  void *p = NULL;
-  // SBackendWrapper *p = streamBackendInit(streamPath, -1, 2);
-  // p = taskDbOpen((char *)streamPath, (char *)"test", -1);
-  // p = bkdMgtCreate((char *)streamPath);
+  // char *absPath = NULL;
+  // // SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(streamPath, -1, 2);
+  // STaskDbWrapper *p = taskDbOpen((char *)streamPath, (char *)"stream-backend", -1);
+  // ASSERT(p != NULL);
+  return p;
+}
 
+SStreamState *stateCreate(void *pBackend, char *keyidr) {
+  const char  *streamPath = "/tmp";
+  SStreamTask *pTask = (SStreamTask *)taosMemoryCalloc(1, sizeof(SStreamTask));
+  pTask->ver = 1024;
+  pTask->id.streamId = 1023;
+  pTask->id.taskId = 1111111;
+
+  SStreamState *p = streamStateOpen((char *)streamPath, pTask, true, 32, 32 * 1024);
   ASSERT(p != NULL);
   return p;
 }
 void backendOpen() {
   void *p = backendCreate();
   ASSERT(p != NULL);
+  taskDbDestroy(p, true);
 }
 
-TEST_F(BackendEnv, checkOpen) { backendOpen(); }
+TEST_F(BackendEnv, checkOpen) {
+  backendOpen();
+}
 TEST_F(BackendEnv, backendOpt) {}
 TEST_F(BackendEnv, backendDestroy) {}
 
diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp
index 0dc2cc13f5..eb7f38744f 100644
--- a/source/libs/stream/test/checkpointTest.cpp
+++ b/source/libs/stream/test/checkpointTest.cpp
@@ -25,24 +25,24 @@
 #pragma GCC diagnostic ignored "-Wunused-variable"
 #pragma GCC diagnostic ignored "-Wsign-compare"
 
+#include "cos.h"
 #include "rsync.h"
 #include "streamInt.h"
-#include "cos.h"
 
-int main(int argc, char **argv) {
-  testing::InitGoogleTest(&argc, argv);
+// int main(int argc, char **argv) {
+//   testing::InitGoogleTest(&argc, argv);
 
-  if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) {
-    printf("error");
-  }
-  if (s3Init() < 0) {
-    return -1;
-  }
-  strcpy(tsSnodeAddress, "127.0.0.1");
-  int ret = RUN_ALL_TESTS();
-  s3CleanUp();
-  return ret;
-}
+//   if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) {
+//     printf("error");
+//   }
+//   if (s3Init() < 0) {
+//     return -1;
+//   }
+//   strcpy(tsSnodeAddress, "127.0.0.1");
+//   int ret = RUN_ALL_TESTS();
+//   s3CleanUp();
+//   return ret;
+// }
 
 TEST(testCase, checkpointUpload_Test) {
   stopRsync();

From d021138f3b7515feca1d27ae35f711ac7c0eb85d Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 27 Feb 2024 09:47:55 +0800
Subject: [PATCH 062/201] remove sort by row id and comments

---
 source/libs/executor/src/scanoperator.c | 2 --
 source/libs/executor/src/tsort.c        | 6 ------
 2 files changed, 8 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 43a716a079..7f1f0700d0 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4351,8 +4351,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
     pInfo->bSortRowId = false;
   }
 
-  pInfo->bSortRowId = true;
-
   pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order);
   pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false);
 
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index a779a278cd..2505223273 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1012,12 +1012,6 @@ static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) {
   return (int32_t)(pStart - (char*)buf);
 }
 
-// pageId * pageSize == pageStartOffset in file. write in pages
-// when pass the page boundaries, the page is move to the front(old). 
-// find hash from pageid to page entry. if the page can not be found, 
-//   1) unused inactive pages, 2) then new pages if not exceeding mem limit, 3) then active pages
-// new pages is added or moved to the back.
-
 static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char** ppPage) {
   SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
   SSortMemPageEntry** ppPageEntry = tSimpleHashGet(pMemFile->mActivePages, &pageId, sizeof(pageId));

From 8d36445bb58b89e8c219fe8a844be7c9997d7c96 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 27 Feb 2024 05:29:58 +0000
Subject: [PATCH 063/201] add  stream backend  test

---
 source/libs/stream/test/backendTest.cpp | 53 +++++++++++++++++++++----
 1 file changed, 45 insertions(+), 8 deletions(-)

diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index d70f16b461..df0ab9fe7e 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -3,6 +3,7 @@
 #include <taoserror.h>
 #include <tglobal.h>
 #include <iostream>
+#include <vector>
 #include "streamBackendRocksdb.h"
 #include "streamSnapshot.h"
 #include "streamState.h"
@@ -27,26 +28,62 @@ void *backendCreate() {
   return p;
 }
 
-SStreamState *stateCreate(void *pBackend, char *keyidr) {
-  const char  *streamPath = "/tmp";
+SStreamState *stateCreate(const char *path) {
   SStreamTask *pTask = (SStreamTask *)taosMemoryCalloc(1, sizeof(SStreamTask));
   pTask->ver = 1024;
   pTask->id.streamId = 1023;
   pTask->id.taskId = 1111111;
+  SStreamMeta *pMeta = streamMetaOpen((path), NULL, NULL, 0, 0, NULL);
+  pTask->pMeta = pMeta;
 
-  SStreamState *p = streamStateOpen((char *)streamPath, pTask, true, 32, 32 * 1024);
+  SStreamState *p = streamStateOpen((char *)path, pTask, true, 32, 32 * 1024);
   ASSERT(p != NULL);
   return p;
 }
 void backendOpen() {
-  void *p = backendCreate();
+  const char   *path = "/tmp/backend";
+  SStreamState *p = stateCreate(path);
   ASSERT(p != NULL);
-  taskDbDestroy(p, true);
+
+  // write bacth
+  // default/state/fill/sess/func/parname/partag
+  int32_t              size = 100;
+  std::vector<int64_t> tsArray;
+  for (int i = 0; i < size; i++) {
+    int64_t     ts = taosGetTimestampMs();
+    SWinKey     key = {.groupId = (uint64_t)(i), .ts = ts};
+    const char *val = "value data";
+    int32_t     vlen = strlen(val);
+    streamStatePut_rocksdb(p, &key, (char *)val, vlen);
+
+    tsArray.push_back(ts);
+  }
+  for (int i = 0; i < size; i++) {
+    int64_t ts = tsArray[i];
+    SWinKey key = {.groupId = (uint64_t)(i), .ts = ts};
+
+    const char *val = "value data";
+    int32_t     len = 0;
+    char       *newVal = NULL;
+    streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
+    ASSERT(len == strlen(val));
+  }
+  int64_t ts = tsArray[0];
+  SWinKey key = {.groupId = (uint64_t)(0), .ts = ts};
+  streamStateDel_rocksdb(p, &key);
+
+  
+
+  // read
+  // iterator
+  // rebuild chkp, reload from chkp
+  // sync
+  //
+  streamStateClose((SStreamState *)p, true);
+  // taskDbDestroy(p, true);
 }
 
-TEST_F(BackendEnv, checkOpen) {
-  backendOpen();
-}
+TEST_F(BackendEnv, checkOpen) { backendOpen(); }
 TEST_F(BackendEnv, backendOpt) {}
 TEST_F(BackendEnv, backendDestroy) {}
 

From 6b6f5aa5d1d8d132ff7635b091926ff395b0f0d2 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 27 Feb 2024 05:51:32 +0000
Subject: [PATCH 064/201] add test case

---
 source/libs/stream/test/backendTest.cpp | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index df0ab9fe7e..1ccc0b4285 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -72,7 +72,18 @@ void backendOpen() {
   SWinKey key = {.groupId = (uint64_t)(0), .ts = ts};
   streamStateDel_rocksdb(p, &key);
 
-  
+  streamStateClear_rocksdb(p);
+
+  for (int i = 0; i < size; i++) {
+    int64_t     ts = tsArray[i];
+    SWinKey     key = {.groupId = (uint64_t)(i), .ts = ts};
+    const char *val = "value data";
+    int32_t     len = 0;
+    char       *newVal = NULL;
+    int32_t     code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
+    ASSERT(code != 0);
+  }
+  tsArray.clear();
 
   // read
   // iterator

From d62b82c295b2db35017cd8383baef70ea8cce680 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 27 Feb 2024 16:02:15 +0800
Subject: [PATCH 065/201] opti:deal with escape in the end of field in
 schemaless

---
 source/client/inc/clientSml.h     |  2 +-
 source/client/src/clientSmlLine.c | 59 ++++++++++---------
 utils/test/c/sml_test.c           | 94 +++++++++++++++++++++++++++++--
 3 files changed, 122 insertions(+), 33 deletions(-)

diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h
index b732abffb1..122914fd34 100644
--- a/source/client/inc/clientSml.h
+++ b/source/client/inc/clientSml.h
@@ -80,7 +80,7 @@ extern "C" {
 #define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0)
 
 #define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
-  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
+  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH))
 
 #define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
 
diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c
index 0c610a4611..8bdbec0f2b 100644
--- a/source/client/src/clientSmlLine.c
+++ b/source/client/src/clientSmlLine.c
@@ -20,14 +20,14 @@
 
 #include "clientSml.h"
 
-#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH)
-#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH)
-#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH)
+#define IS_COMMA(sql,escapeChar) (*(sql) == COMMA && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar)))
+#define IS_SPACE(sql,escapeChar) (*(sql) == SPACE && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar)))
+#define IS_EQUAL(sql,escapeChar) (*(sql) == EQUAL && (*((sql)-1) != SLASH || ((sql)-1 == escapeChar)))
 
 #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH))
 
 #define IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql) \
-  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL))
+  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == SLASH))
 
 #define PROCESS_SLASH_IN_FIELD_VALUE(key, keyLen)  \
   for (int i = 1; i < keyLen; ++i) {               \
@@ -198,7 +198,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
   int     cnt = 0;
 
   while (*sql < sqlEnd) {
-    if (unlikely(IS_SPACE(*sql))) {
+    if (unlikely(IS_SPACE(*sql,NULL))) {
       break;
     }
 
@@ -207,18 +207,21 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
     size_t      keyLen = 0;
     bool        keyEscaped = false;
     size_t      keyLenEscaped = 0;
+    const char *escapeChar = NULL;
+
     while (*sql < sqlEnd) {
-      if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
+      if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
         terrno = TSDB_CODE_SML_INVALID_DATA;
         return -1;
       }
-      if (unlikely(IS_EQUAL(*sql))) {
+      if (unlikely(IS_EQUAL(*sql,escapeChar))) {
         keyLen = *sql - key;
         (*sql)++;
         break;
       }
       if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
+        escapeChar = *sql;
         keyLenEscaped++;
         keyEscaped = true;
       }
@@ -238,15 +241,16 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
     size_t      valueLenEscaped = 0;
     while (*sql < sqlEnd) {
       // parse value
-      if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
+      if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) {
         break;
-      } else if (unlikely(IS_EQUAL(*sql))) {
+      } else if (unlikely(IS_EQUAL(*sql,escapeChar))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
         terrno = TSDB_CODE_SML_INVALID_DATA;
         return -1;
       }
 
       if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
+        escapeChar = *sql;
         valueLenEscaped++;
         valueEscaped = true;
       }
@@ -293,7 +297,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
     }
 
     cnt++;
-    if (IS_SPACE(*sql)) {
+    if (IS_SPACE(*sql,escapeChar)) {
       break;
     }
     (*sql)++;
@@ -326,7 +330,7 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
 static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) {
   int cnt = 0;
   while (*sql < sqlEnd) {
-    if (unlikely(IS_SPACE(*sql))) {
+    if (unlikely(IS_SPACE(*sql,NULL))) {
       break;
     }
 
@@ -335,17 +339,19 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
     size_t      keyLen = 0;
     bool        keyEscaped = false;
     size_t      keyLenEscaped = 0;
+    const char *escapeChar = NULL;
     while (*sql < sqlEnd) {
-      if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
+      if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
         return TSDB_CODE_SML_INVALID_DATA;
       }
-      if (unlikely(IS_EQUAL(*sql))) {
+      if (unlikely(IS_EQUAL(*sql,escapeChar))) {
         keyLen = *sql - key;
         (*sql)++;
         break;
       }
       if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
+        escapeChar = *sql;
         keyLenEscaped++;
         keyEscaped = true;
       }
@@ -363,7 +369,6 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
     bool        valueEscaped = false;
     size_t      valueLenEscaped = 0;
     int         quoteNum = 0;
-    const char *escapeChar = NULL;
     while (*sql < sqlEnd) {
       // parse value
       if (unlikely(*(*sql) == QUOTE && (*(*sql - 1) != SLASH || (*sql - 1) == escapeChar))) {
@@ -374,7 +379,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
         }
         continue;
       }
-      if (quoteNum % 2 == 0 && (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql)))) {
+      if (quoteNum % 2 == 0 && (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar)))) {
         break;
       }
       if (IS_SLASH_LETTER_IN_FIELD_VALUE(*sql) && (*sql - 1) != escapeChar) {
@@ -437,7 +442,7 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
     }
 
     cnt++;
-    if (IS_SPACE(*sql)) {
+    if (IS_SPACE(*sql,escapeChar)) {
       break;
     }
     (*sql)++;
@@ -453,19 +458,18 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   elements->measure = sql;
   // parse measure
   size_t measureLenEscaped = 0;
+  const char *escapeChar = NULL;
   while (sql < sqlEnd) {
-    if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) {
-      elements->measureEscaped = true;
-      measureLenEscaped++;
-      sql++;
-      continue;
-    }
-    if (unlikely(IS_COMMA(sql))) {
+    if (unlikely(IS_COMMA(sql,escapeChar) || IS_SPACE(sql,escapeChar))) {
       break;
     }
 
-    if (unlikely(IS_SPACE(sql))) {
-      break;
+    if (unlikely((sql != elements->measure) && IS_SLASH_LETTER_IN_MEASUREMENT(sql))) {
+      elements->measureEscaped = true;
+      escapeChar = sql;
+      measureLenEscaped++;
+      sql++;
+      continue;
     }
     sql++;
   }
@@ -478,9 +482,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   // to get measureTagsLen before
   const char *tmp = sql;
   while (tmp < sqlEnd) {
-    if (unlikely(IS_SPACE(tmp))) {
+    if (unlikely(IS_SPACE(tmp,escapeChar))) {
       break;
     }
+    if(unlikely(IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql))){
+      escapeChar = sql;
+    }
     tmp++;
   }
   elements->measureTagsLen = tmp - elements->measure;
diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c
index 2c334eb67b..f81f13fc68 100644
--- a/utils/test/c/sml_test.c
+++ b/utils/test/c/sml_test.c
@@ -1018,7 +1018,7 @@ int sml_escape_Test() {
   ASSERT(numFields == 5);
   ASSERT(strncmp(fields[1].name, "inode\"i,= s_used", sizeof("inode\"i,= s_used") - 1) == 0);
   ASSERT(strncmp(fields[2].name, "total", sizeof("total") - 1) == 0);
-  ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\\\ree", sizeof("inode\"i,= s_f\\\\ree") - 1) == 0);
+  ASSERT(strncmp(fields[3].name, "inode\"i,= s_f\\ree", sizeof("inode\"i,= s_f\\ree") - 1) == 0);
   ASSERT(strncmp(fields[4].name, "dev\"i,= ce", sizeof("dev\"i,= ce") - 1) == 0);
 
   TAOS_ROW row = NULL;
@@ -1044,6 +1044,91 @@ int sml_escape_Test() {
   return code;
 }
 
+// test field with end of escape
+int sml_escape1_Test() {
+  TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+
+  TAOS_RES *pRes = taos_query(taos, "create database if not exists db_escape");
+  taos_free_result(pRes);
+
+  pRes = taos_query(taos, "use db_escape");
+  taos_free_result(pRes);
+
+  const char *sql[] = {
+      "stab,t1\\=1 c1=3,c2=\"32fw\" 1661943970000000000",
+      "stab,t1=1\\ c1=3,c2=\"32fw\" 1661943980000000000",
+      "stab,t1=1 c1\\=3,c2=\"32fw\" 1661943990000000000",
+  };
+  for(int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++){
+    pRes = taos_schemaless_insert(taos, (char**)&sql[i], 1, TSDB_SML_LINE_PROTOCOL, 0);
+    int code = taos_errno(pRes);
+    ASSERT(code);
+  }
+
+  const char *sql1[] = {
+      "stab\\,t1=1 c1=3,c2=\"32fw\" 1661943960000000000",
+      "stab\\\\,t1=1 c1=3,c2=\"32fw\" 1661943960000000000",
+      "stab,t1\\\\=1 c1=3,c2=\"32fw\" 1661943970000000000",
+      "stab,t1=1\\\\ c1=3,c2=\"32fw\" 1661943980000000000",
+      "stab,t1=1 c1\\\\=3,c2=\"32fw\" 1661943990000000000",
+  };
+  pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, 0);
+  printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes));
+  int code = taos_errno(pRes);
+  ASSERT(!code);
+  ASSERT(taos_affected_rows(pRes) == 5);
+  taos_free_result(pRes);
+
+  pRes = taos_query(taos, "select * from stab order by _ts");     //check stable name
+  ASSERT(pRes);
+  int fieldNum = taos_field_count(pRes);
+  ASSERT(fieldNum == 6);
+  printf("fieldNum:%d\n", fieldNum);
+
+  int         numFields = taos_num_fields(pRes);
+  TAOS_FIELD *fields = taos_fetch_fields(pRes);
+  ASSERT(numFields == 6);
+  ASSERT(strncmp(fields[1].name, "c1", sizeof("c1") - 1) == 0);
+  ASSERT(strncmp(fields[2].name, "c2", sizeof("c2") - 1) == 0);
+  ASSERT(strncmp(fields[3].name, "c1\\", sizeof("c1\\") - 1) == 0);
+  ASSERT(strncmp(fields[4].name, "t1\\", sizeof("t1\\") - 1) == 0);
+  ASSERT(strncmp(fields[5].name, "t1", sizeof("t1") - 1) == 0);
+
+  TAOS_ROW row = NULL;
+  int32_t  rowIndex = 0;
+  while ((row = taos_fetch_row(pRes)) != NULL) {
+    int64_t ts = *(int64_t *)row[0];
+
+    if (rowIndex == 0) {
+      ASSERT(ts == 1661943970000);
+      ASSERT(*(double *)row[1] == 3);
+      ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0);
+      ASSERT(row[3] == NULL);
+      ASSERT(strncmp(row[4], "1", sizeof("1") - 1) == 0);
+      ASSERT(row[5] == NULL);
+    }else if (rowIndex == 1) {
+      ASSERT(ts == 1661943980000);
+      ASSERT(*(double *)row[1] == 3);
+      ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0);
+      ASSERT(row[3] == NULL);
+      ASSERT(row[4] == NULL);
+      ASSERT(strncmp(row[5], "1\\", sizeof("1\\") - 1) == 0);
+    }else if (rowIndex == 2) {
+      ASSERT(ts == 1661943990000);
+      ASSERT(row[1] == NULL);
+      ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0);
+      ASSERT(*(double *)row[3] == 3);
+      ASSERT(row[4] == NULL);
+      ASSERT(strncmp(row[5], "1", sizeof("1") - 1) == 0);
+    }
+    rowIndex++;
+  }
+  taos_free_result(pRes);
+  taos_close(taos);
+
+  return code;
+}
+
 int sml_19221_Test() {
   TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
 
@@ -1775,17 +1860,14 @@ int main(int argc, char *argv[]) {
   ASSERT(ret);
   ret = sml_escape_Test();
   ASSERT(!ret);
+  ret = sml_escape1_Test();
+  ASSERT(!ret);
   ret = sml_ts3116_Test();
   ASSERT(!ret);
   ret = sml_ts2385_Test();    // this test case need config sml table name using ./sml_test config_file
   ASSERT(!ret);
   ret = sml_ts3303_Test();
   ASSERT(!ret);
-
-  //  for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){
-  //    printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i])));
-  //  }
-  //  int ret = 0;
   ret = sml_ttl_Test();
   ASSERT(!ret);
   ret = sml_ts2164_Test();

From 6bc12c872818e8b1bdf137c19a1c625fdc8c82e6 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 27 Feb 2024 16:27:59 +0800
Subject: [PATCH 066/201] refactor: do some internal refactor.

---
 include/util/tqueue.h                       | 36 ++-----------------
 source/client/src/clientTmq.c               |  7 ++--
 source/dnode/mgmt/mgmt_vnode/src/vmInt.c    | 10 +++---
 source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 10 +++---
 source/util/src/tqueue.c                    | 40 +++++++++++++++++++++
 source/util/src/tworker.c                   |  6 ++--
 6 files changed, 59 insertions(+), 50 deletions(-)

diff --git a/include/util/tqueue.h b/include/util/tqueue.h
index 9f09bd2930..eb887596d0 100644
--- a/include/util/tqueue.h
+++ b/include/util/tqueue.h
@@ -72,40 +72,6 @@ struct STaosQnode {
   char        item[];
 };
 
-struct STaosQueue {
-  STaosQnode   *head;
-  STaosQnode   *tail;
-  STaosQueue   *next;     // for queue set
-  STaosQset    *qset;     // for queue set
-  void         *ahandle;  // for queue set
-  FItem         itemFp;
-  FItems        itemsFp;
-  TdThreadMutex mutex;
-  int64_t       memOfItems;
-  int32_t       numOfItems;
-  int64_t       threadId;
-  int64_t       memLimit;
-  int64_t       itemLimit;
-};
-
-struct STaosQset {
-  STaosQueue   *head;
-  STaosQueue   *current;
-  TdThreadMutex mutex;
-  tsem_t        sem;
-  int32_t       numOfQueues;
-  int32_t       numOfItems;
-};
-
-struct STaosQall {
-  STaosQnode *current;
-  STaosQnode *start;
-  int32_t     numOfItems;
-  int64_t     memOfItems;
-  int32_t     unAccessedNumOfItems;
-  int64_t     unAccessMemOfItems;
-};
-
 STaosQueue *taosOpenQueue();
 void        taosCloseQueue(STaosQueue *queue);
 void        taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp);
@@ -140,6 +106,8 @@ int32_t    taosGetQueueNumber(STaosQset *qset);
 int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo);
 int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo);
 void    taosResetQsetThread(STaosQset *qset, void *pItem);
+void    taosQueueSetThreadId(STaosQueue *pQueue, int64_t threadId);
+int64_t taosQueueGetThreadId(STaosQueue *pQueue);
 
 #ifdef __cplusplus
 }
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 9b74456da2..a893b27896 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -876,12 +876,13 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
   STaosQall* qall = taosAllocateQall();
   taosReadAllQitems(pTmq->delayedTask, qall);
 
-  if (qall->numOfItems == 0) {
+  int32_t numOfItems = taosQallItemSize(qall);
+  if (numOfItems == 0) {
     taosFreeQall(qall);
     return TSDB_CODE_SUCCESS;
   }
 
-  tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, qall->numOfItems);
+  tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, numOfItems);
   int8_t* pTaskType = NULL;
   taosGetQitem(qall, (void**)&pTaskType);
 
@@ -1839,7 +1840,7 @@ static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal
 }
 
 static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
-  tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, tmq->qall->numOfItems);
+  tscDebug("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, taosQallItemSize(tmq->qall));
 
   while (1) {
     SMqRspWrapper* pRspWrapper = NULL;
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
index be88e8b3fd..3dfc2bd96f 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c
@@ -194,26 +194,26 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
   while (pVnode->refCount > 0) taosMsleep(10);
 
   dInfo("vgId:%d, wait for vnode write queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
-        pVnode->pWriteW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pWriteW.queue));
   tMultiWorkerCleanup(&pVnode->pWriteW);
 
   dInfo("vgId:%d, wait for vnode sync queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
-        pVnode->pSyncW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pSyncW.queue));
   tMultiWorkerCleanup(&pVnode->pSyncW);
 
   dInfo("vgId:%d, wait for vnode sync rd queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue,
-        pVnode->pSyncRdW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pSyncRdW.queue));
   tMultiWorkerCleanup(&pVnode->pSyncRdW);
 
   dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
-        pVnode->pApplyW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pApplyW.queue));
   tMultiWorkerCleanup(&pVnode->pApplyW);
 
   dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ);
   while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
 
   dInfo("vgId:%d, wait for vnode fetch queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ,
-        pVnode->pFetchQ->threadId);
+        taosQueueGetThreadId(pVnode->pFetchQ));
   while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
 
   tqNotifyClose(pVnode->pImpl->pTq);
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
index 8b80527447..a6abe5ab4d 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
@@ -365,16 +365,16 @@ int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   }
 
   dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
-        pVnode->pWriteW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pWriteW.queue));
   dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
-        pVnode->pSyncW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pSyncW.queue));
   dInfo("vgId:%d, sync-rd-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncRdW.queue,
-        pVnode->pSyncRdW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pSyncRdW.queue));
   dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
-        pVnode->pApplyW.queue->threadId);
+        taosQueueGetThreadId(pVnode->pApplyW.queue));
   dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ);
   dInfo("vgId:%d, fetch-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ,
-        pVnode->pFetchQ->threadId);
+        taosQueueGetThreadId(pVnode->pFetchQ));
   dInfo("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ);
   return 0;
 }
diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c
index 2cc13be6ba..7a4eb09b99 100644
--- a/source/util/src/tqueue.c
+++ b/source/util/src/tqueue.c
@@ -21,6 +21,40 @@
 int64_t tsRpcQueueMemoryAllowed = 0;
 int64_t tsRpcQueueMemoryUsed = 0;
 
+struct STaosQueue {
+  STaosQnode   *head;
+  STaosQnode   *tail;
+  STaosQueue   *next;     // for queue set
+  STaosQset    *qset;     // for queue set
+  void         *ahandle;  // for queue set
+  FItem         itemFp;
+  FItems        itemsFp;
+  TdThreadMutex mutex;
+  int64_t       memOfItems;
+  int32_t       numOfItems;
+  int64_t       threadId;
+  int64_t       memLimit;
+  int64_t       itemLimit;
+};
+
+struct STaosQset {
+  STaosQueue   *head;
+  STaosQueue   *current;
+  TdThreadMutex mutex;
+  tsem_t        sem;
+  int32_t       numOfQueues;
+  int32_t       numOfItems;
+};
+
+struct STaosQall {
+  STaosQnode *current;
+  STaosQnode *start;
+  int32_t     numOfItems;
+  int64_t     memOfItems;
+  int32_t     unAccessedNumOfItems;
+  int64_t     unAccessMemOfItems;
+};
+
 void taosSetQueueMemoryCapacity(STaosQueue *queue, int64_t cap) { queue->memLimit = cap; }
 void taosSetQueueCapacity(STaosQueue *queue, int64_t size) { queue->itemLimit = size; }
 
@@ -497,6 +531,12 @@ int64_t taosQallUnAccessedMemSize(STaosQall *qall) { return qall->unAccessMemOfI
 void    taosResetQitems(STaosQall *qall) { qall->current = qall->start; }
 int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; }
 
+void taosQueueSetThreadId(STaosQueue* pQueue, int64_t threadId) {
+  pQueue->threadId = threadId;
+}
+
+int64_t taosQueueGetThreadId(STaosQueue *pQueue) { return pQueue->threadId; }
+
 #if 0
 
 void taosResetQsetThread(STaosQset *qset, void *pItem) {
diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c
index 3e591c7d7f..138d4bc1f4 100644
--- a/source/util/src/tworker.c
+++ b/source/util/src/tworker.c
@@ -417,9 +417,9 @@ _OVER:
     return NULL;
   } else {
     while (worker->pid <= 0) taosMsleep(10);
-    queue->threadId = worker->pid;
-    uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle,
-          queue->threadId);
+
+    taosQueueSetThreadId(queue, worker->pid);
+    uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, worker->pid);
     return queue;
   }
 }

From 0e522fbf5a4fdc50786b966695c9383986fd130e Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 27 Feb 2024 16:30:38 +0800
Subject: [PATCH 067/201] set history task param

---
 source/libs/executor/src/executor.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c
index 2534c5e9f0..831fd4e883 100644
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@@ -1009,6 +1009,22 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
       pSup->deleteMark = INT64_MAX;
       pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData;
       pInfo->ignoreExpiredData = false;
+    } else if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT) {
+      SStreamCountAggOperatorInfo* pInfo = pOperator->info;
+      STimeWindowAggSupp* pSup = &pInfo->twAggSup;
+
+      ASSERT(pSup->calTrigger == STREAM_TRIGGER_AT_ONCE || pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE);
+      ASSERT(pSup->calTriggerSaved == 0 && pSup->deleteMarkSaved == 0);
+
+      qInfo("save stream param for state: %d,  %" PRId64, pSup->calTrigger, pSup->deleteMark);
+
+      pSup->calTriggerSaved = pSup->calTrigger;
+      pSup->deleteMarkSaved = pSup->deleteMark;
+      pSup->calTrigger = STREAM_TRIGGER_AT_ONCE;
+      pSup->deleteMark = INT64_MAX;
+      pInfo->ignoreExpiredDataSaved = pInfo->ignoreExpiredData;
+      pInfo->ignoreExpiredData = false;
+      qInfo("save stream task:%s, param for state: %d", GET_TASKID(pTaskInfo), pInfo->ignoreExpiredData);
     }
 
     // iterate operator tree

From 5be0cc7c8483a5f4baebaa1ba3efdc1baab1da5e Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 27 Feb 2024 18:07:04 +0800
Subject: [PATCH 068/201] opti:deal with escape in the end of field in
 schemaless

---
 source/client/src/clientSmlLine.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c
index 8bdbec0f2b..7535cbfd0c 100644
--- a/source/client/src/clientSmlLine.c
+++ b/source/client/src/clientSmlLine.c
@@ -485,8 +485,8 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     if (unlikely(IS_SPACE(tmp,escapeChar))) {
       break;
     }
-    if(unlikely(IS_SLASH_LETTER_IN_TAG_FIELD_KEY(sql))){
-      escapeChar = sql;
+    if(unlikely(IS_SLASH_LETTER_IN_TAG_FIELD_KEY(tmp))){
+      escapeChar = tmp;
     }
     tmp++;
   }

From 828889a6d8ca7ab7dab065db91ef5ca93b9d1963 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 27 Feb 2024 19:16:11 +0800
Subject: [PATCH 069/201] opti:deal with escape in the end of field in
 schemaless

---
 utils/test/c/sml_test.c | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c
index f81f13fc68..01619decc5 100644
--- a/utils/test/c/sml_test.c
+++ b/utils/test/c/sml_test.c
@@ -1079,7 +1079,7 @@ int sml_escape1_Test() {
   ASSERT(taos_affected_rows(pRes) == 5);
   taos_free_result(pRes);
 
-  pRes = taos_query(taos, "select * from stab order by _ts");     //check stable name
+  pRes = taos_query(taos, "select * from stab");     //check stable name
   ASSERT(pRes);
   int fieldNum = taos_field_count(pRes);
   ASSERT(fieldNum == 6);
@@ -1099,22 +1099,19 @@ int sml_escape1_Test() {
   while ((row = taos_fetch_row(pRes)) != NULL) {
     int64_t ts = *(int64_t *)row[0];
 
-    if (rowIndex == 0) {
-      ASSERT(ts == 1661943970000);
+    if (ts == 1661943970000) {
       ASSERT(*(double *)row[1] == 3);
       ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0);
       ASSERT(row[3] == NULL);
       ASSERT(strncmp(row[4], "1", sizeof("1") - 1) == 0);
       ASSERT(row[5] == NULL);
-    }else if (rowIndex == 1) {
-      ASSERT(ts == 1661943980000);
+    }else if (ts == 1661943980000) {
       ASSERT(*(double *)row[1] == 3);
       ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0);
       ASSERT(row[3] == NULL);
       ASSERT(row[4] == NULL);
       ASSERT(strncmp(row[5], "1\\", sizeof("1\\") - 1) == 0);
-    }else if (rowIndex == 2) {
-      ASSERT(ts == 1661943990000);
+    }else if (ts == 1661943990000) {
       ASSERT(row[1] == NULL);
       ASSERT(strncmp(row[2], "32fw", sizeof("32fw") - 1) == 0);
       ASSERT(*(double *)row[3] == 3);

From b3c18c31da19f3cb5b72001351eae62273067f57 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 27 Feb 2024 12:19:37 +0000
Subject: [PATCH 070/201] add test case to stream  backend

---
 source/libs/stream/inc/streamBackendRocksdb.h |   1 +
 source/libs/stream/src/streamBackendRocksdb.c |  12 +-
 source/libs/stream/test/backendTest.cpp       | 309 +++++++++++++++++-
 3 files changed, 305 insertions(+), 17 deletions(-)

diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h
index 6e6f85d8f9..1dc1db8e9c 100644
--- a/source/libs/stream/inc/streamBackendRocksdb.h
+++ b/source/libs/stream/inc/streamBackendRocksdb.h
@@ -258,6 +258,7 @@ void     bkdMgtDestroy(SBkdMgt* bm);
 
 int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** path, SArray* list);
 
+uint32_t nextPow2(uint32_t x); 
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index f173157da6..910fd93989 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -2788,7 +2788,6 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState) {
   STREAM_STATE_DEL_ROCKSDB(pState, "state", &maxStateKey);
   return pCur;
 }
-#ifdef BUILD_NO_CALL
 SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) {
   stDebug("streamStateGetCur_rocksdb");
   STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend;
@@ -2838,7 +2837,6 @@ int32_t streamStateFuncDel_rocksdb(SStreamState* pState, const STupleKey* key) {
   STREAM_STATE_DEL_ROCKSDB(pState, "func", key);
   return 0;
 }
-#endif
 
 // session cf
 int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) {
@@ -3432,7 +3430,6 @@ int32_t streamStateStateAddIfNotExist_rocksdb(SStreamState* pState, SSessionKey*
   SSessionKey tmpKey = *key;
   int32_t     valSize = *pVLen;
   void*       tmp = taosMemoryMalloc(valSize);
-  // tdbRealloc(NULL, valSize);
   if (!tmp) {
     return -1;
   }
@@ -3506,13 +3503,11 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi
   return code;
 }
 
-#ifdef BUILD_NO_CALL
 int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen) {
   int code = 0;
   STREAM_STATE_PUT_ROCKSDB(pState, "default", key, pVal, pVLen);
   return code;
 }
-#endif
 int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen) {
   int code = 0;
   STREAM_STATE_GET_ROCKSDB(pState, "default", key, pVal, pVLen);
@@ -3535,10 +3530,10 @@ int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, co
   if (pIter == NULL) {
     return -1;
   }
-
+  size_t klen = 0;
   rocksdb_iter_seek(pIter, start, strlen(start));
   while (rocksdb_iter_valid(pIter)) {
-    const char* key = rocksdb_iter_key(pIter, NULL);
+    const char* key = rocksdb_iter_key(pIter, &klen);
     int32_t     vlen = 0;
     const char* vval = rocksdb_iter_value(pIter, (size_t*)&vlen);
     char*       val = NULL;
@@ -3700,6 +3695,8 @@ uint32_t nextPow2(uint32_t x) {
   x = x | (x >> 16);
   return x + 1;
 }
+
+#ifdef BUILD_NO_CALL
 int32_t copyFiles(const char* src, const char* dst) {
   int32_t code = 0;
   // opt later, just hard link
@@ -3739,6 +3736,7 @@ _err:
   taosCloseDir(&pDir);
   return code >= 0 ? 0 : -1;
 }
+#endif
 
 int32_t isBkdDataMeta(char* name, int32_t len) {
   const char* pCurrent = "CURRENT";
diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index 1ccc0b4285..83adaeb21c 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -40,7 +40,8 @@ SStreamState *stateCreate(const char *path) {
   ASSERT(p != NULL);
   return p;
 }
-void backendOpen() {
+void *backendOpen() {
+  streamMetaInit();
   const char   *path = "/tmp/backend";
   SStreamState *p = stateCreate(path);
   ASSERT(p != NULL);
@@ -85,18 +86,306 @@ void backendOpen() {
   }
   tsArray.clear();
 
-  // read
-  // iterator
-  // rebuild chkp, reload from chkp
-  // sync
-  //
+  for (int i = 0; i < size; i++) {
+    int64_t ts = taosGetTimestampMs();
+    tsArray.push_back(ts);
+
+    SWinKey     key = {.groupId = (uint64_t)(i), .ts = ts};
+    const char *val = "value data";
+    int32_t     vlen = strlen(val);
+    streamStatePut_rocksdb(p, &key, (char *)val, vlen);
+  }
+
+  SWinKey winkey;
+  int32_t code = streamStateGetFirst_rocksdb(p, &key);
+  ASSERT(code == 0);
+  ASSERT(key.ts == tsArray[0]);
+
+  SStreamStateCur *pCurr = streamStateSeekToLast_rocksdb(p);
+  ASSERT(pCurr != NULL);
+  streamStateFreeCur(pCurr);
+
+  winkey.groupId = 0;
+  winkey.ts = tsArray[0];
+  char   *val = NULL;
+  int32_t len = 0;
+
+  pCurr = streamStateSeekKeyNext_rocksdb(p, &winkey);
+  ASSERT(pCurr != NULL);
+
+  streamStateFreeCur(pCurr);
+
+  tsArray.clear();
+  for (int i = 0; i < size; i++) {
+    int64_t ts = taosGetTimestampMs();
+    tsArray.push_back(ts);
+    STupleKey key = {.groupId = (uint64_t)(0), .ts = ts, .exprIdx = i};
+
+    const char *val = "Value";
+    int32_t     len = strlen(val);
+    streamStateFuncPut_rocksdb(p, &key, val, len);
+  }
+  for (int i = 0; i < size; i++) {
+    STupleKey key = {.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
+    char     *val = NULL;
+    int32_t   len = 0;
+    streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len);
+    ASSERT(len == strlen("Value"));
+  }
+  for (int i = 0; i < size; i++) {
+    STupleKey key = {.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
+    char     *val = NULL;
+    int32_t   len = 0;
+    streamStateFuncDel_rocksdb(p, &key);
+  }
+
+  // session put
+  tsArray.clear();
+
+  for (int i = 0; i < size; i++) {
+    SSessionKey key = {.win = {.skey = i, .ekey = i}, .groupId = (uint64_t)(0)};
+    tsArray.push_back(i);
+
+    const char *val = "Value";
+    int32_t     len = strlen(val);
+    streamStateSessionPut_rocksdb(p, &key, val, len);
+
+    char *pval = NULL;
+    ASSERT(0 == streamStateSessionGet_rocksdb(p, &key, (void **)&pval, &len));
+    ASSERT(strncmp(pval, val, len) == 0);
+  }
+
+  for (int i = 0; i < size; i++) {
+    SSessionKey key = {.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)};
+
+    const char *val = "Value";
+    int32_t     len = strlen(val);
+
+    char *pval = NULL;
+    ASSERT(0 == streamStateSessionGet_rocksdb(p, &key, (void **)&pval, &len));
+    ASSERT(strncmp(pval, val, len) == 0);
+    taosMemoryFreeClear(pval);
+  }
+
+  pCurr = streamStateSessionSeekToLast_rocksdb(p, 0);
+  ASSERT(pCurr != NULL);
+
+  {
+    SSessionKey key;
+    char       *val = NULL;
+    int32_t     vlen = 0;
+    code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen);
+    ASSERT(code == 0);
+    pCurr = streamStateSessionSeekKeyPrev_rocksdb(p, &key);
+
+    code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen);
+    ASSERT(code == 0);
+
+    ASSERT(key.groupId == 0 && key.win.ekey == tsArray[tsArray.size() - 2]);
+
+    pCurr = streamStateSessionSeekKeyNext_rocksdb(p, &key);
+    code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen);
+    ASSERT(code == 0);
+    ASSERT(vlen == strlen("Value"));
+    ASSERT(key.groupId == 0 && key.win.skey == tsArray[tsArray.size() - 1]);
+
+    ASSERT(0 == streamStateSessionAddIfNotExist_rocksdb(p, &key, 10, (void **)&val, &len));
+
+    ASSERT(0 ==
+           streamStateStateAddIfNotExist_rocksdb(p, &key, (char *)"key", strlen("key"), NULL, (void **)&val, &len));
+  }
+  for (int i = 0; i < size; i++) {
+    SSessionKey key = {.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)};
+
+    const char *val = "Value";
+    int32_t     len = strlen(val);
+
+    char *pval = NULL;
+    ASSERT(0 == streamStateSessionDel_rocksdb(p, &key));
+  }
+
+  for (int i = 0; i < size; i++) {
+    SWinKey     key = {.groupId = (uint64_t)(i), .ts = tsArray[i]};
+    const char *val = "Value";
+    int32_t     vlen = strlen(val);
+    ASSERT(streamStateFillPut_rocksdb(p, &key, val, vlen) == 0);
+  }
+  for (int i = 0; i < size; i++) {
+    SWinKey key = {.groupId = (uint64_t)(i), .ts = tsArray[i]};
+    char   *val = NULL;
+    int32_t vlen = 0;
+    ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0);
+    taosMemoryFreeClear(val);
+  }
+  {
+    SWinKey          key = {.groupId = (uint64_t)(0), .ts = tsArray[0]};
+    SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key);
+    ASSERT(pCurr != NULL);
+
+    char   *val = NULL;
+    int32_t vlen = 0;
+    ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen));
+    ASSERT(vlen == strlen("Value"));
+    streamStateFreeCur(pCurr);
+
+    pCurr = streamStateFillSeekKeyNext_rocksdb(p, &key);
+    ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen));
+    ASSERT(vlen == strlen("Value") && key.groupId == 1 && key.ts == tsArray[1]);
+
+    key.groupId = 1;
+    key.ts = tsArray[1];
+
+    pCurr = streamStateFillSeekKeyPrev_rocksdb(p, &key);
+    ASSERT(pCurr != NULL);
+    ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen));
+
+    ASSERT(vlen == strlen("Value") && key.groupId == 0 && key.ts == tsArray[0]);
+  }
+
+  for (int i = 0; i < size - 1; i++) {
+    SWinKey key = {.groupId = (uint64_t)(i), .ts = tsArray[i]};
+    char   *val = NULL;
+    int32_t vlen = 0;
+    ASSERT(streamStateFillDel_rocksdb(p, &key) == 0);
+    taosMemoryFreeClear(val);
+  }
+  streamStateSessionClear_rocksdb(p);
+
+  for (int i = 0; i < size; i++) {
+    char tbname[TSDB_TABLE_NAME_LEN] = {0};
+    sprintf(tbname, "%s_%d", "tbname", i);
+    ASSERT(0 == streamStatePutParName_rocksdb(p, i, tbname));
+  }
+  for (int i = 0; i < size; i++) {
+    char *val = NULL;
+    ASSERT(0 == streamStateGetParName_rocksdb(p, i, (void **)&val));
+    ASSERT(strncmp(val, "tbname", strlen("tbname")) == 0);
+    taosMemoryFree(val);
+  }
+
+  for (int i = 0; i < size; i++) {
+    char tbname[TSDB_TABLE_NAME_LEN] = {0};
+    sprintf(tbname, "%s_%d", "tbname", i);
+    ASSERT(0 == streamStatePutParName_rocksdb(p, i, tbname));
+  }
+  for (int i = 0; i < size; i++) {
+    char *val = NULL;
+    ASSERT(0 == streamStateGetParName_rocksdb(p, i, (void **)&val));
+    ASSERT(strncmp(val, "tbname", strlen("tbname")) == 0);
+    taosMemoryFree(val);
+  }
+  for (int i = 0; i < size; i++) {
+    char key[128] = {0};
+    sprintf(key, "tbname_%d", i);
+    char val[128] = {0};
+    sprintf(val, "val_%d", i);
+    code = streamDefaultPut_rocksdb(p, key, val, strlen(val));
+    ASSERT(code == 0);
+  }
+  for (int i = 0; i < size; i++) {
+    char key[128] = {0};
+    sprintf(key, "tbname_%d", i);
+
+    char   *val = NULL;
+    int32_t len = 0;
+    code = streamDefaultGet_rocksdb(p, key, (void **)&val, &len);
+    ASSERT(code == 0);
+  }
+  SArray *result = taosArrayInit(8, sizeof(void *));
+  streamDefaultIterGet_rocksdb(p, "tbname", "tbname_99", result);
+  ASSERT(taosArrayGetSize(result) >= 0);
+
+  return p;
+  // streamStateClose((SStreamState *)p, true);
+}
+TEST_F(BackendEnv, checkOpen) {
+  SStreamState *p = (SStreamState *)backendOpen();
+  int64_t       tsStart = taosGetTimestampMs();
+  {
+    void   *pBatch = streamStateCreateBatch();
+    int32_t size = 0;
+    for (int i = 0; i < size; i++) {
+      char key[128] = {0};
+      sprintf(key, "key_%d", i);
+      char val[128] = {0};
+      sprintf(val, "val_%d", i);
+      streamStatePutBatch(p, "default", (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
+                          (int32_t)(strlen(val)), tsStart + 100000);
+    }
+    streamStatePutBatch_rocksdb(p, pBatch);
+    streamStateDestroyBatch(pBatch);
+  }
+  {
+    void   *pBatch = streamStateCreateBatch();
+    int32_t size = 0;
+    char    valBuf[256] = {0};
+    for (int i = 0; i < size; i++) {
+      char key[128] = {0};
+      sprintf(key, "key_%d", i);
+      char val[128] = {0};
+      sprintf(val, "val_%d", i);
+      streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
+                                  (int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf);
+    }
+    streamStatePutBatch_rocksdb(p, pBatch);
+    streamStateDestroyBatch(pBatch);
+  }
+  // do checkpoint 2
+  taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2);
+  {
+    void   *pBatch = streamStateCreateBatch();
+    int32_t size = 0;
+    char    valBuf[256] = {0};
+    for (int i = 0; i < size; i++) {
+      char key[128] = {0};
+      sprintf(key, "key_%d", i);
+      char val[128] = {0};
+      sprintf(val, "val_%d", i);
+      streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
+                                  (int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf);
+    }
+    streamStatePutBatch_rocksdb(p, pBatch);
+    streamStateDestroyBatch(pBatch);
+  }
+
+  taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 3);
+
+  const char *path = "/tmp/backend/stream";
+  const char *dump = "/tmp/backend/stream/dump";
+  // taosMkDir(dump);
+  taosMulMkDir(dump);
+  SBkdMgt *mgt = bkdMgtCreate((char *)path);
+  SArray  *result = taosArrayInit(4, sizeof(void *));
+  bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump);
+
+  bkdMgtDestroy(mgt);
   streamStateClose((SStreamState *)p, true);
-  // taskDbDestroy(p, true);
 }
 
-TEST_F(BackendEnv, checkOpen) { backendOpen(); }
-TEST_F(BackendEnv, backendOpt) {}
-TEST_F(BackendEnv, backendDestroy) {}
+TEST_F(BackendEnv, backendChkp) { const char *path = "/tmp"; }
+
+typedef struct BdKV {
+  uint32_t k;
+  uint32_t v;
+} BdKV;
+
+BdKV kvDict[] = {{0, 2},
+                 {1, 2},
+                 {.k = 15, .v = 16},
+                 {.k = 31, .v = 32},
+                 {.k = 56, .v = 64},
+                 {.k = 100, .v = 128},
+                 {.k = 200, .v = 256},
+                 {.k = 500, .v = 512},
+                 {.k = 1000, .v = 1024},
+                 {.k = 2000, .v = 2048},
+                 {.k = 3000, .v = 4096}};
+
+TEST_F(BackendEnv, backendUtil) {
+  for (int i = 0; i < sizeof(kvDict) / sizeof(kvDict[0]); i++) {
+    ASSERT_EQ(nextPow2((uint32_t)(kvDict[i].k)), kvDict[i].v);
+  }
+}
 
 int main(int argc, char **argv) {
   testing::InitGoogleTest(&argc, argv);

From e1f11d1639e1d3214706cf1272e9c0ea1d2f9dd2 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 28 Feb 2024 00:48:20 +0000
Subject: [PATCH 071/201] add test case to stream  backend

---
 source/libs/stream/test/checkpointTest.cpp | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp
index eb7f38744f..0caad479e5 100644
--- a/source/libs/stream/test/checkpointTest.cpp
+++ b/source/libs/stream/test/checkpointTest.cpp
@@ -25,6 +25,9 @@
 #pragma GCC diagnostic ignored "-Wunused-variable"
 #pragma GCC diagnostic ignored "-Wsign-compare"
 
+// tsSnodeAddress = "";
+// tsS3StreamEnabled = 0;
+
 #include "cos.h"
 #include "rsync.h"
 #include "streamInt.h"
@@ -45,26 +48,26 @@
 // }
 
 TEST(testCase, checkpointUpload_Test) {
-  stopRsync();
-  startRsync();
+  // stopRsync();
+  // startRsync();
 
   taosSsleep(5);
   char* id = "2013892036";
 
-  uploadCheckpoint(id, "/root/offset/");
+  // uploadCheckpoint(id, "/root/offset/");
 }
 
 TEST(testCase, checkpointDownload_Test) {
   char* id = "2013892036";
-  downloadCheckpoint(id, "/root/offset/download/");
+  // downloadCheckpoint(id, "/root/offset/download/");
 }
 
 TEST(testCase, checkpointDelete_Test) {
   char* id = "2013892036";
-  deleteCheckpoint(id);
+  // deleteCheckpoint(id);
 }
 
 TEST(testCase, checkpointDeleteFile_Test) {
   char* id = "2013892036";
-  deleteCheckpointFile(id, "offset-ver0");
+  // deleteCheckpointFile(id, "offset-ver0");
 }

From c1129bc8610a2e3035dce3b580596a38edfc29be Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Wed, 28 Feb 2024 09:34:43 +0800
Subject: [PATCH 072/201] add ci

---
 tests/parallel_test/cases.task                |   1 +
 .../stream/distributeMultiLevelInterval0.sim  | 267 ++++++++++++++++++
 2 files changed, 268 insertions(+)
 create mode 100644 tests/script/tsim/stream/distributeMultiLevelInterval0.sim

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index bdccf33c32..62f2d10525 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -1211,6 +1211,7 @@
 ,,y,script,./test.sh -f tsim/stream/deleteState.sim
 ,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim
 ,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim
+,,y,script,./test.sh -f tsim/stream/distributeMultiLevelInterval0.sim
 ,,y,script,./test.sh -f tsim/stream/distributeSession0.sim
 ,,y,script,./test.sh -f tsim/stream/drop_stream.sim
 ,,y,script,./test.sh -f tsim/stream/event0.sim
diff --git a/tests/script/tsim/stream/distributeMultiLevelInterval0.sim b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim
new file mode 100644
index 0000000000..784ab7f4a5
--- /dev/null
+++ b/tests/script/tsim/stream/distributeMultiLevelInterval0.sim
@@ -0,0 +1,267 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+
+system sh/cfg.sh -n dnode1 -c streamAggCnt  -v 2
+
+system sh/exec.sh -n dnode1 -s start 
+sleep 50
+sql connect
+
+
+
+print ===== step1
+sql drop stream if exists streams1;
+sql drop database if exists test;
+sql create database test  vgroups 4;
+sql use test;
+sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create table ts2 using st tags(2,2,2);
+sql create table ts3 using st tags(3,2,2);
+sql create table ts4 using st tags(4,2,2);
+sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select  _wstart, count(*) c1, sum(a) c3 , max(b) c4 from st interval(10s);
+
+sleep 1000
+
+sql insert into ts1 values(1648791213000,1,1,3,4.1);
+sql insert into ts1 values(1648791223000,2,2,3,1.1);
+sql insert into ts1 values(1648791233000,3,3,3,2.1);
+sql insert into ts1 values(1648791243000,4,4,3,3.1);
+
+sql insert into ts2 values(1648791213000,1,5,3,4.1);
+sql insert into ts2 values(1648791223000,2,6,3,1.1);
+sql insert into ts2 values(1648791233000,3,7,3,2.1);
+sql insert into ts2 values(1648791243000,4,8,3,3.1);
+
+
+$loop_count = 0
+loop0:
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+sleep 1000
+print 2 select * from streamt1;
+sql select * from streamt1;
+
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+print $data20 $data21 $data22 $data23
+print $data30 $data31 $data32 $data33
+print $data40 $data41 $data42 $data43
+
+if $rows != 4 then
+  print =====rows=$rows
+  goto loop0
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop0
+endi
+
+if $data11 != 2 then
+  print =====data11=$data11
+  goto loop0
+endi
+
+if $data21 != 2 then
+  print =====data21=$data21
+  goto loop0
+endi
+
+if $data31 != 2 then
+  print =====data31=$data31
+  goto loop0
+endi
+
+
+sql insert into ts1 values(1648791213000,1,9,3,4.1);
+
+$loop_count = 0
+loop1:
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+sleep 1000
+print 2 select * from streamt1;
+sql select * from streamt1;
+
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+print $data20 $data21 $data22 $data23
+print $data30 $data31 $data32 $data33
+print $data40 $data41 $data42 $data43
+
+if $rows != 4 then
+  print =====rows=$rows
+  goto loop1
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop1
+endi
+
+if $data11 != 2 then
+  print =====data11=$data11
+  goto loop1
+endi
+
+if $data21 != 2 then
+  print =====data21=$data21
+  goto loop1
+endi
+
+if $data31 != 2 then
+  print =====data31=$data31
+  goto loop1
+endi
+
+sql delete from ts2 where ts = 1648791243000 ;
+
+$loop_count = 0
+loop2:
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+sleep 1000
+print 2 select * from streamt1;
+sql select * from streamt1;
+
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+print $data20 $data21 $data22 $data23
+print $data30 $data31 $data32 $data33
+print $data40 $data41 $data42 $data43
+
+if $rows != 4 then
+  print =====rows=$rows
+  goto loop2
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop2
+endi
+
+if $data11 != 2 then
+  print =====data11=$data11
+  goto loop2
+endi
+
+if $data21 != 2 then
+  print =====data21=$data21
+  goto loop2
+endi
+
+if $data31 != 1 then
+  print =====data31=$data31
+  goto loop2
+endi
+
+sql delete from ts2 where ts = 1648791223000 ;
+
+$loop_count = 0
+loop3:
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+sleep 1000
+print 2 select * from streamt1;
+sql select * from streamt1;
+
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+print $data20 $data21 $data22 $data23
+print $data30 $data31 $data32 $data33
+print $data40 $data41 $data42 $data43
+
+if $rows != 4 then
+  print =====rows=$rows
+  goto loop3
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop3
+endi
+
+if $data11 != 1 then
+  print =====data11=$data11
+  goto loop3
+endi
+
+if $data21 != 2 then
+  print =====data21=$data21
+  goto loop3
+endi
+
+if $data31 != 1 then
+  print =====data31=$data31
+  goto loop3
+endi
+
+
+sql insert into ts1 values(1648791233001,3,9,3,2.1);
+
+$loop_count = 0
+loop4:
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+sleep 1000
+print 2 select * from streamt1;
+sql select * from streamt1;
+
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+print $data20 $data21 $data22 $data23
+print $data30 $data31 $data32 $data33
+print $data40 $data41 $data42 $data43
+
+if $rows != 4 then
+  print =====rows=$rows
+  goto loop4
+endi
+
+if $data01 != 2 then
+  print =====data01=$data01
+  goto loop4
+endi
+
+if $data11 != 1 then
+  print =====data11=$data11
+  goto loop4
+endi
+
+if $data21 != 3 then
+  print =====data21=$data21
+  goto loop4
+endi
+
+if $data31 != 1 then
+  print =====data31=$data31
+  goto loop4
+endi
+
+sql select  _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b)  c4, min(c) c5, avg(d) from st interval(10s);
+
+
+print ===== over
+
+system sh/stop_dnodes.sh

From 9f983a07d78f5548f623aa92319be5afe75704be Mon Sep 17 00:00:00 2001
From: yihaoDeng <yhdeng@taosdata.com>
Date: Wed, 28 Feb 2024 10:48:24 +0800
Subject: [PATCH 073/201] add test case

---
 source/libs/stream/test/backendTest.cpp | 33 +++++++++++++------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index 83adaeb21c..8143c6dbaf 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -19,7 +19,7 @@ class BackendEnv : public ::testing::Test {
 
 void *backendCreate() {
   const char *streamPath = "/tmp";
-  void       *p = NULL;
+  void *      p = NULL;
 
   // char *absPath = NULL;
   // // SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(streamPath, -1, 2);
@@ -42,7 +42,7 @@ SStreamState *stateCreate(const char *path) {
 }
 void *backendOpen() {
   streamMetaInit();
-  const char   *path = "/tmp/backend";
+  const char *  path = "/tmp/backend";
   SStreamState *p = stateCreate(path);
   ASSERT(p != NULL);
 
@@ -65,7 +65,7 @@ void *backendOpen() {
 
     const char *val = "value data";
     int32_t     len = 0;
-    char       *newVal = NULL;
+    char *      newVal = NULL;
     streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
     ASSERT(len == strlen(val));
   }
@@ -80,7 +80,7 @@ void *backendOpen() {
     SWinKey     key = {.groupId = (uint64_t)(i), .ts = ts};
     const char *val = "value data";
     int32_t     len = 0;
-    char       *newVal = NULL;
+    char *      newVal = NULL;
     int32_t     code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
     ASSERT(code != 0);
   }
@@ -107,7 +107,7 @@ void *backendOpen() {
 
   winkey.groupId = 0;
   winkey.ts = tsArray[0];
-  char   *val = NULL;
+  char *  val = NULL;
   int32_t len = 0;
 
   pCurr = streamStateSeekKeyNext_rocksdb(p, &winkey);
@@ -127,14 +127,14 @@ void *backendOpen() {
   }
   for (int i = 0; i < size; i++) {
     STupleKey key = {.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
-    char     *val = NULL;
+    char *    val = NULL;
     int32_t   len = 0;
     streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len);
     ASSERT(len == strlen("Value"));
   }
   for (int i = 0; i < size; i++) {
     STupleKey key = {.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
-    char     *val = NULL;
+    char *    val = NULL;
     int32_t   len = 0;
     streamStateFuncDel_rocksdb(p, &key);
   }
@@ -172,7 +172,7 @@ void *backendOpen() {
 
   {
     SSessionKey key;
-    char       *val = NULL;
+    char *      val = NULL;
     int32_t     vlen = 0;
     code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen);
     ASSERT(code == 0);
@@ -212,7 +212,7 @@ void *backendOpen() {
   }
   for (int i = 0; i < size; i++) {
     SWinKey key = {.groupId = (uint64_t)(i), .ts = tsArray[i]};
-    char   *val = NULL;
+    char *  val = NULL;
     int32_t vlen = 0;
     ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0);
     taosMemoryFreeClear(val);
@@ -222,7 +222,7 @@ void *backendOpen() {
     SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key);
     ASSERT(pCurr != NULL);
 
-    char   *val = NULL;
+    char *  val = NULL;
     int32_t vlen = 0;
     ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen));
     ASSERT(vlen == strlen("Value"));
@@ -244,7 +244,7 @@ void *backendOpen() {
 
   for (int i = 0; i < size - 1; i++) {
     SWinKey key = {.groupId = (uint64_t)(i), .ts = tsArray[i]};
-    char   *val = NULL;
+    char *  val = NULL;
     int32_t vlen = 0;
     ASSERT(streamStateFillDel_rocksdb(p, &key) == 0);
     taosMemoryFreeClear(val);
@@ -286,7 +286,7 @@ void *backendOpen() {
     char key[128] = {0};
     sprintf(key, "tbname_%d", i);
 
-    char   *val = NULL;
+    char *  val = NULL;
     int32_t len = 0;
     code = streamDefaultGet_rocksdb(p, key, (void **)&val, &len);
     ASSERT(code == 0);
@@ -302,7 +302,7 @@ TEST_F(BackendEnv, checkOpen) {
   SStreamState *p = (SStreamState *)backendOpen();
   int64_t       tsStart = taosGetTimestampMs();
   {
-    void   *pBatch = streamStateCreateBatch();
+    void *  pBatch = streamStateCreateBatch();
     int32_t size = 0;
     for (int i = 0; i < size; i++) {
       char key[128] = {0};
@@ -316,7 +316,7 @@ TEST_F(BackendEnv, checkOpen) {
     streamStateDestroyBatch(pBatch);
   }
   {
-    void   *pBatch = streamStateCreateBatch();
+    void *  pBatch = streamStateCreateBatch();
     int32_t size = 0;
     char    valBuf[256] = {0};
     for (int i = 0; i < size; i++) {
@@ -333,7 +333,7 @@ TEST_F(BackendEnv, checkOpen) {
   // do checkpoint 2
   taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2);
   {
-    void   *pBatch = streamStateCreateBatch();
+    void *  pBatch = streamStateCreateBatch();
     int32_t size = 0;
     char    valBuf[256] = {0};
     for (int i = 0; i < size; i++) {
@@ -355,11 +355,12 @@ TEST_F(BackendEnv, checkOpen) {
   // taosMkDir(dump);
   taosMulMkDir(dump);
   SBkdMgt *mgt = bkdMgtCreate((char *)path);
-  SArray  *result = taosArrayInit(4, sizeof(void *));
+  SArray * result = taosArrayInit(4, sizeof(void *));
   bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump);
 
   bkdMgtDestroy(mgt);
   streamStateClose((SStreamState *)p, true);
+  taosRemoveDir(path);
 }
 
 TEST_F(BackendEnv, backendChkp) { const char *path = "/tmp"; }

From 4474fcdd695912c48d47671a681cd7d3406b3b6c Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 28 Feb 2024 11:11:42 +0800
Subject: [PATCH 074/201] fix: add error processing related to files

---
 source/libs/executor/src/tsort.c | 49 ++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 15 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 2505223273..1d89e6d15c 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1019,6 +1019,7 @@ static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char*
     *ppPage = (char*)((*ppPageEntry)->data);
   } else {
     SSortMemPageEntry* pEntry = pMemFile->pagesHead->next;
+    bool freeEntryWhenError = false;
     if (pEntry && !pEntry->active || pMemFile->numMemPages >= pMemFile->totalMemPages) {
       if (pEntry->active) {
         tSimpleHashRemove(pMemFile->mActivePages, &pEntry->pageId, sizeof(pEntry->pageId));
@@ -1029,15 +1030,22 @@ static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char*
     } else if (pMemFile->numMemPages < pMemFile->totalMemPages) {
       pEntry = taosMemoryCalloc(1, sizeof(SSortMemPageEntry));
       pEntry->data = taosMemoryMalloc(pMemFile->pageSize);
+      freeEntryWhenError = true; 
       ++pMemFile->numMemPages;
     }
     {
-      // fseeko(pMemFile->pTdFile, ((int64_t)pageId) * pMemFile->pageSize, SEEK_SET);
-      // fread(pEntry->data, pMemFile->pageSize, 1, pMemFile->pTdFile);
-
-      taosLSeekFile(pMemFile->pTdFile, ((int64_t)pageId) * pMemFile->pageSize, SEEK_SET);
-      taosReadFile(pMemFile->pTdFile, pEntry->data, pMemFile->pageSize);
-
+      int64_t ret = taosLSeekFile(pMemFile->pTdFile, ((int64_t)pageId) * pMemFile->pageSize, SEEK_SET);
+      if (ret == 0) {
+        ret = taosReadFile(pMemFile->pTdFile, pEntry->data, pMemFile->pageSize);
+      }
+      if (ret != pMemFile->pageSize) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        if (freeEntryWhenError) {
+          taosMemoryFreeClear(pEntry->data);
+          taosMemoryFreeClear(pEntry);
+        }
+        return terrno;
+      }
       SSortMemPageEntry* tail = pMemFile->pagesTail;
       tail->next = pEntry;
       pEntry->next = NULL;
@@ -1082,6 +1090,10 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
   taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
   pMemFile->pTdFile =
       taosOpenFile(pMemFile->memFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
+  if (pMemFile->pTdFile == NULL) {
+    taosMemoryFree(pMemFile);
+    return TAOS_SYSTEM_ERROR(errno);
+  }
   pMemFile->currPageId = -1;
   pMemFile->currPageOffset = -1;
 
@@ -1121,7 +1133,6 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
   taosMemoryFree(pMemFile->writePageBuf);
   pMemFile->writePageBuf = NULL;
 
-  // fclose(pMemFile->pTdFile);
   taosCloseFile(&pMemFile->pTdFile);
   taosRemoveFile(pMemFile->memFilePath);
   taosMemoryFree(pMemFile);
@@ -1142,10 +1153,14 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p
       pMemFile->currPageOffset = 0;
 
       if (pMemFile->currPageId - pMemFile->startPageId >= pMemFile->numWritePages) {
-        // fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
-        // fwrite(pMemFile->writePageBuf, pMemFile->pageSize * pMemFile->numWritePages, 1, pMemFile->pTdFile);
-        taosLSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
-        taosWriteFile(pMemFile->pTdFile, pMemFile->writePageBuf, pMemFile->pageSize * pMemFile->numWritePages);
+        int64_t ret = taosLSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
+        if (ret == 0) {
+          ret = taosWriteFile(pMemFile->pTdFile, pMemFile->writePageBuf, pMemFile->pageSize * pMemFile->numWritePages);
+        }
+        if (ret !=  pMemFile->pageSize * pMemFile->numWritePages) {
+          terrno = TAOS_SYSTEM_ERROR(errno);
+          return terrno;
+        }          
         pMemFile->startPageId = pMemFile->currPageId;
       }
     }
@@ -1167,11 +1182,15 @@ static int32_t saveDirtyPagesToExtRowsMemFile(SSortHandle* pHandle) {
   if (!pMemFile->bDirty) {
     return TSDB_CODE_SUCCESS;
   }
-  // fseeko(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
-  taosLSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
+  int64_t ret = taosLSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
   int32_t numWriteBytes = pMemFile->pageSize * (pMemFile->currPageId - pMemFile->startPageId) + pMemFile->currPageOffset + 1;
-  // fwrite(pMemFile->writePageBuf, numWriteBytes, 1, pMemFile->pTdFile);
-  taosWriteFile(pMemFile->pTdFile, pMemFile->writePageBuf, numWriteBytes);
+  if (ret == 0) {
+    ret = taosWriteFile(pMemFile->pTdFile, pMemFile->writePageBuf, numWriteBytes);
+  }
+  if (ret != numWriteBytes) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return terrno;
+  }
   pMemFile->bDirty = false;
   return TSDB_CODE_SUCCESS;
 }

From d7f14e916d65754b79e46f0d38ba8a0c74c8dfbf Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 28 Feb 2024 06:01:50 +0000
Subject: [PATCH 075/201] add test case

---
 source/libs/stream/test/backendTest.cpp | 38 ++++++++++++++-----------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index 8143c6dbaf..3245979633 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -11,6 +11,12 @@
 #include "tstreamFileState.h"
 #include "tstreamUpdate.h"
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wwrite-strings"
+#pragma GCC diagnostic ignored "-Wunused-function"
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+
 class BackendEnv : public ::testing::Test {
  protected:
   virtual void SetUp() {}
@@ -19,7 +25,7 @@ class BackendEnv : public ::testing::Test {
 
 void *backendCreate() {
   const char *streamPath = "/tmp";
-  void *      p = NULL;
+  void       *p = NULL;
 
   // char *absPath = NULL;
   // // SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(streamPath, -1, 2);
@@ -42,7 +48,7 @@ SStreamState *stateCreate(const char *path) {
 }
 void *backendOpen() {
   streamMetaInit();
-  const char *  path = "/tmp/backend";
+  const char   *path = "/tmp/backend";
   SStreamState *p = stateCreate(path);
   ASSERT(p != NULL);
 
@@ -65,7 +71,7 @@ void *backendOpen() {
 
     const char *val = "value data";
     int32_t     len = 0;
-    char *      newVal = NULL;
+    char       *newVal = NULL;
     streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
     ASSERT(len == strlen(val));
   }
@@ -80,7 +86,7 @@ void *backendOpen() {
     SWinKey     key = {.groupId = (uint64_t)(i), .ts = ts};
     const char *val = "value data";
     int32_t     len = 0;
-    char *      newVal = NULL;
+    char       *newVal = NULL;
     int32_t     code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
     ASSERT(code != 0);
   }
@@ -107,7 +113,7 @@ void *backendOpen() {
 
   winkey.groupId = 0;
   winkey.ts = tsArray[0];
-  char *  val = NULL;
+  char   *val = NULL;
   int32_t len = 0;
 
   pCurr = streamStateSeekKeyNext_rocksdb(p, &winkey);
@@ -127,14 +133,14 @@ void *backendOpen() {
   }
   for (int i = 0; i < size; i++) {
     STupleKey key = {.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
-    char *    val = NULL;
+    char     *val = NULL;
     int32_t   len = 0;
     streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len);
     ASSERT(len == strlen("Value"));
   }
   for (int i = 0; i < size; i++) {
     STupleKey key = {.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
-    char *    val = NULL;
+    char     *val = NULL;
     int32_t   len = 0;
     streamStateFuncDel_rocksdb(p, &key);
   }
@@ -172,7 +178,7 @@ void *backendOpen() {
 
   {
     SSessionKey key;
-    char *      val = NULL;
+    char       *val = NULL;
     int32_t     vlen = 0;
     code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen);
     ASSERT(code == 0);
@@ -212,7 +218,7 @@ void *backendOpen() {
   }
   for (int i = 0; i < size; i++) {
     SWinKey key = {.groupId = (uint64_t)(i), .ts = tsArray[i]};
-    char *  val = NULL;
+    char   *val = NULL;
     int32_t vlen = 0;
     ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0);
     taosMemoryFreeClear(val);
@@ -222,7 +228,7 @@ void *backendOpen() {
     SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key);
     ASSERT(pCurr != NULL);
 
-    char *  val = NULL;
+    char   *val = NULL;
     int32_t vlen = 0;
     ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen));
     ASSERT(vlen == strlen("Value"));
@@ -244,7 +250,7 @@ void *backendOpen() {
 
   for (int i = 0; i < size - 1; i++) {
     SWinKey key = {.groupId = (uint64_t)(i), .ts = tsArray[i]};
-    char *  val = NULL;
+    char   *val = NULL;
     int32_t vlen = 0;
     ASSERT(streamStateFillDel_rocksdb(p, &key) == 0);
     taosMemoryFreeClear(val);
@@ -286,7 +292,7 @@ void *backendOpen() {
     char key[128] = {0};
     sprintf(key, "tbname_%d", i);
 
-    char *  val = NULL;
+    char   *val = NULL;
     int32_t len = 0;
     code = streamDefaultGet_rocksdb(p, key, (void **)&val, &len);
     ASSERT(code == 0);
@@ -302,7 +308,7 @@ TEST_F(BackendEnv, checkOpen) {
   SStreamState *p = (SStreamState *)backendOpen();
   int64_t       tsStart = taosGetTimestampMs();
   {
-    void *  pBatch = streamStateCreateBatch();
+    void   *pBatch = streamStateCreateBatch();
     int32_t size = 0;
     for (int i = 0; i < size; i++) {
       char key[128] = {0};
@@ -316,7 +322,7 @@ TEST_F(BackendEnv, checkOpen) {
     streamStateDestroyBatch(pBatch);
   }
   {
-    void *  pBatch = streamStateCreateBatch();
+    void   *pBatch = streamStateCreateBatch();
     int32_t size = 0;
     char    valBuf[256] = {0};
     for (int i = 0; i < size; i++) {
@@ -333,7 +339,7 @@ TEST_F(BackendEnv, checkOpen) {
   // do checkpoint 2
   taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2);
   {
-    void *  pBatch = streamStateCreateBatch();
+    void   *pBatch = streamStateCreateBatch();
     int32_t size = 0;
     char    valBuf[256] = {0};
     for (int i = 0; i < size; i++) {
@@ -355,7 +361,7 @@ TEST_F(BackendEnv, checkOpen) {
   // taosMkDir(dump);
   taosMulMkDir(dump);
   SBkdMgt *mgt = bkdMgtCreate((char *)path);
-  SArray * result = taosArrayInit(4, sizeof(void *));
+  SArray  *result = taosArrayInit(4, sizeof(void *));
   bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump);
 
   bkdMgtDestroy(mgt);

From 67559afcae58959b753476c2f63b178df0af0b37 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 28 Feb 2024 14:29:30 +0800
Subject: [PATCH 076/201] fix: check ret value of lseek

---
 source/libs/executor/src/tsort.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 1d89e6d15c..3b2915f81d 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1035,7 +1035,7 @@ static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char*
     }
     {
       int64_t ret = taosLSeekFile(pMemFile->pTdFile, ((int64_t)pageId) * pMemFile->pageSize, SEEK_SET);
-      if (ret == 0) {
+      if (ret >= 0) {
         ret = taosReadFile(pMemFile->pTdFile, pEntry->data, pMemFile->pageSize);
       }
       if (ret != pMemFile->pageSize) {
@@ -1154,7 +1154,7 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p
 
       if (pMemFile->currPageId - pMemFile->startPageId >= pMemFile->numWritePages) {
         int64_t ret = taosLSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
-        if (ret == 0) {
+        if (ret >= 0) {
           ret = taosWriteFile(pMemFile->pTdFile, pMemFile->writePageBuf, pMemFile->pageSize * pMemFile->numWritePages);
         }
         if (ret !=  pMemFile->pageSize * pMemFile->numWritePages) {
@@ -1183,8 +1183,8 @@ static int32_t saveDirtyPagesToExtRowsMemFile(SSortHandle* pHandle) {
     return TSDB_CODE_SUCCESS;
   }
   int64_t ret = taosLSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
-  int32_t numWriteBytes = pMemFile->pageSize * (pMemFile->currPageId - pMemFile->startPageId) + pMemFile->currPageOffset + 1;
-  if (ret == 0) {
+  int32_t numWriteBytes = pMemFile->pageSize * (pMemFile->currPageId - pMemFile->startPageId + 1);
+  if (ret >= 0) {
     ret = taosWriteFile(pMemFile->pTdFile, pMemFile->writePageBuf, numWriteBytes);
   }
   if (ret != numWriteBytes) {
@@ -1460,9 +1460,6 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray*
     }
     blockDataCleanup(pHandle->pDataBlock);
   }
-  if (pHandle->bSortByRowId) {
-    saveDirtyPagesToExtRowsMemFile(pHandle);
-  }
 
   SSDataBlock* pMemSrcBlk = createOneDataBlock(pHandle->pDataBlock, false);
   doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId);
@@ -1631,6 +1628,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   taosArrayDestroy(aExtSrc);
   tSimpleHashCleanup(mTableNumRows);
   if (pHandle->bSortByRowId) {
+    saveDirtyPagesToExtRowsMemFile(pHandle);
     freeExtRowMemFileWriteBuf(pHandle);
   }
   pHandle->type = SORT_SINGLESOURCE_SORT;

From 2fb0c1a00f1136f2981101bb20cabd7772646cd6 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 28 Feb 2024 14:31:31 +0800
Subject: [PATCH 077/201] feat: check temp space availability

---
 source/libs/executor/src/scanoperator.c | 12 ++++++++----
 source/libs/executor/src/tsort.c        |  5 +++++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 7f1f0700d0..1d64b066e9 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4015,9 +4015,10 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
     pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
                                                pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
     int32_t memSize = 512 * 1024 * 1024;
-    // int32_t rowBytes = blockDataGetRowSize(pInfo->pResBlock) + taosArrayGetSize(pInfo->pResBlock->pDataBlock) + sizeof(int32_t);
-    // int32_t pageSize = TMAX(memSize/numOfTable, rowBytes);
-    tsortSetSortByRowId(pInfo->pSortHandle, pInfo->bufPageSize, memSize);
+    code = tsortSetSortByRowId(pInfo->pSortHandle, pInfo->bufPageSize, memSize);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
   } else {
     pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
                                                pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
@@ -4210,7 +4211,10 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
     } else {
       if (pInfo->bNewFilesetEvent) {
         stopDurationForGroupTableMergeScan(pOperator);
-        startDurationForGroupTableMergeScan(pOperator);
+        code = startDurationForGroupTableMergeScan(pOperator);
+        if (code != TSDB_CODE_SUCCESS) {
+          T_LONG_JMP(pTaskInfo->env, terrno);
+        }
       } else {
         // Data of this group are all dumped, let's try the next group
         stopGroupTableMergeScan(pOperator);
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 3b2915f81d..506c79ebaf 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1266,6 +1266,11 @@ int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsPageSize, int32
   SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);
   pHandle->extRowsOrderInfo = *pOrder;
   initRowIdSort(pHandle);
+  if (!osTempSpaceAvailable()) {
+    terrno = TSDB_CODE_NO_DISKSPACE;
+    qError("create sort mem file failed since %s, tempDir:%s", terrstr(), tsTempDir);
+    return terrno;
+  }
   int32_t code = createSortMemFile(pHandle);
   pHandle->bSortByRowId = true;
   return code;

From d5c5dc70269cb8fa498d799d6797361d73b09962 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 28 Feb 2024 06:45:35 +0000
Subject: [PATCH 078/201] add test case

---
 source/libs/stream/test/backendTest.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index 3245979633..9165beaaec 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -178,8 +178,9 @@ void *backendOpen() {
 
   {
     SSessionKey key;
-    char       *val = NULL;
-    int32_t     vlen = 0;
+    memset(&key, 0, sizeof(key));
+    char   *val = NULL;
+    int32_t vlen = 0;
     code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen);
     ASSERT(code == 0);
     pCurr = streamStateSessionSeekKeyPrev_rocksdb(p, &key);

From 64f2dc607a5b698bfbcc66b60d2b67cb94c77e52 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Wed, 28 Feb 2024 15:08:48 +0800
Subject: [PATCH 079/201] fix(tdb/pcache): not use lru page if already in hash

---
 source/libs/tdb/src/db/tdbPCache.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c
index 3ee65f11dd..455128e6ec 100644
--- a/source/libs/tdb/src/db/tdbPCache.c
+++ b/source/libs/tdb/src/db/tdbPCache.c
@@ -316,7 +316,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
   }
 
   // 3. Try to Recycle a page
-  if (!pPage && !pCache->lru.pLruPrev->isAnchor) {
+  if (!pPageH && !pPage && !pCache->lru.pLruPrev->isAnchor) {
     pPage = pCache->lru.pLruPrev;
     tdbPCacheRemovePageFromHash(pCache, pPage);
     tdbPCachePinPage(pCache, pPage);

From 249141a9a25ba5c04cef6a8ab4a6b1b8c9590983 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 28 Feb 2024 08:16:02 +0000
Subject: [PATCH 080/201] add compress dict

---
 source/libs/stream/test/backendTest.cpp | 35 ++++++++++++++++++-------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index 9165beaaec..b07c860da8 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -16,6 +16,10 @@
 #pragma GCC diagnostic ignored "-Wunused-function"
 #pragma GCC diagnostic ignored "-Wunused-variable"
 #pragma GCC diagnostic ignored "-Wsign-compare"
+#pragma GCC diagnostic ignored "-Wsign-compare"
+#pragma GCC diagnostic ignored "-Wformat"
+#pragma GCC diagnostic ignored "-Wint-to-pointer-cast"
+#pragma GCC diagnostic ignored "-Wpointer-arith"
 
 class BackendEnv : public ::testing::Test {
  protected:
@@ -56,18 +60,22 @@ void *backendOpen() {
   // default/state/fill/sess/func/parname/partag
   int32_t              size = 100;
   std::vector<int64_t> tsArray;
-  for (int i = 0; i < size; i++) {
-    int64_t     ts = taosGetTimestampMs();
-    SWinKey     key = {.groupId = (uint64_t)(i), .ts = ts};
+  for (int32_t i = 0; i < size; i++) {
+    int64_t ts = taosGetTimestampMs();
+    SWinKey key;  // = {.groupId = (uint64_t)(i), .ts = ts};
+    key.groupId = (uint64_t)(i);
+    key.ts = ts;
     const char *val = "value data";
     int32_t     vlen = strlen(val);
     streamStatePut_rocksdb(p, &key, (char *)val, vlen);
 
     tsArray.push_back(ts);
   }
-  for (int i = 0; i < size; i++) {
+  for (int32_t i = 0; i < size; i++) {
     int64_t ts = tsArray[i];
-    SWinKey key = {.groupId = (uint64_t)(i), .ts = ts};
+    SWinKey key = {0};  //{.groupId = (uint64_t)(i), .ts = ts};
+    key.groupId = (uint64_t)(i);
+    key.ts = ts;
 
     const char *val = "value data";
     int32_t     len = 0;
@@ -76,14 +84,20 @@ void *backendOpen() {
     ASSERT(len == strlen(val));
   }
   int64_t ts = tsArray[0];
-  SWinKey key = {.groupId = (uint64_t)(0), .ts = ts};
+  SWinKey key = {0};  // {.groupId = (uint64_t)(0), .ts = ts};
+  key.groupId = (uint64_t)(0);
+  key.ts = ts;
+
   streamStateDel_rocksdb(p, &key);
 
   streamStateClear_rocksdb(p);
 
   for (int i = 0; i < size; i++) {
-    int64_t     ts = tsArray[i];
-    SWinKey     key = {.groupId = (uint64_t)(i), .ts = ts};
+    int64_t ts = tsArray[i];
+    SWinKey key = {0};  //{.groupId = (uint64_t)(i), .ts = ts};
+    key.groupId = (uint64_t)(i);
+    key.ts = ts;
+
     const char *val = "value data";
     int32_t     len = 0;
     char       *newVal = NULL;
@@ -96,7 +110,10 @@ void *backendOpen() {
     int64_t ts = taosGetTimestampMs();
     tsArray.push_back(ts);
 
-    SWinKey     key = {.groupId = (uint64_t)(i), .ts = ts};
+    SWinKey key = {0};  //{.groupId = (uint64_t)(i), .ts = ts};
+    key.groupId = (uint64_t)(i);
+    key.ts = ts;
+
     const char *val = "value data";
     int32_t     vlen = strlen(val);
     streamStatePut_rocksdb(p, &key, (char *)val, vlen);

From 00d6611ba77323bf4a403166f52b666d884d2e59 Mon Sep 17 00:00:00 2001
From: dmchen <cademfly@hotmail.com>
Date: Wed, 28 Feb 2024 08:31:45 +0000
Subject: [PATCH 081/201] fix/TD-28895

---
 source/dnode/mnode/impl/src/mndCompact.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c
index deaaf7f2af..de45e8fe9a 100644
--- a/source/dnode/mnode/impl/src/mndCompact.c
+++ b/source/dnode/mnode/impl/src/mndCompact.c
@@ -369,7 +369,7 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
     return -1;
   }
   mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
-
+/*
   SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
   if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
     mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
@@ -377,7 +377,7 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
     return -1;
   }
   (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
-
+*/
   void   *pIter = NULL;
   while (1) {
     SCompactDetailObj *pDetail = NULL;

From 8c46c71c98f0eb190be98e0bd556f0a6d848ff11 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 28 Feb 2024 09:02:55 +0000
Subject: [PATCH 082/201] add test case

---
 source/libs/stream/test/backendTest.cpp | 56 ++++++++++++++++++-------
 1 file changed, 42 insertions(+), 14 deletions(-)

diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index b07c860da8..701ea0a75b 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -142,23 +142,34 @@ void *backendOpen() {
   for (int i = 0; i < size; i++) {
     int64_t ts = taosGetTimestampMs();
     tsArray.push_back(ts);
-    STupleKey key = {.groupId = (uint64_t)(0), .ts = ts, .exprIdx = i};
+    STupleKey key = {0};
+    key.groupId = (uint64_t)(0);  //= {.groupId = (uint64_t)(0), .ts = ts, .exprIdx = i};
+    key.ts = ts;
+    key.exprIdx = i;
 
     const char *val = "Value";
     int32_t     len = strlen(val);
     streamStateFuncPut_rocksdb(p, &key, val, len);
   }
   for (int i = 0; i < size; i++) {
-    STupleKey key = {.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
-    char     *val = NULL;
-    int32_t   len = 0;
+    STupleKey key = {0};  //{.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
+    key.groupId = (uint64_t)(0);
+    key.ts = tsArray[i];
+    key.exprIdx = i;
+
+    char   *val = NULL;
+    int32_t len = 0;
     streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len);
     ASSERT(len == strlen("Value"));
   }
   for (int i = 0; i < size; i++) {
-    STupleKey key = {.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
-    char     *val = NULL;
-    int32_t   len = 0;
+    STupleKey key = {0};  //{.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
+    key.groupId = (uint64_t)(0);
+    key.ts = tsArray[i];
+    key.exprIdx = i;
+
+    char   *val = NULL;
+    int32_t len = 0;
     streamStateFuncDel_rocksdb(p, &key);
   }
 
@@ -166,7 +177,10 @@ void *backendOpen() {
   tsArray.clear();
 
   for (int i = 0; i < size; i++) {
-    SSessionKey key = {.win = {.skey = i, .ekey = i}, .groupId = (uint64_t)(0)};
+    SSessionKey key = {0};  //{.win = {.skey = i, .ekey = i}, .groupId = (uint64_t)(0)};
+    key.win.skey = i;
+    key.win.ekey = i;
+    key.groupId = (uint64_t)(0);
     tsArray.push_back(i);
 
     const char *val = "Value";
@@ -179,7 +193,10 @@ void *backendOpen() {
   }
 
   for (int i = 0; i < size; i++) {
-    SSessionKey key = {.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)};
+    SSessionKey key = {0};  //{.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)};
+    key.win.skey = tsArray[i];
+    key.win.ekey = tsArray[i];
+    key.groupId = (uint64_t)(0);
 
     const char *val = "Value";
     int32_t     len = strlen(val);
@@ -219,7 +236,10 @@ void *backendOpen() {
            streamStateStateAddIfNotExist_rocksdb(p, &key, (char *)"key", strlen("key"), NULL, (void **)&val, &len));
   }
   for (int i = 0; i < size; i++) {
-    SSessionKey key = {.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)};
+    SSessionKey key = {0};  //{.win = {.skey = tsArray[i], .ekey = tsArray[i]}, .groupId = (uint64_t)(0)};
+    key.win.skey = tsArray[i];
+    key.win.ekey = tsArray[i];
+    key.groupId = (uint64_t)(0);
 
     const char *val = "Value";
     int32_t     len = strlen(val);
@@ -229,20 +249,26 @@ void *backendOpen() {
   }
 
   for (int i = 0; i < size; i++) {
-    SWinKey     key = {.groupId = (uint64_t)(i), .ts = tsArray[i]};
+    SWinKey key = {0};  // {.groupId = (uint64_t)(i), .ts = tsArray[i]};
+    key.groupId = (uint64_t)(i);
+    key.ts = tsArray[i];
     const char *val = "Value";
     int32_t     vlen = strlen(val);
     ASSERT(streamStateFillPut_rocksdb(p, &key, val, vlen) == 0);
   }
   for (int i = 0; i < size; i++) {
-    SWinKey key = {.groupId = (uint64_t)(i), .ts = tsArray[i]};
+    SWinKey key = {0};  // {.groupId = (uint64_t)(i), .ts = tsArray[i]};
+    key.groupId = (uint64_t)(i);
+    key.ts = tsArray[i];
     char   *val = NULL;
     int32_t vlen = 0;
     ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0);
     taosMemoryFreeClear(val);
   }
   {
-    SWinKey          key = {.groupId = (uint64_t)(0), .ts = tsArray[0]};
+    SWinKey key = {0};  //{.groupId = (uint64_t)(0), .ts = tsArray[0]};
+    key.groupId = (uint64_t)(0);
+    key.ts = tsArray[0];
     SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key);
     ASSERT(pCurr != NULL);
 
@@ -267,7 +293,9 @@ void *backendOpen() {
   }
 
   for (int i = 0; i < size - 1; i++) {
-    SWinKey key = {.groupId = (uint64_t)(i), .ts = tsArray[i]};
+    SWinKey key = {0};  // {.groupId = (uint64_t)(i), .ts = tsArray[i]};
+    key.groupId = (uint64_t)(i);
+    key.ts = tsArray[i];
     char   *val = NULL;
     int32_t vlen = 0;
     ASSERT(streamStateFillDel_rocksdb(p, &key) == 0);

From eec6b668772e32983d4b54e32c3c51400a2514cb Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 28 Feb 2024 17:05:11 +0800
Subject: [PATCH 083/201] fix:[TD-28869]get error in askEp because consumer is
 dropped when unsubscribe topic

---
 source/client/src/clientTmq.c | 30 ++++--------------------------
 1 file changed, 4 insertions(+), 26 deletions(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 9b74456da2..3ce4e97e22 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -1009,19 +1009,8 @@ int32_t tmq_unsubscribe(tmq_t* tmq) {
   }
   taosSsleep(2);  // sleep 2s for hb to send offset and rows to server
 
-  int32_t     rsp;
-  int32_t     retryCnt = 0;
   tmq_list_t* lst = tmq_list_new();
-  while (1) {
-    rsp = tmq_subscribe(tmq, lst);
-    if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) {
-      break;
-    } else {
-      retryCnt++;
-      taosMsleep(500);
-    }
-  }
-
+  int32_t rsp = tmq_subscribe(tmq, lst);
   tmq_list_destroy(lst);
   return rsp;
 }
@@ -1271,10 +1260,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   }
 
   int32_t retryCnt = 0;
-  while (syncAskEp(tmq) != 0) {
-    if (retryCnt++ > MAX_RETRY_COUNT) {
+  while ((code = syncAskEp(tmq)) != 0) {
+    if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) {
       tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes", tmq->consumerId);
-      code = TSDB_CODE_MND_CONSUMER_NOT_READY;
       goto FAIL;
     }
 
@@ -2154,18 +2142,8 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
     }
     taosSsleep(2);  // sleep 2s for hb to send offset and rows to server
 
-    int32_t     retryCnt = 0;
     tmq_list_t* lst = tmq_list_new();
-    while (1) {
-      int32_t rsp = tmq_subscribe(tmq, lst);
-      if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) {
-        break;
-      } else {
-        retryCnt++;
-        taosMsleep(500);
-      }
-    }
-
+    tmq_subscribe(tmq, lst);
     tmq_list_destroy(lst);
   } else {
     tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId);

From 17e9f1d747c4e5371f2af813dabe70c120eafbea Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 28 Feb 2024 09:33:48 +0000
Subject: [PATCH 084/201] add test case

---
 source/libs/stream/test/backendTest.cpp | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index 701ea0a75b..a949748eb5 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -422,17 +422,8 @@ typedef struct BdKV {
   uint32_t v;
 } BdKV;
 
-BdKV kvDict[] = {{0, 2},
-                 {1, 2},
-                 {.k = 15, .v = 16},
-                 {.k = 31, .v = 32},
-                 {.k = 56, .v = 64},
-                 {.k = 100, .v = 128},
-                 {.k = 200, .v = 256},
-                 {.k = 500, .v = 512},
-                 {.k = 1000, .v = 1024},
-                 {.k = 2000, .v = 2048},
-                 {.k = 3000, .v = 4096}};
+BdKV kvDict[] = {{0, 2},     {1, 2},     {15, 16},     {31, 32},     {56, 64},    {100, 128},
+                 {200, 256}, {500, 512}, {1000, 1024}, {2000, 2048}, {3000, 4096}};
 
 TEST_F(BackendEnv, backendUtil) {
   for (int i = 0; i < sizeof(kvDict) / sizeof(kvDict[0]); i++) {

From 2b5cf56556e82ae3bb678c21709fb9ee1d3776f2 Mon Sep 17 00:00:00 2001
From: Shungang Li <sgli@taosdata.com>
Date: Wed, 28 Feb 2024 18:50:18 +0800
Subject: [PATCH 085/201] fix: shellReadHistory access invalid memory

---
 tools/shell/src/shellEngine.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c
index 23424cea98..995d3d04ec 100644
--- a/tools/shell/src/shellEngine.c
+++ b/tools/shell/src/shellEngine.c
@@ -75,7 +75,7 @@ bool shellIsEmptyCommand(const char *cmd) {
 
 int32_t shellRunSingleCommand(char *command) {
   shellCmdkilled = false;
-  
+
   if (shellIsEmptyCommand(command)) {
     return 0;
   }
@@ -1019,7 +1019,7 @@ void shellReadHistory() {
 
   char   *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
   int32_t read_size = 0;
-  while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) {
+  while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) > 0) {
     line[read_size - 1] = '\0';
     taosMemoryFree(pHistory->hist[pHistory->hend]);
     pHistory->hist[pHistory->hend] = taosStrdup(line);
@@ -1315,7 +1315,7 @@ int32_t shellExecute() {
   shellSetConn(shell.conn, runOnce);
   shellReadHistory();
 
- if(shell.args.is_bi_mode) { 
+ if(shell.args.is_bi_mode) {
   // need set bi mode
   printf("Set BI mode is true.\n");
 #ifndef WEBSOCKET

From dd7f375d8e6e4b1c31d4eee6d22e9b97e12da85f Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Wed, 28 Feb 2024 19:01:22 +0800
Subject: [PATCH 086/201] enh: use reference id of sync node in timer callbacks
 of sync

---
 source/libs/sync/src/syncMain.c | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index e06ea70f70..7ff6116137 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -1343,7 +1343,7 @@ ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->raftCfg
 int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) {
   int32_t ret = 0;
   if (syncIsInit()) {
-    taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, syncEnv()->pTimerManager,
+    taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, (void*)pSyncNode->rid, syncEnv()->pTimerManager,
                  &pSyncNode->pPingTimer);
     atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser);
   } else {
@@ -1415,8 +1415,8 @@ void syncNodeResetElectTimer(SSyncNode* pSyncNode) {
 static int32_t syncNodeDoStartHeartbeatTimer(SSyncNode* pSyncNode) {
   int32_t ret = 0;
   if (syncIsInit()) {
-    taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, syncEnv()->pTimerManager,
-                 &pSyncNode->pHeartbeatTimer);
+    taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, (void*)pSyncNode->rid,
+                 syncEnv()->pTimerManager, &pSyncNode->pHeartbeatTimer);
     atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser);
   } else {
     sError("vgId:%d, start heartbeat timer error, sync env is stop", pSyncNode->vgId);
@@ -2153,7 +2153,11 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex
 static void syncNodeEqPingTimer(void* param, void* tmrId) {
   if (!syncIsInit()) return;
 
-  SSyncNode* pNode = param;
+  int64_t    rid = (int64_t)param;
+  SSyncNode* pNode = syncNodeAcquire(rid);
+
+  if (pNode == NULL) return;
+
   if (atomic_load_64(&pNode->pingTimerLogicClockUser) <= atomic_load_64(&pNode->pingTimerLogicClock)) {
     SRpcMsg rpcMsg = {0};
     int32_t code = syncBuildTimeout(&rpcMsg, SYNC_TIMEOUT_PING, atomic_load_64(&pNode->pingTimerLogicClock),
@@ -2173,7 +2177,8 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) {
     }
 
   _out:
-    taosTmrReset(syncNodeEqPingTimer, pNode->pingTimerMS, pNode, syncEnv()->pTimerManager, &pNode->pPingTimer);
+    taosTmrReset(syncNodeEqPingTimer, pNode->pingTimerMS, (void*)pNode->rid, syncEnv()->pTimerManager,
+                 &pNode->pPingTimer);
   }
 }
 
@@ -2224,7 +2229,11 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) {
 static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {
   if (!syncIsInit()) return;
 
-  SSyncNode* pNode = param;
+  int64_t    rid = (int64_t)param;
+  SSyncNode* pNode = syncNodeAcquire(rid);
+
+  if (pNode == NULL) return;
+
   if (pNode->totalReplicaNum > 1) {
     if (atomic_load_64(&pNode->heartbeatTimerLogicClockUser) <= atomic_load_64(&pNode->heartbeatTimerLogicClock)) {
       SRpcMsg rpcMsg = {0};
@@ -2245,7 +2254,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {
       }
 
     _out:
-      taosTmrReset(syncNodeEqHeartbeatTimer, pNode->heartbeatTimerMS, pNode, syncEnv()->pTimerManager,
+      taosTmrReset(syncNodeEqHeartbeatTimer, pNode->heartbeatTimerMS, (void*)pNode->rid, syncEnv()->pTimerManager,
                    &pNode->pHeartbeatTimer);
 
     } else {
@@ -3385,4 +3394,4 @@ bool syncNodeCanChange(SSyncNode* pSyncNode) {
 
   return true;
 }
-#endif
\ No newline at end of file
+#endif

From 690c82a49eac51f83edde9f7f4378dc38706816f Mon Sep 17 00:00:00 2001
From: chenhaoran <haoran920c@163.com>
Date: Wed, 28 Feb 2024 20:48:42 +0800
Subject: [PATCH 087/201] test: Splitting test cases to avoid case timeouts

---
 tests/parallel_test/cases.task                |   3 +-
 ...m.py => tmqVnodeTransform-db-removewal.py} | 214 ++++++--------
 .../7-tmq/tmqVnodeTransform-stb-removewal.py  | 266 ++++++++++++++++++
 3 files changed, 350 insertions(+), 133 deletions(-)
 rename tests/system-test/7-tmq/{tmqVnodeTransform.py => tmqVnodeTransform-db-removewal.py} (78%)
 create mode 100644 tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index bdccf33c32..dc9286cffe 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -238,7 +238,8 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True
-,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1
+,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1
+,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3
 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1
 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3
 #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3
diff --git a/tests/system-test/7-tmq/tmqVnodeTransform.py b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py
similarity index 78%
rename from tests/system-test/7-tmq/tmqVnodeTransform.py
rename to tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py
index c2b002ead6..a853489c3f 100644
--- a/tests/system-test/7-tmq/tmqVnodeTransform.py
+++ b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py
@@ -122,135 +122,7 @@ class TDTestCase:
         tdLog.debug(f"redistributeSql:{redistributeSql}")
         tdSql.query(redistributeSql)
         tdLog.debug("redistributeSql ok")
-
-    def tmqCase1(self):
-        tdLog.printNoPrefix("======== test case 1: ")
-        paraDict = {'dbName':     'dbt',
-                    'dropFlag':   1,
-                    'event':      '',
-                    'vgroups':    1,
-                    'stbName':    'stb',
-                    'colPrefix':  'c',
-                    'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
-                    'ctbPrefix':  'ctb',
-                    'ctbStartIdx': 0,
-                    'ctbNum':     10,
-                    'rowsPerTbl': 1000,
-                    'batchNum':   10,
-                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    'pollDelay':  60,
-                    'showMsg':    1,
-                    'showRow':    1,
-                    'snapshot':   0}
-
-        paraDict['vgroups'] = self.vgroups
-        paraDict['ctbNum'] = self.ctbNum
-        paraDict['rowsPerTbl'] = self.rowsPerTbl
-
-        topicNameList = ['topic1']
-        # expectRowsList = []
-        tmqCom.initConsumerTable()
-
-        tdLog.info("create topics from stb with filter")
-        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
-        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
-        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
-        tdLog.info("create topic sql: %s"%sqlString)
-        tdSql.execute(sqlString)
-        # tdSql.query(queryString)
-        # expectRowsList.append(tdSql.getRows())
-
-        # init consume info, and start tmq_sim, then check consume result
-        tdLog.info("insert consume info to consume processor")
-        consumerId   = 0
-        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
-        topicList    = topicNameList[0]
-        ifcheckdata  = 1
-        ifManualCommit = 1
-        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
-        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-
-        tdLog.info("start consume processor")
-        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
-        tdLog.info("wait the consume result")
-
-        tdLog.info("create ctb1")
-        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
-                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
-        tdLog.info("insert data")
-        pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
-
-        tmqCom.getStartConsumeNotifyFromTmqsim()
-        tmqCom.getStartCommitNotifyFromTmqsim()
-
-        #restart dnode & remove wal
-        self.restartAndRemoveWal()
-
-        # redistribute vgroup
-        self.redistributeVgroups();
-
-        tdLog.info("create ctb2")
-        paraDict['ctbPrefix'] = "ctbn"
-        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
-                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
-        tdLog.info("insert data")
-        pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict)
-        pInsertThread.join()
-        pInsertThread1.join()
-
-        expectRows = 1
-        resultList = tmqCom.selectConsumeResult(expectRows)
-
-        if expectrowcnt / 2 > resultList[0]:
-            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0]))
-            tdLog.exit("%d tmq consume rows error!"%consumerId)
-
-        # tmqCom.checkFileContent(consumerId, queryString)
-
-        time.sleep(10)
-        for i in range(len(topicNameList)):
-            tdSql.query("drop topic %s"%topicNameList[i])
-
-        tdLog.printNoPrefix("======== test case 1 end ...... ")
-
-    def tmqCase2(self):
-        tdLog.printNoPrefix("======== test case 2: ")
-        paraDict = {'dbName':'dbt'}
-
-        ntbName = "ntb"
-
-        topicNameList = ['topic2']
-        tmqCom.initConsumerTable()
-
-        sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName)
-        tdLog.info("create nomal table sql: %s"%sqlString)
-        tdSql.execute(sqlString)
-
-        tdLog.info("create topics from nomal table")
-        queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName)
-        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
-        tdLog.info("create topic sql: %s"%sqlString)
-        tdSql.execute(sqlString)
-        tdSql.query("flush database %s"%(paraDict['dbName']))
-        #restart dnode & remove wal
-        self.restartAndRemoveWal()
-
-        # redistribute vgroup
-        self.redistributeVgroups();
-
-        sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName)
-        tdLog.info("alter table sql: %s"%sqlString)
-        tdSql.error(sqlString)
-        expectRows = 0
-        resultList = tmqCom.selectConsumeResult(expectRows)
-        time.sleep(1)
-        for i in range(len(topicNameList)):
-            tdSql.query("drop topic %s"%topicNameList[i])
-
-        tdLog.printNoPrefix("======== test case 2 end ...... ")
-
+        
     def tmqCase3(self):
         tdLog.printNoPrefix("======== test case 3: ")
         paraDict = {'dbName':     'dbt',
@@ -330,12 +202,90 @@ class TDTestCase:
 
         tdLog.printNoPrefix("======== test case 3 end ...... ")
 
+    def tmqCaseDbname(self):
+        tdLog.printNoPrefix("======== test case  4 subscrib Dbname start: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stbn',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        topicNameList = ['topic4']
+        tmqCom.initConsumerTable()
+
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+
+        tdLog.info("create topics from database ")
+        queryString = "database  %s "%(paraDict['dbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result")
+
+        time.sleep(1)
+        # restart dnode & remove wal
+        self.restartAndRemoveWal()
+
+        # redistribute vgroup
+        self.redistributeVgroups()
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result")
+        expectRows = 2
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        time.sleep(6)
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+ 
+        tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ")
+
     def run(self):
-        self.prepareTestEnv()
-        self.tmqCase1()
-        self.tmqCase2()
         self.prepareTestEnv()
         self.tmqCase3()
+        self.prepareTestEnv()
+        self.tmqCaseDbname()
 
     def stop(self):
         tdSql.close()
diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py
new file mode 100644
index 0000000000..40879d5c66
--- /dev/null
+++ b/tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py
@@ -0,0 +1,266 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+from util.cluster import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def __init__(self):
+        self.vgroups    = 1
+        self.ctbNum     = 10
+        self.rowsPerTbl = 1000
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), True)
+
+    def getDataPath(self):
+        selfPath = tdCom.getBuildPath()
+
+        return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*';
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  60,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        tdCom.drop_all_db()
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar)
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        # tdLog.info("create ctb")
+        # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+        #                      ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        # tdLog.info("insert data")
+        # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+        #                                        ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+        #                                        startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+
+        # tdLog.info("restart taosd to ensure that the data falls into the disk")
+        # tdDnodes.stop(1)
+        # tdDnodes.start(1)
+        # tdSql.query("flush database %s"%(paraDict['dbName']))
+        return
+
+    def restartAndRemoveWal(self):
+        tdDnodes = cluster.dnodes
+        tdSql.query("select * from information_schema.ins_vnodes")
+        for result in tdSql.queryResult:
+            if result[2] == 'dbt':
+                tdLog.debug("dnode is %d"%(result[0]))
+                dnodeId = result[0]
+                vnodeId = result[1]
+
+                tdDnodes[dnodeId - 1].stoptaosd()
+                time.sleep(1)
+                dataPath = self.getDataPath()
+                dataPath = dataPath%(dnodeId,vnodeId)
+                os.system('rm -rf ' + dataPath)
+                tdLog.debug("dataPath:%s"%dataPath)
+                tdDnodes[dnodeId - 1].starttaosd()
+                time.sleep(1)
+                break
+        tdLog.debug("restart dnode ok")
+
+    def redistributeVgroups(self):
+        dnodesList = []
+        tdSql.query("show dnodes")
+        for result in tdSql.queryResult:
+            dnodesList.append(result[0])
+        print("dnodeList:",dnodesList)
+        tdSql.query("select * from information_schema.ins_vnodes")
+        vnodeId = 0
+        for result in tdSql.queryResult:
+            if result[2] == 'dbt':
+                tdLog.debug("dnode is %d"%(result[0]))
+                dnodesList.remove(result[0])
+                vnodeId = result[1]
+        print("its all data",dnodesList)
+        # if self.replicaVar == 1:
+        #     redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0])
+        # else:
+        redistributeSql = f"redistribute vgroup {vnodeId} " 
+        for vgdnode in dnodesList:
+            redistributeSql += f"dnode {vgdnode} "
+        print(redistributeSql)
+        
+        tdLog.debug(f"redistributeSql:{redistributeSql}")
+        tdSql.query(redistributeSql)
+        tdLog.debug("redistributeSql ok")
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  60,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        topicNameList = ['topic1']
+        # expectRowsList = []
+        tmqCom.initConsumerTable()
+
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        # tdSql.query(queryString)
+        # expectRowsList.append(tdSql.getRows())
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result")
+
+        tdLog.info("create ctb1")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
+
+        tmqCom.getStartConsumeNotifyFromTmqsim()
+        tmqCom.getStartCommitNotifyFromTmqsim()
+
+        #restart dnode & remove wal
+        self.restartAndRemoveWal()
+
+        # redistribute vgroup
+        self.redistributeVgroups();
+
+        tdLog.info("create ctb2")
+        paraDict['ctbPrefix'] = "ctbn"
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict)
+        pInsertThread.join()
+        pInsertThread1.join()
+
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+
+        if expectrowcnt / 2 > resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0]))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        # tmqCom.checkFileContent(consumerId, queryString)
+
+        time.sleep(10)
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def tmqCase2(self):
+        tdLog.printNoPrefix("======== test case 2: ")
+        paraDict = {'dbName':'dbt'}
+
+        ntbName = "ntb"
+
+        topicNameList = ['topic2']
+        tmqCom.initConsumerTable()
+
+        sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName)
+        tdLog.info("create nomal table sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+
+        tdLog.info("create topics from nomal table")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName)
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query("flush database %s"%(paraDict['dbName']))
+        #restart dnode & remove wal
+        self.restartAndRemoveWal()
+
+        # redistribute vgroup
+        self.redistributeVgroups();
+
+        sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName)
+        tdLog.info("alter table sql: %s"%sqlString)
+        tdSql.error(sqlString)
+        expectRows = 0
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        time.sleep(1)
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 2 end ...... ")
+
+    def run(self):
+        self.prepareTestEnv()
+        self.tmqCase1()
+        self.tmqCase2()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

From e81afaf0a2ae14aca11dab5f778f8ba3ee3b8efd Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 29 Feb 2024 09:12:15 +0800
Subject: [PATCH 088/201] fix: no memory trim when no para tables scan

---
 source/libs/executor/src/scanoperator.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 8cc2f72adb..9024f7a341 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -3763,8 +3763,9 @@ static int32_t stopSubTablesTableMergeScan(STableMergeScanInfo* pInfo) {
 
     taosMemoryFree(pSubTblsInfo);
     pInfo->pSubTablesMergeInfo = NULL;
+
+    taosMemoryTrim(0);
   }
-  taosMemoryTrim(0);
   return TSDB_CODE_SUCCESS;
 }
 

From 049f27bf3e52e1f1346e31182aa714d14b38ad92 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 29 Feb 2024 09:47:16 +0800
Subject: [PATCH 089/201] fix: use C File API

---
 source/libs/executor/src/tsort.c | 42 +++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 506c79ebaf..1ceb5403c9 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -63,8 +63,8 @@ typedef struct SSortMemFile {
   int32_t numMemPages;
   SSHashObj* mActivePages;
 
-  //FILE* pTdFile;
-  TdFilePtr pTdFile;
+  FILE* pTdFile;
+  // TdFilePtr pTdFile;
   char memFilePath[PATH_MAX];
 } SSortMemFile;
 
@@ -239,6 +239,14 @@ void destroyTuple(void* t) {
   }
 }
 
+int tsortSeekFile(FILE* file, int64_t offset, int whence) {
+  #ifdef WINDOWS
+  return _fseeki64(file, offset, whence);
+  #else
+  return fseeko(file, offset, whence);
+  #endif
+}
+
 /**
  *
  * @param type
@@ -1034,11 +1042,11 @@ static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char*
       ++pMemFile->numMemPages;
     }
     {
-      int64_t ret = taosLSeekFile(pMemFile->pTdFile, ((int64_t)pageId) * pMemFile->pageSize, SEEK_SET);
-      if (ret >= 0) {
-        ret = taosReadFile(pMemFile->pTdFile, pEntry->data, pMemFile->pageSize);
+      int ret = tsortSeekFile(pMemFile->pTdFile, ((int64_t)pageId) * pMemFile->pageSize, SEEK_SET);
+      if (ret == 0) {
+        ret = fread(pEntry->data, pMemFile->pageSize, 1, pMemFile->pTdFile);
       }
-      if (ret != pMemFile->pageSize) {
+      if (ret != 1) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         if (freeEntryWhenError) {
           taosMemoryFreeClear(pEntry->data);
@@ -1088,8 +1096,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
   SSortMemFile* pMemFile = taosMemoryCalloc(1, sizeof(SSortMemFile));
 
   taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
-  pMemFile->pTdFile =
-      taosOpenFile(pMemFile->memFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
+  pMemFile->pTdFile = fopen(pMemFile->memFilePath, "w+");
   if (pMemFile->pTdFile == NULL) {
     taosMemoryFree(pMemFile);
     return TAOS_SYSTEM_ERROR(errno);
@@ -1133,7 +1140,8 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
   taosMemoryFree(pMemFile->writePageBuf);
   pMemFile->writePageBuf = NULL;
 
-  taosCloseFile(&pMemFile->pTdFile);
+  fclose(pMemFile->pTdFile);
+  pMemFile->pTdFile = NULL;
   taosRemoveFile(pMemFile->memFilePath);
   taosMemoryFree(pMemFile);
   pHandle->pExtRowsMemFile = NULL;
@@ -1153,11 +1161,11 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p
       pMemFile->currPageOffset = 0;
 
       if (pMemFile->currPageId - pMemFile->startPageId >= pMemFile->numWritePages) {
-        int64_t ret = taosLSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
-        if (ret >= 0) {
-          ret = taosWriteFile(pMemFile->pTdFile, pMemFile->writePageBuf, pMemFile->pageSize * pMemFile->numWritePages);
+        int ret = tsortSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
+        if (ret == 0) {
+          ret = fwrite(pMemFile->writePageBuf, pMemFile->pageSize * pMemFile->numWritePages, 1, pMemFile->pTdFile);
         }
-        if (ret !=  pMemFile->pageSize * pMemFile->numWritePages) {
+        if (ret !=  1) {
           terrno = TAOS_SYSTEM_ERROR(errno);
           return terrno;
         }          
@@ -1182,12 +1190,12 @@ static int32_t saveDirtyPagesToExtRowsMemFile(SSortHandle* pHandle) {
   if (!pMemFile->bDirty) {
     return TSDB_CODE_SUCCESS;
   }
-  int64_t ret = taosLSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
+  int ret = tsortSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
   int32_t numWriteBytes = pMemFile->pageSize * (pMemFile->currPageId - pMemFile->startPageId + 1);
-  if (ret >= 0) {
-    ret = taosWriteFile(pMemFile->pTdFile, pMemFile->writePageBuf, numWriteBytes);
+  if (ret == 0) {
+    ret = fwrite(pMemFile->writePageBuf, numWriteBytes, 1, pMemFile->pTdFile);
   }
-  if (ret != numWriteBytes) {
+  if (ret != 1) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return terrno;
   }

From d37adfbd859f967df7830040f0c9660ba42386f6 Mon Sep 17 00:00:00 2001
From: Adam Ji <jrgwyz@163.com>
Date: Thu, 29 Feb 2024 10:31:28 +0800
Subject: [PATCH 090/201] docs: fix python demo code

---
 docs/en/08-client-libraries/07-python.mdx | 14 +++++++-------
 docs/zh/08-connector/30-python.mdx        | 14 +++++++-------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/docs/en/08-client-libraries/07-python.mdx b/docs/en/08-client-libraries/07-python.mdx
index aacfd0fe53..556bb84042 100644
--- a/docs/en/08-client-libraries/07-python.mdx
+++ b/docs/en/08-client-libraries/07-python.mdx
@@ -887,13 +887,13 @@ The `poll` function is used to consume data in tmq. The parameter of the `poll`
 
 ```python
 while True:
-    res = consumer.poll(1)
-    if not res:
+    message = consumer.poll(1)
+    if not message:
         continue
-    err = res.error()
+    err = message.error()
     if err is not None:
         raise err
-    val = res.value()
+    val = message.value()
 
     for block in val:
         print(block.fetchall())
@@ -906,10 +906,10 @@ The `poll` function is used to consume data in tmq. The parameter of the `poll`
 
 ```python
 while True:
-    res = consumer.poll(timeout=1.0)
-    if not res:
+    message = consumer.poll(1)
+    if not message:
         continue
-    err = res.error()
+    err = message.error()
     if err is not None:
         raise err
     for block in message:
diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx
index e7160ab094..d49c96b69f 100644
--- a/docs/zh/08-connector/30-python.mdx
+++ b/docs/zh/08-connector/30-python.mdx
@@ -896,13 +896,13 @@ Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 flo
 
 ```python
 while True:
-    res = consumer.poll(1)
-    if not res:
+    message = consumer.poll(1)
+    if not message:
         continue
-    err = res.error()
+    err = message.error()
     if err is not None:
         raise err
-    val = res.value()
+    val = message.value()
 
     for block in val:
         print(block.fetchall())
@@ -915,10 +915,10 @@ Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 flo
 
 ```python
 while True:
-    res = consumer.poll(timeout=1.0)
-    if not res:
+    message = consumer.poll(1)
+    if not message:
         continue
-    err = res.error()
+    err = message.error()
     if err is not None:
         raise err
     for block in message:

From 7f7b61af5cc9b58c1b60cf023bbae4c5cd23743d Mon Sep 17 00:00:00 2001
From: Adam Ji <jrgwyz@163.com>
Date: Tue, 20 Feb 2024 10:03:29 +0800
Subject: [PATCH 091/201] docs: fix connector-python create consumer

---
 docs/en/08-client-libraries/07-python.mdx | 4 ++--
 docs/zh/08-connector/30-python.mdx        | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/08-client-libraries/07-python.mdx b/docs/en/08-client-libraries/07-python.mdx
index 556bb84042..82b889e0d3 100644
--- a/docs/en/08-client-libraries/07-python.mdx
+++ b/docs/en/08-client-libraries/07-python.mdx
@@ -842,12 +842,12 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"})
 
 In addition to native connections, the client library also supports subscriptions via websockets.
 
-The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer).
+The syntax for creating a consumer is "consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer).
 
 ```python
 import taosws
 
-consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
+consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
 ```
 
 </TabItem>
diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx
index d49c96b69f..491566e4e3 100644
--- a/docs/zh/08-connector/30-python.mdx
+++ b/docs/zh/08-connector/30-python.mdx
@@ -856,7 +856,7 @@ taosws `Consumer` API 提供了基于 Websocket 订阅 TMQ 数据的 API。创
 ```python
 import taosws
 
-consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
+consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
 ```
 
 </TabItem>

From d5eda8c0a9368bf111f2174883515fe1023b9b5c Mon Sep 17 00:00:00 2001
From: Benguang Zhao <bgzhao@taosdata.com>
Date: Thu, 29 Feb 2024 10:42:27 +0800
Subject: [PATCH 092/201] enh: adjust an error logging msg in process sync msg

---
 source/dnode/mnode/impl/src/mndMain.c  | 3 ++-
 source/dnode/vnode/src/vnd/vnodeSync.c | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c
index b188d314d9..69a0bd477d 100644
--- a/source/dnode/mnode/impl/src/mndMain.c
+++ b/source/dnode/mnode/impl/src/mndMain.c
@@ -709,7 +709,8 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
 
   int32_t code = syncProcessMsg(pMgmt->sync, pMsg);
   if (code != 0) {
-    mGError("vgId:1, failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr());
+    mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType),
+            terrstr(), code);
   }
 
   return code;
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index 5f4b7b8442..f9f2ae6b21 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -372,8 +372,8 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
 
   int32_t code = syncProcessMsg(pVnode->sync, pMsg);
   if (code != 0) {
-    vGError("vgId:%d, failed to process sync msg:%p type:%s since %s", pVnode->config.vgId, pMsg,
-            TMSG_INFO(pMsg->msgType), terrstr());
+    vGError("vgId:%d, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pVnode->config.vgId, pMsg,
+            TMSG_INFO(pMsg->msgType), terrstr(), code);
   }
 
   return code;

From 85b78b0947285c26d85ec0e1580345e4203de4f6 Mon Sep 17 00:00:00 2001
From: dmchen <cademfly@hotmail.com>
Date: Thu, 29 Feb 2024 02:43:15 +0000
Subject: [PATCH 093/201] fix/TD-28910

---
 source/dnode/mnode/impl/src/mndCompact.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c
index de45e8fe9a..83254b0f73 100644
--- a/source/dnode/mnode/impl/src/mndCompact.c
+++ b/source/dnode/mnode/impl/src/mndCompact.c
@@ -363,13 +363,15 @@ static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *p
 }
 
 static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) {
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "kill-compact");
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact");
   if (pTrans == NULL) {
     mError("compact:%" PRId32 ", failed to drop since %s" , pCompact->compactId, terrstr());
     return -1;
   }
   mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
-/*
+
+  mndTransSetDbName(pTrans, pCompact->dbname, NULL);
+
   SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
   if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
     mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
@@ -377,8 +379,8 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
     return -1;
   }
   (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
-*/
-  void   *pIter = NULL;
+
+  void *pIter = NULL;
   while (1) {
     SCompactDetailObj *pDetail = NULL;
     pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail);
@@ -612,15 +614,17 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
     return 0;
   }
 
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "update-compact-progress");
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress");
   if (pTrans == NULL) {
     mError("trans:%" PRId32 ", failed to create since %s" , pTrans->id, terrstr());
     return -1;
   }
   mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id);
-
+  
   SCompactObj   *pCompact = mndAcquireCompact(pMnode, compactId);
 
+  mndTransSetDbName(pTrans, pCompact->dbname, NULL);
+
   pIter = NULL;
   while (1) {
     SCompactDetailObj *pDetail = NULL;

From d4900d06339cd2085663d133421f2dfbe7a2fe6e Mon Sep 17 00:00:00 2001
From: Adam Ji <jrgwyz@163.com>
Date: Thu, 29 Feb 2024 10:42:12 +0800
Subject: [PATCH 094/201] docs: fix python tmq demo

---
 docs/en/08-client-libraries/07-python.mdx | 6 ++----
 docs/zh/08-connector/30-python.mdx        | 6 ++----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/docs/en/08-client-libraries/07-python.mdx b/docs/en/08-client-libraries/07-python.mdx
index 82b889e0d3..3110afcf10 100644
--- a/docs/en/08-client-libraries/07-python.mdx
+++ b/docs/en/08-client-libraries/07-python.mdx
@@ -902,16 +902,14 @@ while True:
 </TabItem>
 <TabItem value="websocket" label="WebSocket connection">
 
-The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data.
+The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. 
 
 ```python
 while True:
     message = consumer.poll(1)
     if not message:
         continue
-    err = message.error()
-    if err is not None:
-        raise err
+    
     for block in message:
         for row in block:
             print(row)
diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx
index 491566e4e3..71dc82316e 100644
--- a/docs/zh/08-connector/30-python.mdx
+++ b/docs/zh/08-connector/30-python.mdx
@@ -911,16 +911,14 @@ while True:
 </TabItem>
 <TabItem value="websocket" label="WebSocket 连接">
 
-Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。
+Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。
 
 ```python
 while True:
     message = consumer.poll(1)
     if not message:
         continue
-    err = message.error()
-    if err is not None:
-        raise err
+    
     for block in message:
         for row in block:
             print(row)

From 551a65e16108deef13dd5149459bfbb080f0f5d9 Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Thu, 29 Feb 2024 10:44:25 +0800
Subject: [PATCH 095/201] fix: varmeta length reset

---
 source/common/src/tdatablock.c | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index 6b6c3642de..a20ecd344d 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1688,17 +1688,19 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) {
 }
 
 static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
+  if (n >= total || n == 0) return;
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
-    int32_t newLen = pColInfoData->varmeta.offset[n - 1];
-    if (newLen != -1) {
-      if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
-        newLen += getJsonValueLen(pColInfoData->pData + newLen);
-      } else {
-        newLen += varDataTLen(pColInfoData->pData + newLen);
+    int32_t newLen = -1;
+    for (int i = n; i >= 0; --i) {
+      if (pColInfoData->varmeta.offset[i] != -1) {
+        newLen = pColInfoData->varmeta.offset[i];
+        break;
       }
-    } else {
-      newLen = pColInfoData->varmeta.length;
     }
+    if(-1 == newLen) {
+      return;
+    }
+
     pColInfoData->varmeta.length = newLen;
     // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
     memset(&pColInfoData->varmeta.offset[n], 0, total - n);

From 6619175fdec33438d1793288a53c3040db2e4bed Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 29 Feb 2024 09:16:31 +0800
Subject: [PATCH 096/201] opt bloom filter

---
 include/util/tscalablebf.h            |  2 ++
 source/libs/stream/src/streamUpdate.c |  6 ++++--
 source/util/src/tscalablebf.c         | 21 +++++++++++++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/include/util/tscalablebf.h b/include/util/tscalablebf.h
index 2cf170cf04..d3ce2eb23b 100644
--- a/include/util/tscalablebf.h
+++ b/include/util/tscalablebf.h
@@ -26,6 +26,8 @@ typedef struct SScalableBf {
   SArray  *bfArray;  // array of bloom filters
   uint32_t growth;
   uint64_t numBits;
+  uint32_t maxBloomFilters;
+  int8_t   status;
   _hash_fn_t hashFn1;
   _hash_fn_t hashFn2;
 } SScalableBf;
diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c
index 454ed4297c..34ae909f03 100644
--- a/source/libs/stream/src/streamUpdate.c
+++ b/source/libs/stream/src/streamUpdate.c
@@ -22,7 +22,7 @@
 #define DEFAULT_MAP_CAPACITY     131072
 #define DEFAULT_MAP_SIZE         (DEFAULT_MAP_CAPACITY * 100)
 #define ROWS_PER_MILLISECOND     1
-#define MAX_NUM_SCALABLE_BF      100000
+#define MAX_NUM_SCALABLE_BF      32
 #define MIN_NUM_SCALABLE_BF      10
 #define DEFAULT_PREADD_BUCKET    1
 #define MAX_INTERVAL             MILLISECOND_PER_MINUTE
@@ -81,7 +81,9 @@ static int64_t adjustInterval(int64_t interval, int32_t precision) {
 static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t watermark) {
   if (watermark <= adjInterval) {
     watermark = TMAX(originInt / adjInterval, 1) * adjInterval;
-  } else if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) {
+  }
+
+  if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) {
     watermark = MAX_NUM_SCALABLE_BF * adjInterval;
   }
   return watermark;
diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c
index 3b4975b701..a6e5b563d8 100644
--- a/source/util/src/tscalablebf.c
+++ b/source/util/src/tscalablebf.c
@@ -20,6 +20,9 @@
 
 #define DEFAULT_GROWTH           2
 #define DEFAULT_TIGHTENING_RATIO 0.5
+#define DEFAULT_MAX_BLOOMFILTERS 5
+#define SBF_INVALID              -1
+#define SBF_VALID                0
 
 static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate);
 
@@ -41,10 +44,15 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) {
   pSBf->growth = DEFAULT_GROWTH;
   pSBf->hashFn1 = HASH_FUNCTION_1;
   pSBf->hashFn2 = HASH_FUNCTION_2;
+  pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS;
+  pSBf->status = SBF_VALID;
   return pSBf;
 }
 
 int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
+  if (pSBf->status == SBF_INVALID) {
+    return TSDB_CODE_FAILED;
+  }
   int32_t size = taosArrayGetSize(pSBf->bfArray);
   SBloomFilter *pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
   ASSERT(pNormalBf);
@@ -52,6 +60,7 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le
     pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth,
                                      pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO);
     if (pNormalBf == NULL) {
+      pSBf->status = SBF_INVALID;
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
@@ -59,6 +68,9 @@ int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t le
 }
 
 int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
+  if (pSBf->status == SBF_INVALID) {
+    return TSDB_CODE_FAILED;
+  }
   uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len);
   uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len);
   int32_t size = taosArrayGetSize(pSBf->bfArray);
@@ -74,6 +86,7 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
     pNormalBf = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth,
                                      pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO);
     if (pNormalBf == NULL) {
+      pSBf->status = SBF_INVALID;
       return TSDB_CODE_OUT_OF_MEMORY;
     }
   }
@@ -93,6 +106,10 @@ int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32
 }
 
 static SBloomFilter *tScalableBfAddFilter(SScalableBf *pSBf, uint64_t expectedEntries, double errorRate) {
+  if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) {
+    return NULL;
+  }
+
   SBloomFilter *pNormalBf = tBloomFilterInit(expectedEntries, errorRate);
   if (pNormalBf == NULL) {
     return NULL;
@@ -128,6 +145,8 @@ int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder) {
   }
   if (tEncodeU32(pEncoder, pSBf->growth) < 0) return -1;
   if (tEncodeU64(pEncoder, pSBf->numBits) < 0) return -1;
+  if (tEncodeU32(pEncoder, pSBf->maxBloomFilters) < 0) return -1;
+  if (tEncodeI8(pEncoder, pSBf->status) < 0) return -1;
   return 0;
 }
 
@@ -150,6 +169,8 @@ SScalableBf *tScalableBfDecode(SDecoder *pDecoder) {
   }
   if (tDecodeU32(pDecoder, &pSBf->growth) < 0) goto _error;
   if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) goto _error;
+  if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) goto _error;
+  if (tDecodeI8(pDecoder, &pSBf->status) < 0) goto _error;
   return pSBf;
 
 _error:

From 4c4733209828aba4b1d091048fcc3de6f7ff0263 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 29 Feb 2024 09:24:39 +0800
Subject: [PATCH 097/201] opt bloom filter

---
 source/libs/stream/src/streamUpdate.c | 2 +-
 source/util/src/tscalablebf.c         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c
index 34ae909f03..764bf6e026 100644
--- a/source/libs/stream/src/streamUpdate.c
+++ b/source/libs/stream/src/streamUpdate.c
@@ -22,7 +22,7 @@
 #define DEFAULT_MAP_CAPACITY     131072
 #define DEFAULT_MAP_SIZE         (DEFAULT_MAP_CAPACITY * 100)
 #define ROWS_PER_MILLISECOND     1
-#define MAX_NUM_SCALABLE_BF      32
+#define MAX_NUM_SCALABLE_BF      64
 #define MIN_NUM_SCALABLE_BF      10
 #define DEFAULT_PREADD_BUCKET    1
 #define MAX_INTERVAL             MILLISECOND_PER_MINUTE
diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c
index a6e5b563d8..5ffa93c6f9 100644
--- a/source/util/src/tscalablebf.c
+++ b/source/util/src/tscalablebf.c
@@ -20,7 +20,7 @@
 
 #define DEFAULT_GROWTH           2
 #define DEFAULT_TIGHTENING_RATIO 0.5
-#define DEFAULT_MAX_BLOOMFILTERS 5
+#define DEFAULT_MAX_BLOOMFILTERS 4
 #define SBF_INVALID              -1
 #define SBF_VALID                0
 

From 9d4bf1edba56e803b79e2c4da689b72b94859922 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 29 Feb 2024 09:35:51 +0800
Subject: [PATCH 098/201] opt bloom filter

---
 source/util/src/tscalablebf.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c
index 5ffa93c6f9..743858eee9 100644
--- a/source/util/src/tscalablebf.c
+++ b/source/util/src/tscalablebf.c
@@ -94,6 +94,9 @@ int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
 }
 
 int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len) {
+  if (pSBf->status == SBF_INVALID) {
+    return TSDB_CODE_FAILED;
+  }
   uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len);
   uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len);
   int32_t size = taosArrayGetSize(pSBf->bfArray);

From 00812c8a5c68a3d92651456290699156f9a851d0 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 29 Feb 2024 11:16:54 +0800
Subject: [PATCH 099/201] opt bloom filter

---
 source/util/src/tscalablebf.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/util/src/tscalablebf.c b/source/util/src/tscalablebf.c
index 743858eee9..7af794546b 100644
--- a/source/util/src/tscalablebf.c
+++ b/source/util/src/tscalablebf.c
@@ -35,6 +35,8 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) {
   if (pSBf == NULL) {
     return NULL;
   }
+  pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS;
+  pSBf->status = SBF_VALID;
   pSBf->numBits = 0;
   pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void *));
   if (tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO) == NULL) {
@@ -44,8 +46,6 @@ SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate) {
   pSBf->growth = DEFAULT_GROWTH;
   pSBf->hashFn1 = HASH_FUNCTION_1;
   pSBf->hashFn2 = HASH_FUNCTION_2;
-  pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS;
-  pSBf->status = SBF_VALID;
   return pSBf;
 }
 

From 02379524a1e2878aa48af2caa9b8881e78059b94 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 29 Feb 2024 10:59:45 +0800
Subject: [PATCH 100/201] fix count window issue

---
 include/common/tglobal.h                         | 1 -
 source/common/src/tglobal.c                      | 8 +-------
 source/libs/executor/src/countwindowoperator.c   | 2 +-
 source/libs/planner/src/planLogicCreater.c       | 4 ----
 tests/script/tsim/query/query_count1.sim         | 2 --
 tests/script/tsim/query/query_count_sliding0.sim | 2 --
 6 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index ef18d1fefb..93f17fa887 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -219,7 +219,6 @@ extern bool    tsFilterScalarMode;
 extern int32_t tsMaxStreamBackendCache;
 extern int32_t tsPQSortMemThreshold;
 extern int32_t tsResolveFQDNRetryTime;
-extern bool    tsDisableCount;
 
 extern bool tsExperimental;
 // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 5c51d83e4d..c5a26c5c10 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -269,7 +269,6 @@ int64_t tsStreamBufferSize = 128 * 1024 * 1024;
 bool    tsFilterScalarMode = false;
 int     tsResolveFQDNRetryTime = 100;  // seconds
 int     tsStreamAggCnt = 1000;
-bool    tsDisableCount = true;
 
 char   tsS3Endpoint[TSDB_FQDN_LEN] = "<endpoint>";
 char   tsS3AccessKey[TSDB_FQDN_LEN] = "<accesskey>";
@@ -541,8 +540,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
 
   if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-
-  if (cfgAddBool(pCfg, "disableCount", tsDisableCount, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
   return 0;
 }
 
@@ -1109,8 +1106,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
 
   tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
-
-  tsDisableCount = cfgGetItem(pCfg, "disableCount")->bval;
   return 0;
 }
 
@@ -1739,8 +1734,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
                                          {"shellActivityTimer", &tsShellActivityTimer},
                                          {"slowLogThreshold", &tsSlowLogThreshold},
                                          {"useAdapter", &tsUseAdapter},
-                                         {"experimental", &tsExperimental},
-                                         {"disableCount", &tsDisableCount}};
+                                         {"experimental", &tsExperimental}};
 
     if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
       taosCfgSetOption(options, tListLen(options), pItem, false);
diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c
index 1d2a55fac8..0838a38312 100644
--- a/source/libs/executor/src/countwindowoperator.c
+++ b/source/libs/executor/src/countwindowoperator.c
@@ -94,10 +94,10 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
   int32_t                   code = TSDB_CODE_SUCCESS;
 
   for (int32_t i = 0; i < pBlock->info.rows;) {
-    int32_t             step = pInfo->windowSliding;
     SCountWindowResult* pBuffInfo = setCountWindowOutputBuff(pExprSup, &pInfo->countSup, &pInfo->pRow);
     int32_t prevRows = pBuffInfo->winRows;
     int32_t num = updateCountWindowInfo(i, pBlock->info.rows, pInfo->windowCount, &pBuffInfo->winRows);
+    int32_t step = num;
     if (prevRows == 0) {
       pInfo->pRow->win.skey = tsCols[i];
     }
diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c
index 0f5ddf926d..37cdc31ae0 100644
--- a/source/libs/planner/src/planLogicCreater.c
+++ b/source/libs/planner/src/planLogicCreater.c
@@ -1016,10 +1016,6 @@ static int32_t createWindowLogicNodeByCount(SLogicPlanContext* pCxt, SCountWindo
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  if (!pCxt->pPlanCxt->streamQuery && tsDisableCount) {
-    return TSDB_CODE_FAILED;
-  }
-
   pWindow->winType = WINDOW_TYPE_COUNT;
   pWindow->node.groupAction = getGroupAction(pCxt, pSelect);
   pWindow->node.requireDataOrder =
diff --git a/tests/script/tsim/query/query_count1.sim b/tests/script/tsim/query/query_count1.sim
index 0c40303e57..0694ab062a 100644
--- a/tests/script/tsim/query/query_count1.sim
+++ b/tests/script/tsim/query/query_count1.sim
@@ -9,8 +9,6 @@ print =============== create database
 sql create database test vgroups 4;
 sql use test;
 
-sql alter local 'disableCount' '0' ;
-
 sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
 sql create table t1 using st tags(1,1,1);
 sql create table t2 using st tags(2,2,2);
diff --git a/tests/script/tsim/query/query_count_sliding0.sim b/tests/script/tsim/query/query_count_sliding0.sim
index 13a6c94451..464aec6b97 100644
--- a/tests/script/tsim/query/query_count_sliding0.sim
+++ b/tests/script/tsim/query/query_count_sliding0.sim
@@ -9,8 +9,6 @@ print =============== create database
 sql create database test vgroups 1;
 sql use test;
 
-sql alter local 'disableCount' '0' ;
-
 sql create table t1(ts timestamp, a int, b int , c int, d double);
 
 sql insert into t1 values(1648791213000,0,1,1,1.0);

From 6de92f7c151df4d1155e42805e4820ddae15c30a Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 29 Feb 2024 11:24:06 +0800
Subject: [PATCH 101/201] fix count window issue

---
 tests/script/tsim/query/query_count0.sim | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/script/tsim/query/query_count0.sim b/tests/script/tsim/query/query_count0.sim
index b7c629e538..5b95d4fad7 100644
--- a/tests/script/tsim/query/query_count0.sim
+++ b/tests/script/tsim/query/query_count0.sim
@@ -9,8 +9,6 @@ print =============== create database
 sql create database test vgroups 1;
 sql use test;
 
-sql alter local 'disableCount' '0' ;
-
 sql create table t1(ts timestamp, a int, b int , c int, d double);
 
 sql insert into t1 values(1648791213000,0,1,1,1.0);

From b744eabce0cd0cc6c5f50f14c01d35888fb5f624 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 29 Feb 2024 11:34:57 +0800
Subject: [PATCH 102/201] fix(util): fix the new size remaining bug.

---
 source/util/src/tarray.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index 88eb51d500..d9686d77f8 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -89,12 +89,14 @@ static int32_t taosArrayResize(SArray* pArray) {
 int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) {
   if (newCap > pArray->capacity) {
     float factor = BOUNDARY_BIG_FACTOR;
-    if(newCap * pArray->elemSize > BOUNDARY_SIZE){
+    if (newCap * pArray->elemSize > BOUNDARY_SIZE) {
       factor = BOUNDARY_SMALL_FACTOR;
     }
+
     size_t tsize = (pArray->capacity * factor);
     while (newCap > tsize) {
-      tsize = (tsize * factor);
+      size_t newSize = (tsize * factor);
+      tsize = (newSize == tsize) ? (tsize + 2) : newSize;
     }
 
     pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize);

From b37351333f5e2a7f7903cc8d94a41dcbda2b9874 Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Thu, 29 Feb 2024 12:35:30 +0800
Subject: [PATCH 103/201] release 3.2.3.0

---
 cmake/cmake.version                | 2 +-
 docs/en/28-releases/01-tdengine.md | 4 ++++
 docs/zh/28-releases/01-tdengine.md | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/cmake/cmake.version b/cmake/cmake.version
index 0e35fa316f..4abc854e71 100644
--- a/cmake/cmake.version
+++ b/cmake/cmake.version
@@ -2,7 +2,7 @@
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "3.2.3.0.alpha")
+  SET(TD_VER_NUMBER "3.2.4.0.alpha")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)
diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md
index f5a4789976..902e62de73 100644
--- a/docs/en/28-releases/01-tdengine.md
+++ b/docs/en/28-releases/01-tdengine.md
@@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t
 
 import Release from "/components/ReleaseV3";
 
+## 3.2.3.0
+
+<Release type="tdengine" version="3.2.3.0" />
+
 ## 3.2.2.0
 
 <Release type="tdengine" version="3.2.2.0" />
diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md
index b0a81e01a1..1c51f934fe 100644
--- a/docs/zh/28-releases/01-tdengine.md
+++ b/docs/zh/28-releases/01-tdengine.md
@@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
 
 import Release from "/components/ReleaseV3";
 
+## 3.2.3.0
+
+<Release type="tdengine" version="3.2.3.0" />
+
 ## 3.2.2.0
 
 <Release type="tdengine" version="3.2.2.0" />

From fa45a5abd2cdfcf8935aacd0a5efe52917d1654b Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 29 Feb 2024 14:04:59 +0800
Subject: [PATCH 104/201] fix count window issue

---
 source/libs/executor/src/countwindowoperator.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c
index 0838a38312..1f38264644 100644
--- a/source/libs/executor/src/countwindowoperator.c
+++ b/source/libs/executor/src/countwindowoperator.c
@@ -118,6 +118,8 @@ int32_t doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
       if (prevRows <= pInfo->windowSliding) {
         if (pBuffInfo->winRows > pInfo->windowSliding) {
           step = pInfo->windowSliding - prevRows;
+        } else {
+          step = pInfo->windowSliding;
         }
       } else {
         step = 0;

From e17832e27eb5cdb0860504111fba46d4ba28e6d7 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 29 Feb 2024 14:38:09 +0800
Subject: [PATCH 105/201] fix:[TD-28869]get error in askEp because consumer is
 dropped when unsubscribe

---
 source/client/src/clientTmq.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 3ce4e97e22..11e88dc306 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -2135,16 +2135,19 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
   if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
     // if auto commit is set, commit before close consumer. Otherwise, do nothing.
     if (tmq->autoCommit) {
-      int32_t rsp = tmq_commit_sync(tmq, NULL);
-      if (rsp != 0) {
-        return rsp;
+      int32_t code = tmq_commit_sync(tmq, NULL);
+      if (code != 0) {
+        return code;
       }
     }
     taosSsleep(2);  // sleep 2s for hb to send offset and rows to server
 
     tmq_list_t* lst = tmq_list_new();
-    tmq_subscribe(tmq, lst);
+    int32_t code = tmq_subscribe(tmq, lst);
     tmq_list_destroy(lst);
+    if (code != 0) {
+      return code;
+    }
   } else {
     tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId);
   }

From f4c4465c0ef06ecc70f7b451bc90e82748074fcd Mon Sep 17 00:00:00 2001
From: dmchen <cademfly@hotmail.com>
Date: Thu, 29 Feb 2024 06:54:24 +0000
Subject: [PATCH 106/201] fix/TD-28490

---
 include/util/taoserror.h                 |  2 +-
 source/common/src/tmsg.c                 | 10 +++++-----
 source/dnode/mnode/impl/src/mndCompact.c | 14 +++++++++-----
 source/util/src/terror.c                 |  1 +
 4 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 556479f547..3a689fc63d 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -433,7 +433,7 @@ int32_t* taosGetErrno();
 
 //mnode-compact
 #define TSDB_CODE_MND_INVALID_COMPACT_ID        TAOS_DEF_ERROR_CODE(0, 0x04B1)
-
+#define TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST  TAOS_DEF_ERROR_CODE(0, 0x04B2)
 
 // vnode
 // #define TSDB_CODE_VND_ACTION_IN_PROGRESS     TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 4640c20e07..a1d279b494 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -5238,11 +5238,11 @@ int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCo
 
   if (tStartDecode(&decoder) < 0) return -1;
 
-  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->finished) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -2;
+  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -3;
+  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -4;
+  if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -5;
+  if (tDecodeI32(&decoder, &pReq->finished) < 0) return -6;
 
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c
index 83254b0f73..2d714596a9 100644
--- a/source/dnode/mnode/impl/src/mndCompact.c
+++ b/source/dnode/mnode/impl/src/mndCompact.c
@@ -490,13 +490,17 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
     sdbRelease(pMnode->pSdb, pDetail);
   }
 
-  return -1;
+  return TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST;
 }
 
 int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){
   SQueryCompactProgressRsp req = {0};
-  if (tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req) != 0) {
+  int32_t code = 0;
+  code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req);
+  if (code != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
+    mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", 
+          code, pReq->pCont, pReq->contLen);
     return -1;
   }
 
@@ -504,10 +508,10 @@ int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq){
         req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
 
   SMnode        *pMnode = pReq->info.node;
-  int32_t       code = -1;
-  
 
-  if(mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req) != 0){
+  code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req);
+  if(code != 0){
+    terrno = code;
     mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", 
         req.compactId, req.vgId, req.dnodeId, req.numberFileset, req.finished);
     return -1;
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 836fd980d0..1f3aaa3835 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -329,6 +329,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VIEW_NOT_EXIST,           "view not exists in db
 
 //mnode-compact
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COMPACT_ID,       "Invalid compact id")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST, "compact detail doesn't exist")
 
 // dnode
 TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_OFFLINE,                "Dnode is offline")

From ae660f95ed01007a4c1614a393493038f0c4afb1 Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Thu, 29 Feb 2024 15:02:55 +0800
Subject: [PATCH 107/201] fix: get firstNRows Length

---
 source/common/src/tdatablock.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index a20ecd344d..f2cd9313e1 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1690,17 +1690,20 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) {
 static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
   if (n >= total || n == 0) return;
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
-    int32_t newLen = -1;
-    for (int i = n; i >= 0; --i) {
-      if (pColInfoData->varmeta.offset[i] != -1) {
+    int32_t newLen = pColInfoData->varmeta.offset[n];
+    if (-1 == newLen) {
+      for (int i = n - 1; i >= 0; --i) {
         newLen = pColInfoData->varmeta.offset[i];
-        break;
+        if (newLen != -1) {
+          if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
+            newLen += getJsonValueLen(pColInfoData->pData + newLen);
+          } else {
+            newLen += varDataTLen(pColInfoData->pData + newLen);
+          }
+          break;
+        }
       }
     }
-    if(-1 == newLen) {
-      return;
-    }
-
     pColInfoData->varmeta.length = newLen;
     // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
     memset(&pColInfoData->varmeta.offset[n], 0, total - n);

From 4e67da70175cc9fc9d766cd877124507d91db5dc Mon Sep 17 00:00:00 2001
From: dmchen <cademfly@hotmail.com>
Date: Thu, 29 Feb 2024 07:24:29 +0000
Subject: [PATCH 108/201] fix/TD-28920

---
 include/util/tdef.h                      | 2 ++
 source/dnode/mnode/impl/src/mndCompact.c | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/include/util/tdef.h b/include/util/tdef.h
index 8371ee24bb..e2d1beb5a5 100644
--- a/include/util/tdef.h
+++ b/include/util/tdef.h
@@ -187,6 +187,8 @@ typedef enum ELogicConditionType {
   LOGIC_COND_TYPE_NOT,
 } ELogicConditionType;
 
+#define TSDB_INT32_ID_LEN 11
+
 #define TSDB_NAME_DELIMITER_LEN 1
 
 #define TSDB_UNI_LEN  24
diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c
index 83254b0f73..2de968f90a 100644
--- a/source/dnode/mnode/impl/src/mndCompact.c
+++ b/source/dnode/mnode/impl/src/mndCompact.c
@@ -454,7 +454,7 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){
 
   code = TSDB_CODE_ACTION_IN_PROGRESS;
 
-  char obj[MND_COMPACT_ID_LEN] = {0};
+  char obj[TSDB_INT32_ID_LEN] = {0};
   sprintf(obj, "%d", pCompact->compactId);
 
   auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen);

From 1fe35031e089dac1ac1a500029b21b8d6bc2794a Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Thu, 29 Feb 2024 16:32:58 +0800
Subject: [PATCH 109/201] fix: get firstNRows Length

---
 source/common/src/tdatablock.c | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index f2cd9313e1..c996bc2e4a 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1690,21 +1690,27 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) {
 static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
   if (n >= total || n == 0) return;
   if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
-    int32_t newLen = pColInfoData->varmeta.offset[n];
-    if (-1 == newLen) {
-      for (int i = n - 1; i >= 0; --i) {
-        newLen = pColInfoData->varmeta.offset[i];
-        if (newLen != -1) {
-          if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
-            newLen += getJsonValueLen(pColInfoData->pData + newLen);
-          } else {
-            newLen += varDataTLen(pColInfoData->pData + newLen);
+    if (pColInfoData->varmeta.length != 0) {
+      int32_t newLen = pColInfoData->varmeta.offset[n];
+      if (-1 == newLen) {
+        for (int i = n - 1; i >= 0; --i) {
+          newLen = pColInfoData->varmeta.offset[i];
+          if (newLen != -1) {
+            if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
+              newLen += getJsonValueLen(pColInfoData->pData + newLen);
+            } else {
+              newLen += varDataTLen(pColInfoData->pData + newLen);
+            }
+            break;
           }
-          break;
         }
       }
+      if (newLen <= -1) {
+        uFatal("colDataKeepFirstNRows: newLen:%d  old:%d", newLen, pColInfoData->varmeta.length);
+      } else {
+        pColInfoData->varmeta.length = newLen;
+      }
     }
-    pColInfoData->varmeta.length = newLen;
     // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
     memset(&pColInfoData->varmeta.offset[n], 0, total - n);
   }

From 474ec01cdb2018cd85d7aed93f3f0e1b790cb1fb Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Thu, 29 Feb 2024 18:27:34 +0800
Subject: [PATCH 110/201] delete unused code

---
 include/common/tmsg.h    |  14 +--
 source/common/src/tmsg.c | 227 ++++++++++++++++++++-------------------
 2 files changed, 121 insertions(+), 120 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 36242ddea8..958789178a 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -581,8 +581,8 @@ typedef struct {
   };
 } SSubmitRsp;
 
-int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
-int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
+// int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
+// int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
 // void    tFreeSSubmitBlkRsp(void* param);
 void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
@@ -885,8 +885,8 @@ typedef struct {
   int64_t maxStorage;
 } SCreateAcctReq, SAlterAcctReq;
 
-int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
-int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
+// int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
+// int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
 
 typedef struct {
   char    user[TSDB_USER_LEN];
@@ -3446,7 +3446,7 @@ int32_t tDeserializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexR
 
 typedef SMDropSmaReq SDropTagIndexReq;
 
-int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
+// int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
 int32_t tDeserializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
 
 typedef struct {
@@ -3567,8 +3567,8 @@ typedef struct {
   int8_t igNotExists;
 } SMDropFullTextReq;
 
-int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
-int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
+// int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
+// int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
 
 typedef struct {
   char indexFName[TSDB_INDEX_FNAME_LEN];
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 4640c20e07..5eb62c6389 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -1009,19 +1009,19 @@ int32_t tDeserializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexR
   tDecoderClear(&decoder);
   return 0;
 }
-int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
-  if (tStartEncode(&encoder) < 0) return -1;
-  tEndEncode(&encoder);
+// int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
+//   SEncoder encoder = {0};
+//   tEncoderInit(&encoder, buf, bufLen);
+//   if (tStartEncode(&encoder) < 0) return -1;
+//   tEndEncode(&encoder);
 
-  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
 
-  int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
-  return tlen;
-}
+//   int32_t tlen = encoder.pos;
+//   tEncoderClear(&encoder);
+//   return tlen;
+// }
 int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
@@ -1035,6 +1035,7 @@ int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *
 
   return 0;
 }
+
 int32_t tSerializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -1059,32 +1060,32 @@ void tFreeSMCreateFullTextReq(SMCreateFullTextReq *pReq) {
   // impl later
   return;
 }
-int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+// int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
+//   SEncoder encoder = {0};
+//   tEncoderInit(&encoder, buf, bufLen);
 
-  if (tStartEncode(&encoder) < 0) return -1;
+//   if (tStartEncode(&encoder) < 0) return -1;
 
-  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
 
-  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
 
-  tEndEncode(&encoder);
-  int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
-  return tlen;
-}
-int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+//   tEndEncode(&encoder);
+//   int32_t tlen = encoder.pos;
+//   tEncoderClear(&encoder);
+//   return tlen;
+// }
+// int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+//   if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
 
-  tEndDecode(&decoder);
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tEndDecode(&decoder);
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) {
   SEncoder encoder = {0};
@@ -1474,44 +1475,44 @@ void tFreeSStatisReq(SStatisReq *pReq) {
   taosMemoryFreeClear(pReq->pCont);
 }
 
-int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+// int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
+//   SEncoder encoder = {0};
+//   tEncoderInit(&encoder, buf, bufLen);
 
-  if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1;
-  if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1;
-  tEndEncode(&encoder);
+//   if (tStartEncode(&encoder) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1;
+//   if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1;
+//   tEndEncode(&encoder);
 
-  int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
-  return tlen;
-}
+//   int32_t tlen = encoder.pos;
+//   tEncoderClear(&encoder);
+//   return tlen;
+// }
 
-int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+// int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
+//   SDecoder decoder = {0};
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1;
-  if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1;
-  tEndDecode(&decoder);
+//   if (tStartDecode(&decoder) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1;
+//   if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1;
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
-  return 0;
-}
+//   tDecoderClear(&decoder);
+//   return 0;
+// }
 
 int32_t tSerializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) {
   SEncoder encoder = {0};
@@ -7934,64 +7935,64 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl
   return 0;
 }
 
-static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
-  if (tStartDecode(pDecoder) < 0) return -1;
+// static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
+//   if (tStartDecode(pDecoder) < 0) return -1;
 
-  if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
-  if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
-  pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
-  if (NULL == pBlock->tblFName) return -1;
-  if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1;
-  if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1;
+//   if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
+//   if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
+//   pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
+//   if (NULL == pBlock->tblFName) return -1;
+//   if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1;
+//   if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1;
 
-  int32_t meta = 0;
-  if (tDecodeI32(pDecoder, &meta) < 0) return -1;
-  if (meta) {
-    pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
-    if (NULL == pBlock->pMeta) return -1;
-    if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1;
-  } else {
-    pBlock->pMeta = NULL;
-  }
+//   int32_t meta = 0;
+//   if (tDecodeI32(pDecoder, &meta) < 0) return -1;
+//   if (meta) {
+//     pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+//     if (NULL == pBlock->pMeta) return -1;
+//     if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1;
+//   } else {
+//     pBlock->pMeta = NULL;
+//   }
 
-  tEndDecode(pDecoder);
-  return 0;
-}
+//   tEndDecode(pDecoder);
+//   return 0;
+// }
 
-int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) {
-  int32_t nBlocks = taosArrayGetSize(pRsp->pArray);
+// int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) {
+//   int32_t nBlocks = taosArrayGetSize(pRsp->pArray);
 
-  if (tStartEncode(pEncoder) < 0) return -1;
+//   if (tStartEncode(pEncoder) < 0) return -1;
 
-  if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1;
-  if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1;
-  if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1;
-  for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) {
-    if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1;
-  }
+//   if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1;
+//   if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1;
+//   if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1;
+//   for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) {
+//     if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1;
+//   }
 
-  tEndEncode(pEncoder);
-  return 0;
-}
+//   tEndEncode(pEncoder);
+//   return 0;
+// }
 
-int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
-  if (tStartDecode(pDecoder) < 0) return -1;
+// int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
+//   if (tStartDecode(pDecoder) < 0) return -1;
 
-  if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1;
-  pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks));
-  if (pRsp->pBlocks == NULL) return -1;
-  for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) {
-    if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1;
-  }
+//   if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1;
+//   pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks));
+//   if (pRsp->pBlocks == NULL) return -1;
+//   for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) {
+//     if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1;
+//   }
 
-  tEndDecode(pDecoder);
-  tDecoderClear(pDecoder);
-  return 0;
-}
+//   tEndDecode(pDecoder);
+//   tDecoderClear(pDecoder);
+//   return 0;
+// }
 
 // void tFreeSSubmitBlkRsp(void *param) {
 //   if (NULL == param) {

From a0c77f5baafeeb44ea9588da2577c35368b2b6ad Mon Sep 17 00:00:00 2001
From: yihaoDeng <yhdeng@taosdata.com>
Date: Thu, 29 Feb 2024 18:40:21 +0800
Subject: [PATCH 111/201] fix mem leak while taosd quit

---
 source/dnode/mgmt/node_mgmt/src/dmTransport.c |  9 +++---
 source/libs/transport/inc/transComm.h         | 30 +++++++++----------
 source/libs/transport/src/transCli.c          | 18 +++++++++--
 source/libs/transport/src/transSvr.c          |  7 +++--
 4 files changed, 40 insertions(+), 24 deletions(-)

diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index 1a31f08801..c54da860dd 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -87,9 +87,9 @@ static bool dmIsForbiddenIp(int8_t forbidden, char *user, uint32_t clientIp) {
   }
 }
 static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
-  SDnodeTrans  *pTrans = &pDnode->trans;
+  SDnodeTrans * pTrans = &pDnode->trans;
   int32_t       code = -1;
-  SRpcMsg      *pMsg = NULL;
+  SRpcMsg *     pMsg = NULL;
   SMgmtWrapper *pWrapper = NULL;
   SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)];
 
@@ -256,11 +256,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
 
   for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
-    SArray       *pArray = (*pWrapper->func.getHandlesFp)();
+    SArray *      pArray = (*pWrapper->func.getHandlesFp)();
     if (pArray == NULL) return -1;
 
     for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
-      SMgmtHandle  *pMgmt = taosArrayGet(pArray, i);
+      SMgmtHandle * pMgmt = taosArrayGet(pArray, i);
       SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
       if (pMgmt->needCheckVgId) {
         pHandle->needCheckVgId = pMgmt->needCheckVgId;
@@ -345,6 +345,7 @@ int32_t dmInitClient(SDnode *pDnode) {
   rpcInit.parent = pDnode;
   rpcInit.rfp = rpcRfp;
   rpcInit.compressSize = tsCompressMsgSize;
+  rpcInit.dfp = destroyAhandle;
 
   rpcInit.retryMinInterval = tsRedirectPeriod;
   rpcInit.retryStepFactor = tsRedirectFactor;
diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h
index 5b18d56d70..c010e31320 100644
--- a/source/libs/transport/inc/transComm.h
+++ b/source/libs/transport/inc/transComm.h
@@ -256,21 +256,21 @@ void        transAsyncPoolDestroy(SAsyncPool* pool);
 int         transAsyncSend(SAsyncPool* pool, queue* mq);
 bool        transAsyncPoolIsEmpty(SAsyncPool* pool);
 
-#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc) \
-  do {                                                        \
-    for (int i = 0; i < pool->nAsync; i++) {                  \
-      uv_async_t* async = &(pool->asyncs[i]);                 \
-      SAsyncItem* item = async->data;                         \
-      while (!QUEUE_IS_EMPTY(&item->qmsg)) {                  \
-        tTrace("destroy msg in async pool ");                 \
-        queue* h = QUEUE_HEAD(&item->qmsg);                   \
-        QUEUE_REMOVE(h);                                      \
-        msgType* msg = QUEUE_DATA(h, msgType, q);             \
-        if (msg != NULL) {                                    \
-          freeFunc(msg);                                      \
-        }                                                     \
-      }                                                       \
-    }                                                         \
+#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc, param) \
+  do {                                                               \
+    for (int i = 0; i < pool->nAsync; i++) {                         \
+      uv_async_t* async = &(pool->asyncs[i]);                        \
+      SAsyncItem* item = async->data;                                \
+      while (!QUEUE_IS_EMPTY(&item->qmsg)) {                         \
+        tTrace("destroy msg in async pool ");                        \
+        queue* h = QUEUE_HEAD(&item->qmsg);                          \
+        QUEUE_REMOVE(h);                                             \
+        msgType* msg = QUEUE_DATA(h, msgType, q);                    \
+        if (msg != NULL) {                                           \
+          freeFunc(msg, param);                                      \
+        }                                                            \
+      }                                                              \
+    }                                                                \
   } while (0)
 
 #define ASYNC_CHECK_HANDLE(exh1, id)                                                                         \
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index e937d2e65e..e369bf8f7d 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -219,6 +219,8 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq,
 /// NULL,cliHandleUpdate};
 
 static FORCE_INLINE void destroyCmsg(void* cmsg);
+
+static FORCE_INLINE void destroyQueuedMsg(void* arg, void* param);
 static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg);
 static FORCE_INLINE int  cliRBChoseIdx(STrans* pTransInst);
 static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx);
@@ -1963,7 +1965,19 @@ static FORCE_INLINE void destroyCmsg(void* arg) {
   transFreeMsg(pMsg->msg.pCont);
   taosMemoryFree(pMsg);
 }
-
+static FORCE_INLINE void destroyQueuedMsg(void* arg, void* param) {
+  SCliMsg* pMsg = arg;
+  if (pMsg == NULL) {
+    return;
+  }
+  if (param != NULL) {
+    SCliThrd* pThrd = param;
+    if (pThrd->destroyAhandleFp) (*pThrd->destroyAhandleFp)(pMsg->msg.info.ahandle);
+  }
+  transDestroyConnCtx(pMsg->ctx);
+  transFreeMsg(pMsg->msg.pCont);
+  taosMemoryFree(pMsg);
+}
 static FORCE_INLINE void destroyCmsgAndAhandle(void* param) {
   if (param == NULL) return;
 
@@ -2057,7 +2071,7 @@ static void destroyThrdObj(SCliThrd* pThrd) {
   taosThreadJoin(pThrd->thread, NULL);
   CLI_RELEASE_UV(pThrd->loop);
   taosThreadMutexDestroy(&pThrd->msgMtx);
-  TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsg);
+  TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyQueuedMsg, (void*)pThrd);
   transAsyncPoolDestroy(pThrd->asyncPool);
 
   transDQDestroy(pThrd->delayQueue, destroyCmsgAndAhandle);
diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index b324ca5f91..5a1ef31b7d 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -159,7 +159,7 @@ static void uvStartSendResp(SSvrMsg* msg);
 
 static void uvNotifyLinkBrokenToApp(SSvrConn* conn);
 
-static FORCE_INLINE void      destroySmsg(SSvrMsg* smsg);
+static FORCE_INLINE void destroySmsg(SSvrMsg* smsg);
 static FORCE_INLINE SSvrConn* createConn(void* hThrd);
 static FORCE_INLINE void      destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
 static FORCE_INLINE void      destroyConnRegArg(SSvrConn* conn);
@@ -671,7 +671,8 @@ static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) {
   transFreeMsg(smsg->msg.pCont);
   taosMemoryFree(smsg);
 }
-static void destroyAllConn(SWorkThrd* pThrd) {
+static FORCE_INLINE void destroySmsgWrapper(void* smsg, void* param) { destroySmsg((SSvrMsg*)smsg); }
+static void              destroyAllConn(SWorkThrd* pThrd) {
   tTrace("thread %p destroy all conn ", pThrd);
   while (!QUEUE_IS_EMPTY(&pThrd->conn)) {
     queue* h = QUEUE_HEAD(&pThrd->conn);
@@ -1394,7 +1395,7 @@ void destroyWorkThrd(SWorkThrd* pThrd) {
   }
   taosThreadJoin(pThrd->thread, NULL);
   SRV_RELEASE_UV(pThrd->loop);
-  TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsg);
+  TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL);
   transAsyncPoolDestroy(pThrd->asyncPool);
 
   uvWhiteListDestroy(pThrd->pWhiteList);

From c3220aec44583595a5c2aae8c55675849c2bb79a Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 29 Feb 2024 10:46:09 +0000
Subject: [PATCH 112/201] fix mem  leak while taosd quit

---
 source/libs/transport/src/thttp.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c
index 6de10cbb9e..e76f073d63 100644
--- a/source/libs/transport/src/thttp.c
+++ b/source/libs/transport/src/thttp.c
@@ -183,7 +183,8 @@ static void* httpThread(void* arg) {
   return NULL;
 }
 
-static void httpDestroyMsg(SHttpMsg* msg) {
+static void httpDestroyMsg(void* cont, void* param) {
+  SHttpMsg* msg = cont;
   if (msg == NULL) return;
 
   taosMemoryFree(msg->server);
@@ -554,7 +555,7 @@ void transHttpEnvDestroy() {
   httpSendQuit();
   taosThreadJoin(load->thread, NULL);
 
-  TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg);
+  TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg, NULL);
   transAsyncPoolDestroy(load->asyncPool);
   uv_loop_close(load->loop);
   taosMemoryFree(load->loop);

From 881ffacb4f6050ec04fdedca85de2e86a692e9f5 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 29 Feb 2024 10:52:50 +0000
Subject: [PATCH 113/201] fix mem  leak while taosd quit

---
 source/libs/transport/src/thttp.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c
index e76f073d63..c4ca39c323 100644
--- a/source/libs/transport/src/thttp.c
+++ b/source/libs/transport/src/thttp.c
@@ -183,8 +183,7 @@ static void* httpThread(void* arg) {
   return NULL;
 }
 
-static void httpDestroyMsg(void* cont, void* param) {
-  SHttpMsg* msg = cont;
+static void httpDestroyMsg(SHttpMsg* msg) {
   if (msg == NULL) return;
 
   taosMemoryFree(msg->server);
@@ -192,6 +191,15 @@ static void httpDestroyMsg(void* cont, void* param) {
   taosMemoryFree(msg->cont);
   taosMemoryFree(msg);
 }
+static void httpDestroyMsgWrapper(void* cont, void* param) {
+  httpDestroyMsg((SHttpMsg*)cont);
+  // if (msg == NULL) return;
+
+  // taosMemoryFree(msg->server);
+  // taosMemoryFree(msg->uri);
+  // taosMemoryFree(msg->cont);
+  // taosMemoryFree(msg);
+}
 
 static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) {
   SHttpMsg *msg = NULL, *quitMsg = NULL;
@@ -555,7 +563,7 @@ void transHttpEnvDestroy() {
   httpSendQuit();
   taosThreadJoin(load->thread, NULL);
 
-  TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg, NULL);
+  TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsgWrapper, NULL);
   transAsyncPoolDestroy(load->asyncPool);
   uv_loop_close(load->loop);
   taosMemoryFree(load->loop);

From e0b1b5a1c62958e3f9c9a41d497bf9c107fff3aa Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 29 Feb 2024 10:54:35 +0000
Subject: [PATCH 114/201] fix mem  leak while taosd quit

---
 source/dnode/mgmt/node_mgmt/src/dmTransport.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index c54da860dd..77760b16f4 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -87,9 +87,9 @@ static bool dmIsForbiddenIp(int8_t forbidden, char *user, uint32_t clientIp) {
   }
 }
 static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
-  SDnodeTrans * pTrans = &pDnode->trans;
+  SDnodeTrans  *pTrans = &pDnode->trans;
   int32_t       code = -1;
-  SRpcMsg *     pMsg = NULL;
+  SRpcMsg      *pMsg = NULL;
   SMgmtWrapper *pWrapper = NULL;
   SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)];
 
@@ -256,11 +256,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
 
   for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
     SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
-    SArray *      pArray = (*pWrapper->func.getHandlesFp)();
+    SArray       *pArray = (*pWrapper->func.getHandlesFp)();
     if (pArray == NULL) return -1;
 
     for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
-      SMgmtHandle * pMgmt = taosArrayGet(pArray, i);
+      SMgmtHandle  *pMgmt = taosArrayGet(pArray, i);
       SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
       if (pMgmt->needCheckVgId) {
         pHandle->needCheckVgId = pMgmt->needCheckVgId;

From 3c80f171ef4b9f5c5fb000ade2ccb3f1b00b39e4 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 29 Feb 2024 18:56:14 +0800
Subject: [PATCH 115/201] fix(tsdb/cache): new case for column changing

---
 source/dnode/vnode/src/tsdb/tsdbCache.c       |  63 ++++----
 tests/parallel_test/cases.task                |   1 +
 tests/script/tsim/parser/last_both.sim        | 150 ++++++++++++++++++
 tests/script/tsim/parser/last_cache_query.sim | 108 +++++++++++++
 4 files changed, 291 insertions(+), 31 deletions(-)
 create mode 100644 tests/script/tsim/parser/last_both.sim

diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index 1ef2a451a7..0848fd0076 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -789,25 +789,6 @@ int32_t tsdbCacheDropSTableColumn(STsdb *pTsdb, SArray *uids, int16_t cid, int8_
   return code;
 }
 
-static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) {
-  SLastCol *pLastCol = NULL;
-
-  char     *err = NULL;
-  size_t    vlen = 0;
-  SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid};
-  size_t    klen = ROCKS_KEY_LEN;
-  char     *value = NULL;
-  value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err);
-  if (NULL != err) {
-    tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err);
-    rocksdb_free(err);
-  }
-
-  pLastCol = tsdbCacheDeserialize(value);
-
-  return pLastCol;
-}
-
 typedef struct {
   int      idx;
   SLastKey key;
@@ -1052,6 +1033,25 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
 static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols,
                                int nCols, int16_t *slotIds);
 #ifdef BUILD_NO_CALL
+static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) {
+  SLastCol *pLastCol = NULL;
+
+  char     *err = NULL;
+  size_t    vlen = 0;
+  SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid};
+  size_t    klen = ROCKS_KEY_LEN;
+  char     *value = NULL;
+  value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err);
+  if (NULL != err) {
+    tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err);
+    rocksdb_free(err);
+  }
+
+  pLastCol = tsdbCacheDeserialize(value);
+
+  return pLastCol;
+}
+
 int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) {
   rocksdb_writebatch_t *wb = NULL;
   int32_t               code = 0;
@@ -1233,10 +1233,10 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
   int16_t *lastSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t));
   int16_t *lastrowColIds = taosMemoryMalloc(num_keys * sizeof(int16_t));
   int16_t *lastrowSlotIds = taosMemoryMalloc(num_keys * sizeof(int16_t));
-  SArray* lastTmpColArray = NULL;
-  SArray* lastTmpIndexArray = NULL;
-  SArray* lastrowTmpColArray = NULL;
-  SArray* lastrowTmpIndexArray = NULL;
+  SArray  *lastTmpColArray = NULL;
+  SArray  *lastTmpIndexArray = NULL;
+  SArray  *lastrowTmpColArray = NULL;
+  SArray  *lastrowTmpIndexArray = NULL;
 
   int lastIndex = 0;
   int lastrowIndex = 0;
@@ -1245,7 +1245,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
     SIdxKey *idxKey = taosArrayGet(remainCols, i);
     slotIds[i] = pr->pSlotIds[idxKey->idx];
     if (idxKey->key.ltype == CACHESCAN_RETRIEVE_LAST >> 3) {
-      if(NULL == lastTmpIndexArray) {
+      if (NULL == lastTmpIndexArray) {
         lastTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t));
       }
       taosArrayPush(lastTmpIndexArray, &(i));
@@ -1253,7 +1253,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
       lastSlotIds[lastIndex] = pr->pSlotIds[idxKey->idx];
       lastIndex++;
     } else {
-      if(NULL == lastrowTmpIndexArray) {
+      if (NULL == lastrowTmpIndexArray) {
         lastrowTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t));
       }
       taosArrayPush(lastrowTmpIndexArray, &(i));
@@ -1265,17 +1265,18 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
 
   pTmpColArray = taosArrayInit(lastIndex + lastrowIndex, sizeof(SLastCol));
 
-  if(lastTmpIndexArray != NULL) {
+  if (lastTmpIndexArray != NULL) {
     mergeLastCid(uid, pTsdb, &lastTmpColArray, pr, lastColIds, lastIndex, lastSlotIds);
-    for(int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) {
-      taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i));
+    for (int i = 0; i < taosArrayGetSize(lastTmpColArray); i++) {
+      taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastTmpIndexArray, i), taosArrayGet(lastTmpColArray, i));
     }
   }
 
-  if(lastrowTmpIndexArray != NULL) {
+  if (lastrowTmpIndexArray != NULL) {
     mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds);
-    for(int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) {
-      taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastrowTmpIndexArray, i), taosArrayGet(lastrowTmpColArray, i));
+    for (int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) {
+      taosArrayInsert(pTmpColArray, *(int32_t *)taosArrayGet(lastrowTmpIndexArray, i),
+                      taosArrayGet(lastrowTmpColArray, i));
     }
   }
 
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index bdccf33c32..45c8f67da9 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -1085,6 +1085,7 @@
 ,,y,script,./test.sh -f tsim/parser/join_multivnode.sim
 ,,y,script,./test.sh -f tsim/parser/join.sim
 ,,y,script,./test.sh -f tsim/parser/last_cache.sim
+,,y,script,./test.sh -f tsim/parser/last_both.sim
 ,,y,script,./test.sh -f tsim/parser/last_groupby.sim
 ,,y,script,./test.sh -f tsim/parser/lastrow.sim
 ,,y,script,./test.sh -f tsim/parser/lastrow2.sim
diff --git a/tests/script/tsim/parser/last_both.sim b/tests/script/tsim/parser/last_both.sim
new file mode 100644
index 0000000000..d7daf4d333
--- /dev/null
+++ b/tests/script/tsim/parser/last_both.sim
@@ -0,0 +1,150 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+print ======================== dnode1 start
+$db = testdb
+sql drop database if exists $db
+sql create database $db cachemodel 'both' minrows 10 stt_trigger 1 
+sql use $db
+
+sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int)
+sql create table tb1 using st2 tags (1);
+sql create table tb2 using st2 tags (2);
+sql create table tb3 using st2 tags (3);
+sql create table tb4 using st2 tags (4);
+sql create table tb5 using st2 tags (1);
+sql create table tb6 using st2 tags (2);
+sql create table tb7 using st2 tags (3);
+sql create table tb8 using st2 tags (4);
+sql create table tb9 using st2 tags (5);
+sql create table tba using st2 tags (5);
+sql create table tbb using st2 tags (5);
+sql create table tbc using st2 tags (5);
+sql create table tbd using st2 tags (5);
+sql create table tbe using st2 tags (5);
+sql create table tbf using st2 tags (5);
+
+sql insert into tb9 values ("2021-05-09 10:12:26.000",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.001",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.002",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.003",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.004",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.005",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.006",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.007",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.008",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.009",28,  29, '30', -1005)
+sql delete from tb9 where ts = "2021-05-09 10:12:26.000"
+sql flush database $db
+
+sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3',  -1000)
+sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000)
+sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000)
+
+sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3',  -1001)
+sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001)
+sql insert into tb2 values ("2021-05-11 10:11:15",-6,  -7, '-8', -3001)
+
+sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002)
+sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002)
+sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002)
+
+sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003)
+sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003)
+sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003)
+
+sql insert into tb5 values ("2021-05-09 10:12:22",19,  20, '21', -1004)
+sql insert into tb6 values ("2021-05-11 10:12:23",22,  23, NULL, -2004)
+sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004)
+sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004)
+
+sql insert into tba values ("2021-05-10 10:12:27",31,  32, NULL, -2005)
+sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005)
+sql insert into tbc values ("2021-05-11 10:12:29",36,  37, NULL, -4005)
+sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL )
+
+sql drop table tbf;
+sql alter table st2 add column c1 int;
+sql alter table st2 drop column c1;
+
+run tsim/parser/last_cache_query.sim
+
+sql flush database $db
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+
+run tsim/parser/last_cache_query.sim
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+
+sql drop database if exists $db
+sql create database $db minrows 10 stt_trigger 1 
+sql use $db
+
+sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int)
+sql create table tb1 using st2 tags (1);
+sql create table tb2 using st2 tags (2);
+sql create table tb3 using st2 tags (3);
+sql create table tb4 using st2 tags (4);
+sql create table tb5 using st2 tags (1);
+sql create table tb6 using st2 tags (2);
+sql create table tb7 using st2 tags (3);
+sql create table tb8 using st2 tags (4);
+sql create table tb9 using st2 tags (5);
+sql create table tba using st2 tags (5);
+sql create table tbb using st2 tags (5);
+sql create table tbc using st2 tags (5);
+sql create table tbd using st2 tags (5);
+sql create table tbe using st2 tags (5);
+sql create table tbf using st2 tags (5);
+
+sql insert into tb9 values ("2021-05-09 10:12:26.000",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.001",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.002",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.003",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.004",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.005",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.006",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.007",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.008",28,  29, '30', -1005)
+sql insert into tb9 values ("2021-05-09 10:12:26.009",28,  29, '30', -1005)
+sql delete from tb9 where ts = "2021-05-09 10:12:26.000"
+sql flush database $db
+
+sql insert into tb1 values ("2021-05-09 10:10:10", 1, 2.0, '3',  -1000)
+sql insert into tb1 values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000)
+sql insert into tb1 values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000)
+
+sql insert into tb2 values ("2021-05-09 10:11:13",-1,-2.0,'-3',  -1001)
+sql insert into tb2 values ("2021-05-10 10:11:14",-4,-5.0, NULL, -2001)
+sql insert into tb2 values ("2021-05-11 10:11:15",-6,  -7, '-8', -3001)
+
+sql insert into tb3 values ("2021-05-09 10:12:17", 7, 8.0, '9' , -1002)
+sql insert into tb3 values ("2021-05-09 10:12:17",10,11.0, NULL, -2002)
+sql insert into tb3 values ("2021-05-09 10:12:18",12,NULL, NULL, -3002)
+
+sql insert into tb4 values ("2021-05-09 10:12:19",13,14.0,'15' , -1003)
+sql insert into tb4 values ("2021-05-10 10:12:20",16,17.0, NULL, -2003)
+sql insert into tb4 values ("2021-05-11 10:12:21",18,NULL, NULL, -3003)
+
+sql insert into tb5 values ("2021-05-09 10:12:22",19,  20, '21', -1004)
+sql insert into tb6 values ("2021-05-11 10:12:23",22,  23, NULL, -2004)
+sql insert into tb7 values ("2021-05-10 10:12:24",24,NULL, '25', -3004)
+sql insert into tb8 values ("2021-05-11 10:12:25",26,NULL, '27', -4004)
+
+sql insert into tba values ("2021-05-10 10:12:27",31,  32, NULL, -2005)
+sql insert into tbb values ("2021-05-10 10:12:28",33,NULL, '35', -3005)
+sql insert into tbc values ("2021-05-11 10:12:29",36,  37, NULL, -4005)
+sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL )
+
+sql drop table tbf
+sql alter database $db cachemodel 'both'
+sql alter database $db cachesize 2
+sleep 11000
+
+run tsim/parser/last_cache_query.sim
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim
index 6cd5309590..c5961f2183 100644
--- a/tests/script/tsim/parser/last_cache_query.sim
+++ b/tests/script/tsim/parser/last_cache_query.sim
@@ -357,6 +357,112 @@ if $data45 != 5 then
   return -1
 endi
 
+sql select last_row(*), id from st2 group by id order by id
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49
+
+if $rows != 5 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != NULL then
+  print $data02
+  return -1
+endi
+if $data03 != NULL then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+if $data05 != 1 then
+  return -1
+endi
+if $data10 != @21-05-11 10:12:23.000@ then
+  return -1
+endi
+if $data11 != 22 then
+  return -1
+endi
+if $data12 != 23.000000000 then
+  print $data02
+  return -1
+endi
+if $data13 != NULL then
+  return -1
+endi
+if $data14 != @70-01-01 07:59:58.-04@ then
+  return -1
+endi
+if $data15 != 2 then
+  return -1
+endi
+if $data20 != @21-05-10 10:12:24.000@ then
+  return -1
+endi
+if $data21 != 24 then
+  return -1
+endi
+if $data22 != NULL then
+  print expect NULL actual: $data22
+  return -1
+endi
+if $data23 != 25 then
+  return -1
+endi
+if $data24 != @70-01-01 07:59:57.-04@ then =
+  return -1
+endi
+if $data25 != 3 then
+  return -1
+endi
+if $data30 != @21-05-11 10:12:25.000@ then
+  return -1
+endi
+if $data31 != 26 then
+  return -1
+endi
+if $data32 != NULL then
+  print $data02
+  return -1
+endi
+if $data33 != 27 then
+  return -1
+endi
+if $data34 != @70-01-01 07:59:56.-04@ then
+  return -1
+endi
+if $data35 != 4 then
+  return -1
+endi
+if $data40 != @21-05-11 10:12:29.000@ then
+  return -1
+endi
+if $data41 != 36 then
+  return -1
+endi
+if $data42 != 37.000000000 then
+  print $data02
+  return -1
+endi
+if $data43 != NULL then
+  return -1
+endi
+if $data44 != @70-01-01 07:59:56.-05@ then
+  return -1
+endi
+if $data45 != 5 then
+  return -1
+endi
+
 print "test tbn"
 sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp)
 sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3',  -1000)
@@ -386,3 +492,5 @@ if $data04 != @70-01-01 07:59:57.000@ then
   return -1
 endi
 
+sql alter table tbn add column c1 int;
+sql alter table tbn drop column c1;

From 867e81c5f5f07b60cec275ad457fe2b8e1075dc2 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 29 Feb 2024 10:58:49 +0000
Subject: [PATCH 116/201] fix mem  leak while taosd quit

---
 source/libs/transport/src/transCli.c | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index e369bf8f7d..6ae72eac14 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -220,7 +220,7 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq,
 
 static FORCE_INLINE void destroyCmsg(void* cmsg);
 
-static FORCE_INLINE void destroyQueuedMsg(void* arg, void* param);
+static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param);
 static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg);
 static FORCE_INLINE int  cliRBChoseIdx(STrans* pTransInst);
 static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx);
@@ -1965,7 +1965,7 @@ static FORCE_INLINE void destroyCmsg(void* arg) {
   transFreeMsg(pMsg->msg.pCont);
   taosMemoryFree(pMsg);
 }
-static FORCE_INLINE void destroyQueuedMsg(void* arg, void* param) {
+static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param) {
   SCliMsg* pMsg = arg;
   if (pMsg == NULL) {
     return;
@@ -1974,9 +1974,7 @@ static FORCE_INLINE void destroyQueuedMsg(void* arg, void* param) {
     SCliThrd* pThrd = param;
     if (pThrd->destroyAhandleFp) (*pThrd->destroyAhandleFp)(pMsg->msg.info.ahandle);
   }
-  transDestroyConnCtx(pMsg->ctx);
-  transFreeMsg(pMsg->msg.pCont);
-  taosMemoryFree(pMsg);
+  destroyCmsg(pMsg);
 }
 static FORCE_INLINE void destroyCmsgAndAhandle(void* param) {
   if (param == NULL) return;
@@ -2071,7 +2069,7 @@ static void destroyThrdObj(SCliThrd* pThrd) {
   taosThreadJoin(pThrd->thread, NULL);
   CLI_RELEASE_UV(pThrd->loop);
   taosThreadMutexDestroy(&pThrd->msgMtx);
-  TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyQueuedMsg, (void*)pThrd);
+  TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsgWrapper, (void*)pThrd);
   transAsyncPoolDestroy(pThrd->asyncPool);
 
   transDQDestroy(pThrd->delayQueue, destroyCmsgAndAhandle);

From 3febcb96a0dab5ed7199d72d308b78f1b6699733 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Fri, 1 Mar 2024 10:57:57 +0800
Subject: [PATCH 117/201] reset flush state

---
 source/libs/stream/src/streamSessionState.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c
index 3d0241df75..723f04c499 100644
--- a/source/libs/stream/src/streamSessionState.c
+++ b/source/libs/stream/src/streamSessionState.c
@@ -156,6 +156,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey,
       (*pVal) = pPos;
       SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
       pPos->beUsed = true;
+      pPos->beFlushed = false;
       *pKey = *pDestWinKey;
       goto _end;
     }
@@ -167,6 +168,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey,
       (*pVal) = pPos;
       SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
       pPos->beUsed = true;
+      pPos->beFlushed = false;
       *pKey = *pDestWinKey;
       goto _end;
     }
@@ -380,6 +382,14 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co
     (*pWins) = pWinStates;
   }
 
+  if (size > 0 && index == -1) {
+    SRowBuffPos* pPos = taosArrayGetP(pWinStates, 0);
+    SSessionKey* pWin = (SSessionKey*)pPos->pKey;
+    if (pWinKey->win.skey == pWin->win.skey) {
+      index = 0;
+    }
+  }
+
   if (index >= 0) {
     pCur = createSessionStateCursor(pFileState);
     pCur->buffIndex = index;
@@ -387,6 +397,7 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co
       *pIndex = index;
     }
   }
+
   return pCur;
 }
 
@@ -666,6 +677,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch
       (*pVal) = pPos;
       SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
       pPos->beUsed = true;
+      pPos->beFlushed = false;
       *key = *pDestWinKey;
       goto _end;
     }
@@ -679,6 +691,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch
       (*pVal) = pPos;
       SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
       pPos->beUsed = true;
+      pPos->beFlushed = false;
       *key = *pDestWinKey;
       goto _end;
     }
@@ -771,6 +784,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
       (*pVal) = pPos;
       SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
       pPos->beUsed = true;
+      pPos->beFlushed = false;
       *pWinKey = *pDestWinKey;
       goto _end;
     }
@@ -799,6 +813,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
     (*pVal) = pPos;
     SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
     pPos->beUsed = true;
+    pPos->beFlushed = false;
     *pWinKey = *pDestWinKey;
     goto _end;
   }

From 9a5219d5943bd315d3f2e27ecb1d5f0d6b2a972f Mon Sep 17 00:00:00 2001
From: wangjiaming0909 <604227650@qq.com>
Date: Thu, 29 Feb 2024 13:19:26 +0800
Subject: [PATCH 118/201] fix: memory free sequence of sub request caused
 memory use after free

---
 source/client/inc/clientInt.h  |  1 +
 source/client/src/clientEnv.c  | 32 +++++++++++++++++--
 source/client/src/clientMain.c | 56 +++++++++++-----------------------
 3 files changed, 49 insertions(+), 40 deletions(-)

diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index 989c6614a6..59eee6fd9d 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -354,6 +354,7 @@ SRequestObj* acquireRequest(int64_t rid);
 int32_t      releaseRequest(int64_t rid);
 int32_t      removeRequest(int64_t rid);
 void         doDestroyRequest(void* p);
+int64_t      removeFromMostPrevReq(SRequestObj* pRequest);
 
 char* getDbOfConnection(STscObj* pObj);
 void  setConnectionDB(STscObj* pTscObj, const char* db);
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 1df50a51da..6c20813118 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -385,6 +385,33 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri
 
 int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); }
 
+/// return the most previous req ref id
+int64_t removeFromMostPrevReq(SRequestObj* pRequest) {
+  int64_t mostPrevReqRefId = pRequest->self;
+  SRequestObj* pTmp = pRequest;
+  while (pTmp->relation.prevRefId) {
+    pTmp = acquireRequest(pTmp->relation.prevRefId);
+    if (pTmp) {
+      mostPrevReqRefId = pTmp->self;
+      releaseRequest(mostPrevReqRefId);
+    } else {
+      break;
+    }
+  }
+  removeRequest(mostPrevReqRefId);
+  return mostPrevReqRefId;
+}
+
+void destroyNextReq(int64_t nextRefId) {
+  if (nextRefId) {
+    SRequestObj* pObj = acquireRequest(nextRefId);
+    if (pObj) {
+      releaseRequest(nextRefId);
+      releaseRequest(nextRefId);
+    }
+  }
+}
+
 void destroySubRequests(SRequestObj *pRequest) {
   int32_t      reqIdx = -1;
   SRequestObj *pReqList[16] = {NULL};
@@ -435,7 +462,7 @@ void doDestroyRequest(void *p) {
   uint64_t reqId = pRequest->requestId;
   tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest);
 
-  destroySubRequests(pRequest);
+  int64_t nextReqRefId = pRequest->relation.nextRefId;
 
   taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self));
 
@@ -471,6 +498,7 @@ void doDestroyRequest(void *p) {
   taosMemoryFreeClear(pRequest->sqlstr);
   taosMemoryFree(pRequest);
   tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest);
+  destroyNextReq(nextReqRefId);
 }
 
 void destroyRequest(SRequestObj *pRequest) {
@@ -479,7 +507,7 @@ void destroyRequest(SRequestObj *pRequest) {
   }
 
   taos_stop_query(pRequest);
-  removeRequest(pRequest->self);
+  removeFromMostPrevReq(pRequest);
 }
 
 void taosStopQueryImpl(SRequestObj *pRequest) {
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 275ca0d2aa..e9379946b1 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -1254,54 +1254,34 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
 }
 
 void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
-  int32_t      reqIdx = 0;
-  SRequestObj *pReqList[16] = {NULL};
-  SRequestObj *pUserReq = NULL;
-  pReqList[0] = pRequest;
-  uint64_t     tmpRefId = 0;
-  SRequestObj *pTmp = pRequest;
-  while (pTmp->relation.prevRefId) {
-    tmpRefId = pTmp->relation.prevRefId;
-    pTmp = acquireRequest(tmpRefId);
-    if (pTmp) {
-      pReqList[++reqIdx] = pTmp;
-      releaseRequest(tmpRefId);
-    } else {
-      tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId);
+  tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest);
+  SRequestObj* pUserReq = pRequest;
+  acquireRequest(pRequest->self);
+  while (pUserReq) {
+    if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) {
       break;
-    }
-  }
-
-  tmpRefId = pRequest->relation.nextRefId;
-  while (tmpRefId) {
-    pTmp = acquireRequest(tmpRefId);
-    if (pTmp) {
-      tmpRefId = pTmp->relation.nextRefId;
-      removeRequest(pTmp->self);
-      releaseRequest(pTmp->self);
     } else {
-      tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId);
-      break;
+      int64_t nextRefId = pUserReq->relation.nextRefId;
+      releaseRequest(pUserReq->self);
+      if (nextRefId) {
+        pUserReq = acquireRequest(nextRefId);
+      }
     }
   }
-
-  for (int32_t i = reqIdx; i >= 0; i--) {
-    destroyCtxInRequest(pReqList[i]);
-    if (pReqList[i]->relation.userRefId == pReqList[i]->self || 0 == pReqList[i]->relation.userRefId) {
-      pUserReq = pReqList[i];
-    } else {
-      removeRequest(pReqList[i]->self);
-    }
-  }
-
+  bool hasSubRequest = pUserReq != pRequest || pRequest->relation.prevRefId != 0;
   if (pUserReq) {
+    destroyCtxInRequest(pUserReq);
     pUserReq->prevCode = code;
     memset(&pUserReq->relation, 0, sizeof(pUserReq->relation));
   } else {
-    tscError("user req is missing");
+    tscError("User req is missing");
+    removeFromMostPrevReq(pRequest);
     return;
   }
-
+  if (hasSubRequest)
+    removeFromMostPrevReq(pRequest);
+  else
+    releaseRequest(pUserReq->self);
   doAsyncQuery(pUserReq, true);
 }
 

From 8df1f09561420ffbb0d98a1ba5248d60d3081783 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Fri, 1 Mar 2024 11:27:33 +0800
Subject: [PATCH 119/201] optimze plan and add ci

---
 source/libs/parser/src/parTranslater.c   | 42 +++++++++++-------------
 tests/script/tsim/query/query_count1.sim | 22 +++++++++++++
 2 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index cc522c9b5f..3a622eeea6 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -4009,6 +4009,26 @@ static int32_t translateEventWindow(STranslateContext* pCxt, SSelectStmt* pSelec
 }
 
 static int32_t translateCountWindow(STranslateContext* pCxt, SSelectStmt* pSelect) {
+  SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow;
+  if (pCountWin->windowCount <= 1) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
+                                    "Size of Count window must exceed 1.");
+  }
+
+  if (pCountWin->windowSliding <= 0) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
+                                    "Size of Count window must exceed 0.");
+  }
+
+  if (pCountWin->windowSliding > pCountWin->windowCount) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
+                                    "sliding value no larger than the count value.");
+  }
+
+  if (pCountWin->windowCount > INT32_MAX) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
+                                    "Size of Count window must less than 2147483647(INT32_MAX).");
+  }
   if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) &&
       !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
     return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_QUERY,
@@ -7828,29 +7848,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
     if (pStmt->pOptions->ignoreExpired != 1) {
       return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
                                     "Ignore expired data of Count window must be 1.");
-    }    
-
-    SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow;
-    if (pCountWin->windowCount <= 1) {
-      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
-                                     "Size of Count window must exceed 1.");
     }
-
-    if (pCountWin->windowSliding <= 0) {
-      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
-                                     "Size of Count window must exceed 0.");
-    }
-
-    if (pCountWin->windowSliding > pCountWin->windowCount) {
-      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
-                                     "sliding value no larger than the count value.");
-    }
-
-    if (pCountWin->windowCount > INT32_MAX) {
-      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
-                                     "Size of Count window must less than 2147483647(INT32_MAX).");
-    }
-
   }
 
   return TSDB_CODE_SUCCESS;
diff --git a/tests/script/tsim/query/query_count1.sim b/tests/script/tsim/query/query_count1.sim
index 0694ab062a..043b604263 100644
--- a/tests/script/tsim/query/query_count1.sim
+++ b/tests/script/tsim/query/query_count1.sim
@@ -79,5 +79,27 @@ if $data22 != 4 then
   goto loop3
 endi
 
+
+print step2
+print =============== create database
+sql create database test1 vgroups 1;
+sql use test1;
+
+sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+
+#2~INT32_MAX
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(-1);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(0);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(1);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(2147483648);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(10, 0);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(10, -1);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(10, 11);
+
+sql select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(2);
+sql select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(2147483647);
+
 print query_count0 end
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

From dfae8a9471e3bd3b9ed5697ea070759ae6488d89 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Fri, 1 Mar 2024 11:31:46 +0800
Subject: [PATCH 120/201] optimze plan and add ci

---
 source/libs/planner/src/planSpliter.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c
index d9a7475f59..9664be1da2 100644
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@@ -274,7 +274,7 @@ static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) {
     }
   }
 
-  if (WINDOW_TYPE_STATE == pWindow->winType) {
+  if (WINDOW_TYPE_STATE == pWindow->winType || WINDOW_TYPE_COUNT == pWindow->winType) {
     if (!streamQuery) {
       return stbSplHasMultiTbScan(streamQuery, pNode);
     } else {

From 1bf403660f27835fed9dffaaffa1ae3e08c4ce10 Mon Sep 17 00:00:00 2001
From: liuyao <38781207+54liuyao@users.noreply.github.com>
Date: Fri, 1 Mar 2024 15:02:26 +0800
Subject: [PATCH 121/201] Update 14-stream.md

---
 docs/zh/12-taos-sql/14-stream.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md
index 979fc436b9..c4ab450e79 100644
--- a/docs/zh/12-taos-sql/14-stream.md
+++ b/docs/zh/12-taos-sql/14-stream.md
@@ -49,10 +49,13 @@ window_clause: {
     SESSION(ts_col, tol_val)
   | STATE_WINDOW(col)
   | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)]
+  | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
+  | COUNT_WINDOW(count_val[, sliding_val])
 }
 ```
 
-其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。
+其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。COUNT_WINDOW是计数窗口,按固定的数据行数来划分窗口。
+count_val：常量，是正整数，必须大于等于2，最大INT32_MAX。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING
 
 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)
 

From e20dc784f82895d992fd357f0d86300f0272c752 Mon Sep 17 00:00:00 2001
From: liuyao <38781207+54liuyao@users.noreply.github.com>
Date: Fri, 1 Mar 2024 15:09:57 +0800
Subject: [PATCH 122/201] Update 14-stream.md

---
 docs/en/12-taos-sql/14-stream.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md
index 6f2343d347..e6aaaf0663 100644
--- a/docs/en/12-taos-sql/14-stream.md
+++ b/docs/en/12-taos-sql/14-stream.md
@@ -41,10 +41,14 @@ window_clause: {
     SESSION(ts_col, tol_val)
   | STATE_WINDOW(col)
   | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)]
+  | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
+  | COUNT_WINDOW(count_val[, sliding_val])
 }
 ```
 
 `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.
+COUNT_WINDOW is a counting window that is divided by a fixed number of data rows.count_val: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. count_val represents the maximum number of data rows contained in each COUNT_WINDOW. 
+When the total number of data rows cannot be divided by count_val, the number of rows in the last window will be less than count_val. sliding_val: is a constant that represents the number of window slides, similar to SLIDING in intervals
 
 For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable.
 

From 4a53bfb1077a9aea5e4cb9252320cdc03235ef58 Mon Sep 17 00:00:00 2001
From: liuyao <38781207+54liuyao@users.noreply.github.com>
Date: Fri, 1 Mar 2024 15:11:04 +0800
Subject: [PATCH 123/201] Update 14-stream.md

---
 docs/zh/12-taos-sql/14-stream.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md
index c4ab450e79..c36f355ffa 100644
--- a/docs/zh/12-taos-sql/14-stream.md
+++ b/docs/zh/12-taos-sql/14-stream.md
@@ -55,7 +55,7 @@ window_clause: {
 ```
 
 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。COUNT_WINDOW是计数窗口,按固定的数据行数来划分窗口。
-count_val：常量，是正整数，必须大于等于2，最大INT32_MAX。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING
+count_val：常量，是正整数，必须大于等于2，小于2147483648。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING
 
 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)
 

From ce379d077d9559dd128aa4c1b08ac8593329e9d8 Mon Sep 17 00:00:00 2001
From: liuyao <38781207+54liuyao@users.noreply.github.com>
Date: Fri, 1 Mar 2024 16:32:45 +0800
Subject: [PATCH 124/201] Update 14-stream.md

---
 docs/zh/12-taos-sql/14-stream.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md
index c36f355ffa..5a4772bb6c 100644
--- a/docs/zh/12-taos-sql/14-stream.md
+++ b/docs/zh/12-taos-sql/14-stream.md
@@ -54,8 +54,10 @@ window_clause: {
 }
 ```
 
-其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。COUNT_WINDOW是计数窗口,按固定的数据行数来划分窗口。
-count_val：常量，是正整数，必须大于等于2，小于2147483648。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING
+其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。
+EVENT_WINDOW是事件窗口，根据开始条件和结束条件来划定窗口。当start_trigger_condition满足时则窗口开始，直到end_trigger_condition满足时窗口关闭。start_trigger_condition和end_trigger_condition可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
+COUNT_WINDOW是计数窗口,按固定的数据行数来划分窗口。count_val：常量，是正整数，必须大于等于2，小于2147483648。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。
+sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING。
 
 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)
 

From dadaa9055f4d1696c57905be0362c510bcbb7b17 Mon Sep 17 00:00:00 2001
From: liuyao <38781207+54liuyao@users.noreply.github.com>
Date: Fri, 1 Mar 2024 16:40:24 +0800
Subject: [PATCH 125/201] Update 14-stream.md

---
 docs/en/12-taos-sql/14-stream.md | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md
index e6aaaf0663..099fa7a963 100644
--- a/docs/en/12-taos-sql/14-stream.md
+++ b/docs/en/12-taos-sql/14-stream.md
@@ -47,8 +47,12 @@ window_clause: {
 ```
 
 `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.
-COUNT_WINDOW is a counting window that is divided by a fixed number of data rows.count_val: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. count_val represents the maximum number of data rows contained in each COUNT_WINDOW. 
-When the total number of data rows cannot be divided by count_val, the number of rows in the last window will be less than count_val. sliding_val: is a constant that represents the number of window slides, similar to SLIDING in intervals
+
+`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition`
+can be any conditional expressions supported by TDengine and can include multiple columns.
+
+`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. 
+When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`.
 
 For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable.
 

From 413c1ee4ab86c75a5b1aa81a45f963fe5a1179f3 Mon Sep 17 00:00:00 2001
From: dapan1121 <72057773+dapan1121@users.noreply.github.com>
Date: Fri, 1 Mar 2024 16:47:55 +0800
Subject: [PATCH 126/201] Update 14-stream.md

---
 docs/en/12-taos-sql/14-stream.md | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md
index 099fa7a963..b8bbd4c154 100644
--- a/docs/en/12-taos-sql/14-stream.md
+++ b/docs/en/12-taos-sql/14-stream.md
@@ -48,11 +48,9 @@ window_clause: {
 
 `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.
 
-`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition`
-can be any conditional expressions supported by TDengine and can include multiple columns.
+`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns.
 
-`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. 
-When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`.
+`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`.
 
 For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable.
 

From 0926bbf8fa9f6dec9d59591af2f6511021e81ce4 Mon Sep 17 00:00:00 2001
From: liuyao <38781207+54liuyao@users.noreply.github.com>
Date: Fri, 1 Mar 2024 16:56:23 +0800
Subject: [PATCH 127/201] Update 14-stream.md

---
 docs/zh/12-taos-sql/14-stream.md | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md
index 5a4772bb6c..8868b728f8 100644
--- a/docs/zh/12-taos-sql/14-stream.md
+++ b/docs/zh/12-taos-sql/14-stream.md
@@ -55,9 +55,8 @@ window_clause: {
 ```
 
 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。
-EVENT_WINDOW是事件窗口，根据开始条件和结束条件来划定窗口。当start_trigger_condition满足时则窗口开始，直到end_trigger_condition满足时窗口关闭。start_trigger_condition和end_trigger_condition可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
-COUNT_WINDOW是计数窗口,按固定的数据行数来划分窗口。count_val：常量，是正整数，必须大于等于2，小于2147483648。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。
-sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLIDING。
+EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
+COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。
 
 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)
 
@@ -66,6 +65,12 @@ sliding_val：是常量，表示窗口滑动的数量，类似于 interval的SLI
 ```sql
 CREATE STREAM avg_vol_s INTO avg_vol AS
 SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
+
+CREATE STREAM streams0 INTO streamt0 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9;
+
+CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
 ```
 
 ## 流式计算的 partition

From 43a3f85f48679a809288b7e8b81033c2c4de93bc Mon Sep 17 00:00:00 2001
From: liuyao <38781207+54liuyao@users.noreply.github.com>
Date: Fri, 1 Mar 2024 16:57:24 +0800
Subject: [PATCH 128/201] Update 14-stream.md

---
 docs/en/12-taos-sql/14-stream.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md
index b8bbd4c154..e1bf18c854 100644
--- a/docs/en/12-taos-sql/14-stream.md
+++ b/docs/en/12-taos-sql/14-stream.md
@@ -57,6 +57,12 @@ For example, the following SQL statement creates a stream and automatically crea
 ```sql
 CREATE STREAM avg_vol_s INTO avg_vol AS
 SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
+
+CREATE STREAM streams0 INTO streamt0 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9;
+
+CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
 ```
 
 ## Partitions of Stream

From 5554362399a92249053b21b72efb8cace06945de Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 3 Mar 2024 16:41:16 +0800
Subject: [PATCH 129/201] fix(stream):remove unused def.

---
 include/libs/stream/tstream.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 64ce735843..cf980400d6 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -56,7 +56,6 @@ extern "C" {
 #define STREAM_EXEC_T_RESTART_ALL_TASKS (-4)
 #define STREAM_EXEC_T_STOP_ALL_TASKS    (-5)
 #define STREAM_EXEC_T_RESUME_TASK       (-6)
-#define STREAM_EXEC_T_UPDATE_TASK_EPSET (-7)
 
 typedef struct SStreamTask   SStreamTask;
 typedef struct SStreamQueue  SStreamQueue;

From 550d0fe5aef660aa50eab83d833e62e715a969b0 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sun, 3 Mar 2024 16:46:13 +0800
Subject: [PATCH 130/201] fix(stream): add an assert.

---
 source/libs/stream/src/streamDispatch.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 78b914c3db..9702096b60 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -740,6 +740,8 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) {
 
 int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) {
   int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+  ASSERT(dataStrLen > 0);
+
   void*   buf = taosMemoryCalloc(1, dataStrLen);
   if (buf == NULL) return -1;
 

From 8ff0fd346f913d9937569de5f4a86b6d5379f758 Mon Sep 17 00:00:00 2001
From: chenhaoran <haoran920c@163.com>
Date: Mon, 4 Mar 2024 02:42:03 +0800
Subject: [PATCH 131/201] test: add subscribe and commits in compatibility test

---
 .../system-test/0-others/com_alltypedata.json |  2 +-
 tests/system-test/0-others/compatibility.py   | 83 +++++++++++++++++--
 2 files changed, 76 insertions(+), 9 deletions(-)

diff --git a/tests/system-test/0-others/com_alltypedata.json b/tests/system-test/0-others/com_alltypedata.json
index 0e6d8e3a07..1499ca7670 100644
--- a/tests/system-test/0-others/com_alltypedata.json
+++ b/tests/system-test/0-others/com_alltypedata.json
@@ -22,7 +22,7 @@
                 "vgroups": 2,
                 "replica": 1,
                 "precision": "ms",
-                "stt_trigger": 8,
+                "stt_trigger": 1,
                 "minRows": 100,
                 "maxRows": 4096
             },
diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py
index c936cf1ae4..8163177a3b 100644
--- a/tests/system-test/0-others/compatibility.py
+++ b/tests/system-test/0-others/compatibility.py
@@ -1,11 +1,13 @@
 from urllib.parse import uses_relative
 import taos
+import taosws
 import sys
 import os
 import time
 import platform
 import inspect
 from taos.tmq import Consumer
+from taos.tmq import *
 
 from pathlib import Path
 from util.log import *
@@ -17,7 +19,7 @@ from util.dnodes import TDDnode
 from util.cluster import *
 import subprocess
 
-BASEVERSION = "3.0.2.3"
+BASEVERSION = "3.2.0.0"
 class TDTestCase:
     def caseDescription(self):
         f'''
@@ -30,7 +32,7 @@ class TDTestCase:
         self.replicaVar = int(replicaVar)
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
-        self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 4; ;use deldata;
+        self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 1; ;use deldata;
                             create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int);
                             create table deldata.ct1 using deldata.stb1 tags ( 1 );
                             insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a );
@@ -104,8 +106,19 @@ class TDTestCase:
             print(f"{packageName} has been exists")
         os.system(f" cd {packagePath} &&  tar xvf  {packageName} && cd {packageTPath} &&  ./install.sh  -e no  " )
         tdDnodes.stop(1)
-        print(f"start taosd: rm -rf {dataPath}/*  && nohup taosd -c {cPath} & ")
-        os.system(f"rm -rf {dataPath}/*  && nohup taosd -c {cPath} & " )
+        print(f"start taosd: rm -rf {dataPath}/*  && nohup /usr/bin/taosd -c {cPath} & ")
+        os.system(f"rm -rf {dataPath}/*  && nohup  /usr/bin/taosd -c {cPath} & " )
+        os.system(f"killall taosadapter" )
+        os.system(f"cp /etc/taos/taosadapter.toml {cPath}/taosadapter.toml  " )
+        taosadapter_cfg = cPath + "/taosadapter.toml"
+        taosadapter_log_path = cPath + "/../log/"
+        print(f"taosadapter_cfg:{taosadapter_cfg},taosadapter_log_path:{taosadapter_log_path} ")
+        self.alter_string_in_file(taosadapter_cfg,"#path = \"/var/log/taos\"",f"path = \"{taosadapter_log_path}\"")
+        self.alter_string_in_file(taosadapter_cfg,"taosConfigDir = \"\"",f"taosConfigDir = \"{cPath}\"")
+        print("/usr/bin/taosadapter --version")
+        os.system(f"  /usr/bin/taosadapter --version" )
+        print(f" LD_LIBRARY_PATH=/usr/lib -c  {taosadapter_cfg} 2>&1 & ")
+        os.system(f" LD_LIBRARY_PATH=/usr/lib  /usr/bin/taosadapter -c  {taosadapter_cfg} 2>&1 & " )
         sleep(5)
 
 
@@ -116,7 +129,24 @@ class TDTestCase:
     def is_list_same_as_ordered_list(self,unordered_list, ordered_list):
         sorted_list = sorted(unordered_list)
         return sorted_list == ordered_list
-        
+
+    def alter_string_in_file(self,file,old_str,new_str):
+        """
+        replace str in file
+        :param file
+        :param old_str
+        :param new_str
+        :return:
+        """
+        file_data = ""
+        with open(file, "r", encoding="utf-8") as f:
+            for line in f:
+                if old_str in line:
+                    line = line.replace(old_str,new_str)
+                file_data += line
+        with open(file,"w",encoding="utf-8") as f:
+            f.write(file_data)
+
     def run(self):
         scriptsPath = os.path.dirname(os.path.realpath(__file__))
         distro_id = distro.id()
@@ -131,7 +161,7 @@ class TDTestCase:
         dbname = "test"
         stb = f"{dbname}.meters"
         self.installTaosd(bPath,cPath)
-        os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ")
+        # os.system(f"echo 'debugFlag 143' >> {cPath}/taos.cfg ")
         tableNumbers=100
         recordNumbers1=100
         recordNumbers2=1000
@@ -163,11 +193,46 @@ class TDTestCase:
         # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ")
         # os.system('LD_LIBRARY_PATH=/usr/lib taos -s  "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ')
         # os.system('LD_LIBRARY_PATH=/usr/lib taos -s  "use test;show streams;" ')
-        os.system(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/tmqBasic.json ")
+        self.alter_string_in_file("0-others/tmqBasic.json", "/etc/taos/", cPath)
         # os.system("LD_LIBRARY_PATH=/usr/lib  taosBenchmark -f 0-others/tmqBasic.json -y ")
         os.system('LD_LIBRARY_PATH=/usr/lib taos -s  "create topic if not exists tmq_test_topic  as select  current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ')
         os.system('LD_LIBRARY_PATH=/usr/lib taos -s  "use test;show topics;" ')
+        os.system(f"  /usr/bin/taosadapter --version " )        
+        consumer_dict = {
+            "group.id": "g1",
+            "td.connect.user": "root",
+            "td.connect.pass": "taosdata",
+            "auto.offset.reset": "earliest",
+        }
 
+        consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
+        try:
+            consumer.subscribe(["tmq_test_topic"])
+        except TmqError:
+            tdLog.exit(f"subscribe error")
+
+        while True:
+            message = consumer.poll(timeout=1.0)
+            if message:
+                print("message")
+                id = message.vgroup()
+                topic = message.topic()
+                database = message.database()
+
+                for block in message:
+                    nrows = block.nrows()
+                    ncols = block.ncols()
+                    for row in block:
+                        print(row)
+                    values = block.fetchall()
+                    print(nrows, ncols)
+
+                consumer.commit(message)
+            else:
+                print("break")
+                break
+
+        consumer.close()
         tdLog.info(" LD_LIBRARY_PATH=/usr/lib  taosBenchmark -f 0-others/compa4096.json -y  ")
         os.system("LD_LIBRARY_PATH=/usr/lib  taosBenchmark -f 0-others/compa4096.json -y")
         os.system("LD_LIBRARY_PATH=/usr/lib  taos -s 'flush database db4096 '")
@@ -184,7 +249,8 @@ class TDTestCase:
                 
         os.system("pkill  taosd")   # make sure all the data are saved in disk.
         self.checkProcessPid("taosd")
-
+        os.system("pkill  taosadapter")   # make sure all the data are saved in disk.
+        self.checkProcessPid("taosadapter")
 
         tdLog.printNoPrefix("==========step2:update new version ")
         self.buildTaosd(bPath)
@@ -193,6 +259,7 @@ class TDTestCase:
         tdsql=tdCom.newTdSql()
         print(tdsql)
         cmd = f" LD_LIBRARY_PATH=/usr/lib  taos -h localhost ;"
+        print(os.system(cmd))
         if os.system(cmd) == 0:
             raise Exception("failed to execute system command. cmd: %s" % cmd)
         

From 0a814e97b554eae97a2e7b39f9ab3cbc937dece2 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 4 Mar 2024 09:26:54 +0800
Subject: [PATCH 132/201] enh(stream): reduce the threads requirements.

---
 source/common/src/tglobal.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index c5a26c5c10..55df80ca44 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4;
 int32_t tsNumOfMnodeFetchThreads = 1;
 int32_t tsNumOfMnodeReadThreads = 1;
 int32_t tsNumOfVnodeQueryThreads = 4;
-float   tsRatioOfVnodeStreamThreads = 1.5F;
+float   tsRatioOfVnodeStreamThreads = 0.5F;
 int32_t tsNumOfVnodeFetchThreads = 4;
 int32_t tsNumOfVnodeRsmaThreads = 2;
 int32_t tsNumOfQnodeQueryThreads = 4;

From 376872acc6bdd2e18c58711fe3921f7a4765d989 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 4 Mar 2024 09:53:49 +0800
Subject: [PATCH 133/201] fix(test/last_both): remove last_row query

---
 tests/script/tsim/parser/last_both.sim        |   6 +-
 tests/script/tsim/parser/last_both_query.sim  | 496 ++++++++++++++++++
 tests/script/tsim/parser/last_cache_query.sim | 106 ----
 3 files changed, 499 insertions(+), 109 deletions(-)
 create mode 100644 tests/script/tsim/parser/last_both_query.sim

diff --git a/tests/script/tsim/parser/last_both.sim b/tests/script/tsim/parser/last_both.sim
index d7daf4d333..7b6c40c127 100644
--- a/tests/script/tsim/parser/last_both.sim
+++ b/tests/script/tsim/parser/last_both.sim
@@ -69,13 +69,13 @@ sql drop table tbf;
 sql alter table st2 add column c1 int;
 sql alter table st2 drop column c1;
 
-run tsim/parser/last_cache_query.sim
+run tsim/parser/last_both_query.sim
 
 sql flush database $db
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode1 -s start
 
-run tsim/parser/last_cache_query.sim
+run tsim/parser/last_both_query.sim
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode1 -s start
@@ -145,6 +145,6 @@ sql alter database $db cachemodel 'both'
 sql alter database $db cachesize 2
 sleep 11000
 
-run tsim/parser/last_cache_query.sim
+run tsim/parser/last_both_query.sim
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/script/tsim/parser/last_both_query.sim b/tests/script/tsim/parser/last_both_query.sim
new file mode 100644
index 0000000000..775fd482ea
--- /dev/null
+++ b/tests/script/tsim/parser/last_both_query.sim
@@ -0,0 +1,496 @@
+
+sql connect
+
+$db = testdb
+sql use $db
+print "test tb1"
+
+sql select last(ts) from tb1
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+
+sql select last(f1) from tb1
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 6 then
+  print $data00
+  return -1
+endi
+
+sql select last(*) from tb1
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 5.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != 3 then
+  print expect 3, actual: $data03
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+
+sql select last(tb1.*,ts,f4) from tb1
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 5.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != 3 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+if $data05 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data06 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+
+print "test tb2"
+sql select last(ts) from tb2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-11 10:11:15.000@ then
+  print $data00
+  return -1
+endi
+
+sql select last(f1) from tb2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != -6 then
+  print $data00
+  return -1
+endi
+
+sql select last(*) from tb2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-11 10:11:15.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != -6 then
+  return -1
+endi
+if $data02 != -7.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != -8 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:56.999@ then
+  if $data04 != @70-01-01 07:59:57.-01@ then
+    return -1
+  endi
+endi
+
+sql select last(tb2.*,ts,f4) from tb2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-11 10:11:15.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != -6 then
+  return -1
+endi
+if $data02 != -7.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != -8 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:56.999@ then
+  if $data04 != @70-01-01 07:59:57.-01@ then
+    return -1
+  endi
+endi
+if $data05 != @21-05-11 10:11:15.000@ then
+  print $data00
+  return -1
+endi
+if $data06 != @70-01-01 07:59:56.999@ then
+  if $data04 != @70-01-01 07:59:57.-01@ then
+    return -1
+  endi
+endi
+
+print "test tbd"
+sql select last(*) from tbd
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-11 10:12:29.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != NULL then
+  return -1
+endi
+if $data02 != NULL then
+  print $data02
+  return -1
+endi
+if $data03 != NULL then
+  return -1
+endi
+if $data04 != NULL then
+  return -1
+endi
+
+print "test tbe"
+sql select last(*) from tbe
+if $rows != 0 then
+  return -1
+endi
+
+print "test stable"
+sql select last(ts) from st2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+
+sql select last(f1) from st2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 6 then
+  print $data00
+  return -1
+endi
+
+sql select last(*) from st2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 37.000000000 then
+  print expect 37.000000000 actual: $data02
+  return -1
+endi
+if $data03 != 27 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+
+
+sql select last(st2.*,ts,f4) from st2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 37.000000000 then
+  print expect 37.000000000, acutal: $data02
+  return -1
+endi
+if $data03 != 27 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+if $data05 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data06 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+
+sql select last(*), id from st2 group by id order by id
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49
+
+if $rows != 5 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 5.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != 21 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+if $data05 != 1 then
+  return -1
+endi
+if $data10 != @21-05-11 10:12:23.000@ then
+  return -1
+endi
+if $data11 != 22 then
+  return -1
+endi
+if $data12 != 23.000000000 then
+  print $data02
+  return -1
+endi
+if $data13 != -8 then
+  return -1
+endi
+if $data14 != @70-01-01 07:59:58.-04@ then
+  return -1
+endi
+if $data15 != 2 then
+  return -1
+endi
+if $data20 != @21-05-10 10:12:24.000@ then
+  return -1
+endi
+if $data21 != 24 then
+  return -1
+endi
+if $data22 != 11.000000000 then
+  print expect 11.000000000 actual: $data22
+  return -1
+endi
+if $data23 != 25 then
+  return -1
+endi
+if $data24 != @70-01-01 07:59:57.-04@ then =
+  return -1
+endi
+if $data25 != 3 then
+  return -1
+endi
+if $data30 != @21-05-11 10:12:25.000@ then
+  return -1
+endi
+if $data31 != 26 then
+  return -1
+endi
+if $data32 != 17.000000000 then
+  print $data02
+  return -1
+endi
+if $data33 != 27 then
+  return -1
+endi
+if $data34 != @70-01-01 07:59:56.-04@ then
+  return -1
+endi
+if $data35 != 4 then
+  return -1
+endi
+if $data40 != @21-05-11 10:12:29.000@ then
+  return -1
+endi
+if $data41 != 36 then
+  return -1
+endi
+if $data42 != 37.000000000 then
+  print $data02
+  return -1
+endi
+if $data43 != 35 then
+  return -1
+endi
+if $data44 != @70-01-01 07:59:56.-05@ then
+  return -1
+endi
+if $data45 != 5 then
+  return -1
+endi
+
+#sql select last_row(*), id from st2 group by id order by id
+#print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+#print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+#print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+#print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+#print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49
+#
+#if $rows != 5 then
+#  return -1
+#endi
+#if $data00 != @21-05-12 10:10:12.000@ then
+#  return -1
+#endi
+#if $data01 != 6 then
+#  return -1
+#endi
+#if $data02 != NULL then
+#  print $data02
+#  return -1
+#endi
+#if $data03 != NULL then
+#  return -1
+#endi
+#if $data04 != @70-01-01 07:59:57.000@ then
+#  return -1
+#endi
+#if $data05 != 1 then
+#  return -1
+#endi
+#if $data10 != @21-05-11 10:12:23.000@ then
+#  return -1
+#endi
+#if $data11 != 22 then
+#  return -1
+#endi
+#if $data12 != 23.000000000 then
+#  print $data02
+#  return -1
+#endi
+#if $data13 != NULL then
+#  return -1
+#endi
+#if $data14 != @70-01-01 07:59:58.-04@ then
+#  return -1
+#endi
+#if $data15 != 2 then
+#  return -1
+#endi
+#if $data20 != @21-05-10 10:12:24.000@ then
+#  return -1
+#endi
+#if $data21 != 24 then
+#  return -1
+#endi
+#if $data22 != NULL then
+#  print expect NULL actual: $data22
+#  return -1
+#endi
+#if $data23 != 25 then
+#  return -1
+#endi
+#if $data24 != @70-01-01 07:59:57.-04@ then =
+#  return -1
+#endi
+#if $data25 != 3 then
+#  return -1
+#endi
+#if $data30 != @21-05-11 10:12:25.000@ then
+#  return -1
+#endi
+#if $data31 != 26 then
+#  return -1
+#endi
+#if $data32 != NULL then
+#  print $data02
+#  return -1
+#endi
+#if $data33 != 27 then
+#  return -1
+#endi
+#if $data34 != @70-01-01 07:59:56.-04@ then
+#  return -1
+#endi
+#if $data35 != 4 then
+#  return -1
+#endi
+#if $data40 != @21-05-11 10:12:29.000@ then
+#  return -1
+#endi
+#if $data41 != NULL then
+#  return -1
+#endi
+#if $data42 != NULL then
+#  print $data02
+#  return -1
+#endi
+#if $data43 != NULL then
+#  return -1
+#endi
+#if $data44 != NULL then
+#  return -1
+#endi
+#if $data45 != 5 then
+#  return -1
+#endi
+
+print "test tbn"
+sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp)
+sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3',  -1000)
+sql insert into tbn values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000)
+sql insert into tbn values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000)
+sql insert into tbn values ("2021-05-13 10:10:12", NULL,NULL, NULL,NULL)
+
+sql select last(*) from tbn;
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-13 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 5.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != 3 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+
+sql alter table tbn add column c1 int;
+sql alter table tbn drop column c1;
diff --git a/tests/script/tsim/parser/last_cache_query.sim b/tests/script/tsim/parser/last_cache_query.sim
index c5961f2183..30196e0b62 100644
--- a/tests/script/tsim/parser/last_cache_query.sim
+++ b/tests/script/tsim/parser/last_cache_query.sim
@@ -357,112 +357,6 @@ if $data45 != 5 then
   return -1
 endi
 
-sql select last_row(*), id from st2 group by id order by id
-print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
-print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
-print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
-print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
-print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49
-
-if $rows != 5 then
-  return -1
-endi
-if $data00 != @21-05-12 10:10:12.000@ then
-  return -1
-endi
-if $data01 != 6 then
-  return -1
-endi
-if $data02 != NULL then
-  print $data02
-  return -1
-endi
-if $data03 != NULL then
-  return -1
-endi
-if $data04 != @70-01-01 07:59:57.000@ then
-  return -1
-endi
-if $data05 != 1 then
-  return -1
-endi
-if $data10 != @21-05-11 10:12:23.000@ then
-  return -1
-endi
-if $data11 != 22 then
-  return -1
-endi
-if $data12 != 23.000000000 then
-  print $data02
-  return -1
-endi
-if $data13 != NULL then
-  return -1
-endi
-if $data14 != @70-01-01 07:59:58.-04@ then
-  return -1
-endi
-if $data15 != 2 then
-  return -1
-endi
-if $data20 != @21-05-10 10:12:24.000@ then
-  return -1
-endi
-if $data21 != 24 then
-  return -1
-endi
-if $data22 != NULL then
-  print expect NULL actual: $data22
-  return -1
-endi
-if $data23 != 25 then
-  return -1
-endi
-if $data24 != @70-01-01 07:59:57.-04@ then =
-  return -1
-endi
-if $data25 != 3 then
-  return -1
-endi
-if $data30 != @21-05-11 10:12:25.000@ then
-  return -1
-endi
-if $data31 != 26 then
-  return -1
-endi
-if $data32 != NULL then
-  print $data02
-  return -1
-endi
-if $data33 != 27 then
-  return -1
-endi
-if $data34 != @70-01-01 07:59:56.-04@ then
-  return -1
-endi
-if $data35 != 4 then
-  return -1
-endi
-if $data40 != @21-05-11 10:12:29.000@ then
-  return -1
-endi
-if $data41 != 36 then
-  return -1
-endi
-if $data42 != 37.000000000 then
-  print $data02
-  return -1
-endi
-if $data43 != NULL then
-  return -1
-endi
-if $data44 != @70-01-01 07:59:56.-05@ then
-  return -1
-endi
-if $data45 != 5 then
-  return -1
-endi
-
 print "test tbn"
 sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp)
 sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3',  -1000)

From c7d115d8aaa6f20b26575ec73ccdb269f4ec875b Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 4 Mar 2024 14:58:13 +0800
Subject: [PATCH 134/201] fix:add excluded msg for delete in tmq

---
 include/common/tmsg.h                   | 3 +++
 source/client/src/clientRawBlockWrite.c | 2 ++
 source/common/src/tmsg.c                | 9 +++++++++
 source/dnode/vnode/src/tq/tqUtil.c      | 3 +--
 source/libs/qworker/src/qwMsg.c         | 1 +
 source/libs/scheduler/src/schRemote.c   | 1 +
 tests/script/sh/deploy.sh               | 2 +-
 utils/test/c/tmq_taosx_ci.c             | 1 +
 8 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 587e2f9f3e..e400698f69 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -3813,8 +3813,10 @@ typedef struct {
   uint32_t phyLen;
   char*    sql;
   char*    msg;
+  int8_t   source;
 } SVDeleteReq;
 
+extern int8_t deleteFromTaosx;
 int32_t tSerializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq);
 int32_t tDeserializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq);
 
@@ -3834,6 +3836,7 @@ typedef struct SDeleteRes {
   char     tableFName[TSDB_TABLE_NAME_LEN];
   char     tsColName[TSDB_COL_NAME_LEN];
   int64_t  ctimeMs;  // fill by vnode
+  int8_t   source;
 } SDeleteRes;
 
 int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes);
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index f143624bab..3857e47f57 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1256,7 +1256,9 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
   snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName,
            req.tsColName, req.skey, req.tsColName, req.ekey);
 
+  deleteFromTaosx = TD_REQ_FROM_TAOX;
   TAOS_RES*    res = taos_query(taos, sql);
+  deleteFromTaosx = TD_REQ_FROM_APP;
   SRequestObj* pRequest = (SRequestObj*)res;
   code = pRequest->code;
   if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_PAR_GET_META_ERROR) {
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 85f5d462c7..e7414c6004 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -7148,6 +7148,7 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
   return 0;
 }
 
+int8_t deleteFromTaosx = TD_REQ_FROM_APP;
 int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
   int32_t headLen = sizeof(SMsgHead);
   if (buf != NULL) {
@@ -7165,6 +7166,7 @@ int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
   if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
   if (tEncodeBinary(&encoder, pReq->msg, pReq->phyLen) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -7201,6 +7203,9 @@ int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
   if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, &msgLen) < 0) return -1;
   pReq->phyLen = msgLen;
 
+  if (!tDecodeIsEnd(&decoder)) {
+    if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
+  }
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
@@ -8400,6 +8405,7 @@ int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) {
   if (tEncodeCStr(pCoder, pRes->tableFName) < 0) return -1;
   if (tEncodeCStr(pCoder, pRes->tsColName) < 0) return -1;
   if (tEncodeI64(pCoder, pRes->ctimeMs) < 0) return -1;
+  if (tEncodeI8(pCoder, pRes->source) < 0) return -1;
   return 0;
 }
 
@@ -8424,6 +8430,9 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) {
   if (!tDecodeIsEnd(pCoder)) {
     if (tDecodeI64(pCoder, &pRes->ctimeMs) < 0) return -1;
   }
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI8(pCoder, &pRes->source) < 0) return -1;
+  }
   return 0;
 }
 
diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c
index dad1211326..6029575e2c 100644
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@@ -263,8 +263,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
           } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) {
             PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq)
           } else if (pHead->msgType == TDMT_VND_DELETE) {
-            fetchVer++;
-            continue;
+            PROCESS_EXCLUDED_MSG(SDeleteRes, tDecodeDeleteRes)
           }
         }
 
diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c
index 66ec460861..faa90dcbf8 100644
--- a/source/libs/qworker/src/qwMsg.c
+++ b/source/libs/qworker/src/qwMsg.c
@@ -715,6 +715,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD
   uint64_t tId = req.taskId;
   int64_t  rId = 0;
   int32_t  eId = -1;
+  pRes->source = req.source;
 
   SQWMsg qwMsg = {.node = node, .msg = req.msg, .msgLen = req.phyLen, .connInfo = pMsg->info};
   QW_SCH_TASK_DLOG("processDelete start, node:%p, handle:%p, sql:%s", node, pMsg->info.handle, req.sql);
diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c
index 1c0b31109e..f6465316d2 100644
--- a/source/libs/scheduler/src/schRemote.c
+++ b/source/libs/scheduler/src/schRemote.c
@@ -1086,6 +1086,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
       req.sqlLen = strlen(pJob->sql);
       req.sql = (char *)pJob->sql;
       req.msg = pTask->msg;
+      req.source = deleteFromTaosx;
       msgSize = tSerializeSVDeleteReq(NULL, 0, &req);
       msg = taosMemoryCalloc(1, msgSize);
       if (NULL == msg) {
diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh
index 3b3d275a07..9af2525c74 100755
--- a/tests/script/sh/deploy.sh
+++ b/tests/script/sh/deploy.sh
@@ -117,7 +117,7 @@ echo "supportVnodes          1024"                >> $TAOS_CFG
 echo "statusInterval         1"                  >> $TAOS_CFG
 echo "dataDir                $DATA_DIR"          >> $TAOS_CFG
 echo "logDir                 $LOG_DIR"           >> $TAOS_CFG
-echo "debugFlag              0"                  >> $TAOS_CFG
+echo "debugFlag              135"                  >> $TAOS_CFG
 echo "tmrDebugFlag           131"                >> $TAOS_CFG
 echo "uDebugFlag             143"                >> $TAOS_CFG
 echo "rpcDebugFlag           143"                >> $TAOS_CFG
diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c
index 2257089f06..5012e50bab 100644
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@@ -574,6 +574,7 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "msg.with.table.name", "true");
   tmq_conf_set(conf, "enable.auto.commit", "true");
   tmq_conf_set(conf, "auto.offset.reset", "earliest");
+  tmq_conf_set(conf, "msg.consume.excluded", "1");
 
   if (g_conf.snapShot) {
     tmq_conf_set(conf, "experimental.snapshot.enable", "true");

From b6095261f238e486c0530dec0bd3674c19b47daa Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Fri, 1 Mar 2024 17:05:24 +0800
Subject: [PATCH 135/201] feat: ordered region and blocks

---
 source/libs/executor/inc/tsort.h        |   2 +-
 source/libs/executor/src/scanoperator.c |   2 +-
 source/libs/executor/src/tsort.c        | 347 +++++++++++-------------
 3 files changed, 166 insertions(+), 185 deletions(-)

diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h
index b4e0c70f31..ca799673ea 100644
--- a/source/libs/executor/inc/tsort.h
+++ b/source/libs/executor/inc/tsort.h
@@ -194,7 +194,7 @@ void tsortSetClosed(SSortHandle* pHandle);
 void tsortSetSingleTableMerge(SSortHandle* pHandle);
 void tsortSetAbortCheckFn(SSortHandle* pHandle, bool (*checkFn)(void* param), void* param);
 
-int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsPageSize, int32_t extRowsSize);
+int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsSize);
 
 void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHandle* pTupleHandle);
 /**
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 1d64b066e9..1636cd21f0 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4015,7 +4015,7 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
     pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
                                                pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
     int32_t memSize = 512 * 1024 * 1024;
-    code = tsortSetSortByRowId(pInfo->pSortHandle, pInfo->bufPageSize, memSize);
+    code = tsortSetSortByRowId(pInfo->pSortHandle, memSize);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 1ceb5403c9..38ab506918 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -45,23 +45,27 @@ typedef struct SSortMemPageEntry {
 
 } SSortMemPageEntry;
 
-typedef struct SSortMemFile {
-  int32_t pageSize;
-  int32_t cacheSize;
-  
-  char*   writePageBuf;
-  int32_t startPageId;
-  int32_t numWritePages;
-  
-  int32_t currPageId;
-  int32_t currPageOffset;
-  bool bDirty;
+typedef struct SSortMemFileRegion {
+  int64_t fileOffset;
+  int32_t regionSize;
 
-  int32_t totalMemPages;
-  SSortMemPageEntry* pagesHead;
-  SSortMemPageEntry* pagesTail;
-  int32_t numMemPages;
-  SSHashObj* mActivePages;
+  int32_t bufRegOffset;
+  int32_t bufLen;
+  char* buf;
+} SSortMemFileRegion;
+
+typedef struct SSortMemFile {
+  char* writeBuf;
+  int32_t writeBufSize;
+  int64_t writeFileOffset;
+
+  int32_t currRegionId;
+  int32_t currRegionOffset;
+  bool    bRegionDirty;
+
+  SArray* aFileRegions;
+  int32_t cacheSize;
+  int32_t blockSize;
 
   FILE* pTdFile;
   // TdFilePtr pTdFile;
@@ -125,11 +129,8 @@ struct SSortHandle {
 };
 
 static int32_t destroySortMemFile(SSortHandle* pHandle);
-static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char** ppPage);
-static void    setExtMemFilePageUnused(SSortMemFile* pMemFile, int32_t pageId);
-static int32_t saveDirtyPagesToExtRowsMemFile(SSortHandle* pHandle);
-static int32_t freeExtRowMemFileWriteBuf(SSortHandle* pHandle);
-
+static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, int32_t tupleOffset, int32_t rowLen,
+                                       char** ppRow, bool* pFreeRow);
 void tsortSetSingleTableMerge(SSortHandle* pHandle) {
   pHandle->singleTableMerge = true;
 }
@@ -915,14 +916,14 @@ static int32_t createPageBuf(SSortHandle* pHandle) {
 
 void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
   if (pHandle->bSortByRowId) {
-    int32_t pageId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
+    int32_t regionId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
     int32_t offset = *(int32_t*)tsortGetValue(pTupleHandle, 2);
+    int32_t length = *(int32_t*)tsortGetValue(pTupleHandle, 3);
 
-    char* page = NULL;
-    getPageFromExtMemFile(pHandle, pageId, &page);
-
+    char* buf = NULL;
+    bool bFreeRow = false;
+    getRowBufFromExtMemFile(pHandle, regionId, offset, length, &buf, &bFreeRow);
     int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-    char*   buf = (char*)page + offset;
     char*   isNull = (char*)buf;
     char*   pStart = (char*)buf + sizeof(int8_t) * numOfCols;
     for (int32_t i = 0; i < numOfCols; ++i) {
@@ -943,7 +944,9 @@ void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHa
         colDataSetNULL(pColInfo, pBlock->info.rows);
       }
     }
-
+    if (bFreeRow) {
+      taosMemoryFree(buf);
+    }
     if (*(int32_t*)pStart != pStart - buf) {
       qError("table merge scan row buf deserialization. length error %d != %d ", *(int32_t*)pStart,
              (int32_t)(pStart - buf));
@@ -953,9 +956,6 @@ void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHa
     pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag;
     pBlock->info.rows += 1;
 
-    if (offset + pHandle->extRowBytes >= pHandle->pExtRowsMemFile->pageSize) {
-      setExtMemFilePageUnused(pHandle->pExtRowsMemFile, pageId);
-    }
   } else {
     for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
       SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
@@ -1020,102 +1020,60 @@ static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) {
   return (int32_t)(pStart - (char*)buf);
 }
 
-static int32_t getPageFromExtMemFile(SSortHandle* pHandle, int32_t pageId, char** ppPage) {
+static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, int32_t tupleOffset, int32_t rowLen,
+                                       char** ppRow, bool* pFreeRow) {
   SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
-  SSortMemPageEntry** ppPageEntry = tSimpleHashGet(pMemFile->mActivePages, &pageId, sizeof(pageId));
-  if (ppPageEntry) {
-    *ppPage = (char*)((*ppPageEntry)->data);
+  SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, regionId);
+  if (pRegion->buf == NULL) {
+    pRegion->bufRegOffset = 0;
+    pRegion->buf = taosMemoryMalloc(pMemFile->blockSize);
+    tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET);
+    int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize);
+    fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
+    pRegion->bufLen = readBytes;
+  }
+  // TODO: ASSERT(pRegion->offset < tupleOffset);
+  if (pRegion->bufRegOffset + pRegion->bufLen >= tupleOffset + rowLen) {
+    *pFreeRow = false;
+    *ppRow = pRegion->buf + tupleOffset - pRegion->bufRegOffset;
   } else {
-    SSortMemPageEntry* pEntry = pMemFile->pagesHead->next;
-    bool freeEntryWhenError = false;
-    if (pEntry && !pEntry->active || pMemFile->numMemPages >= pMemFile->totalMemPages) {
-      if (pEntry->active) {
-        tSimpleHashRemove(pMemFile->mActivePages, &pEntry->pageId, sizeof(pEntry->pageId));
-      }
-      pEntry->prev->next = pEntry->next;
-      pEntry->next->prev = pEntry->prev;
-      pEntry->active = false;
-    } else if (pMemFile->numMemPages < pMemFile->totalMemPages) {
-      pEntry = taosMemoryCalloc(1, sizeof(SSortMemPageEntry));
-      pEntry->data = taosMemoryMalloc(pMemFile->pageSize);
-      freeEntryWhenError = true; 
-      ++pMemFile->numMemPages;
-    }
-    {
-      int ret = tsortSeekFile(pMemFile->pTdFile, ((int64_t)pageId) * pMemFile->pageSize, SEEK_SET);
-      if (ret == 0) {
-        ret = fread(pEntry->data, pMemFile->pageSize, 1, pMemFile->pTdFile);
-      }
-      if (ret != 1) {
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        if (freeEntryWhenError) {
-          taosMemoryFreeClear(pEntry->data);
-          taosMemoryFreeClear(pEntry);
-        }
-        return terrno;
-      }
-      SSortMemPageEntry* tail = pMemFile->pagesTail;
-      tail->next = pEntry;
-      pEntry->next = NULL;
-      pEntry->prev = tail;
-      pEntry->active = true;
-      pMemFile->pagesTail = pEntry;
-      tSimpleHashPut(pMemFile->mActivePages, &pageId, sizeof(pageId), &pEntry, POINTER_BYTES);
-      *ppPage = pEntry->data;
-    }
+      *ppRow = taosMemoryMalloc(rowLen);
+      int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset);
+      memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset,
+             szThisBlock);
+      tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
+      int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen));
+      fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
+      memcpy(*ppRow + szThisBlock, pRegion->buf, rowLen - szThisBlock);
+      *pFreeRow = true;
+      pRegion->bufRegOffset += pRegion->bufLen;
+      pRegion->bufLen = readBytes;
   }
+  //TODO: free region memory
   return TSDB_CODE_SUCCESS;
 }
 
-static void setExtMemFilePageUnused(SSortMemFile* pMemFile, int32_t pageId) {
-  SSortMemPageEntry** ppPageEntry = tSimpleHashGet(pMemFile->mActivePages, &pageId, sizeof(pageId));
-  SSortMemPageEntry* pEntry = *ppPageEntry;
-  if (pEntry == pMemFile->pagesTail) {
-    pMemFile->pagesTail = pEntry->prev;
-  }
-
-  pEntry->prev->next = pEntry->next;
-  pEntry->next->prev = pEntry->prev;
-
-  SSortMemPageEntry* first = pMemFile->pagesHead->next;
-  SSortMemPageEntry* head = pMemFile->pagesHead;
-  head->next = pEntry;
-  pEntry->next = first;
-  first->prev = pEntry;
-  pEntry->prev = head;
-
-  pEntry->active = false;
-  tSimpleHashRemove(pMemFile->mActivePages, &pageId, sizeof(pageId));
-  return;
-}
-
 static int32_t createSortMemFile(SSortHandle* pHandle) {
   if (pHandle->pExtRowsMemFile != NULL) {
     return TSDB_CODE_SUCCESS;
   }
   SSortMemFile* pMemFile = taosMemoryCalloc(1, sizeof(SSortMemFile));
-
+  pMemFile->cacheSize = pHandle->extRowsMemSize;
   taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
   pMemFile->pTdFile = fopen(pMemFile->memFilePath, "w+");
   if (pMemFile->pTdFile == NULL) {
     taosMemoryFree(pMemFile);
     return TAOS_SYSTEM_ERROR(errno);
   }
-  pMemFile->currPageId = -1;
-  pMemFile->currPageOffset = -1;
+  pMemFile->currRegionId = -1;
+  pMemFile->currRegionOffset = -1;
 
-  pMemFile->pageSize = pHandle->extRowsPageSize;
-  pMemFile->cacheSize = pHandle->extRowsMemSize;
-  pMemFile->numWritePages = pMemFile->cacheSize/pMemFile->pageSize;
-  pMemFile->writePageBuf = taosMemoryMalloc(pMemFile->pageSize * pMemFile->numWritePages);
-  pMemFile->bDirty = false;
+  pMemFile->writeBufSize = 64 * 1024 * 1024;
+  pMemFile->writeBuf = taosMemoryMalloc(pMemFile->writeBufSize);
+  pMemFile->writeFileOffset = -1;
+  pMemFile->bRegionDirty = false;
 
-  pMemFile->mActivePages = tSimpleHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
-  pMemFile->pagesHead = taosMemoryCalloc(1, sizeof(SSortMemPageEntry));
-  pMemFile->pagesTail = pMemFile->pagesHead;
-
-  pMemFile->totalMemPages = pMemFile->cacheSize / pMemFile->pageSize;
-  pMemFile->numMemPages = 0;
+  pMemFile->aFileRegions = taosArrayInit(64, sizeof(SSortMemFileRegion));
 
   pHandle->pExtRowsMemFile = pMemFile;
   return TSDB_CODE_SUCCESS;
@@ -1125,20 +1083,15 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
   if (pHandle->pExtRowsMemFile == NULL) return TSDB_CODE_SUCCESS;
 
   SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
-  SSortMemPageEntry* pEntry = pMemFile->pagesHead;
-  while (pEntry != NULL) {
-    if (pEntry->data) {
-      taosMemoryFree(pEntry->data);
-    }
-    SSortMemPageEntry* pCurr = pEntry;
-    pEntry = pEntry->next;
-    taosMemoryFree(pCurr);
+  for (int32_t i = 0; i < taosArrayGetSize(pMemFile->aFileRegions); ++i) {
+    SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, i);
+    taosMemoryFree(pRegion->buf);
   }
-  tSimpleHashCleanup(pMemFile->mActivePages);
-  pMemFile->mActivePages = NULL;
+  taosArrayDestroy(pMemFile->aFileRegions);
+  pMemFile->aFileRegions = NULL;
 
-  taosMemoryFree(pMemFile->writePageBuf);
-  pMemFile->writePageBuf = NULL;
+  taosMemoryFree(pMemFile->writeBuf);
+  pMemFile->writeBuf = NULL;
 
   fclose(pMemFile->pTdFile);
   pMemFile->pTdFile = NULL;
@@ -1148,68 +1101,89 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pPageId, int32_t* pOffset, int32_t* pLength) {
+static int32_t tsortOpenRegion(SSortHandle* pHandle) {
   SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
-  if (pMemFile->currPageId == -1) {
-    pMemFile->currPageId = 0;
-    pMemFile->currPageOffset = 0;
-    pMemFile->startPageId = 0;
+  if (pMemFile->currRegionId == -1) {
+    SSortMemFileRegion region = {0};
+    region.fileOffset = 0;
+    region.bufRegOffset = 0;
+    taosArrayPush(pMemFile->aFileRegions, &region);
+    pMemFile->currRegionId = 0;
+    pMemFile->currRegionOffset = 0;
+    pMemFile->writeFileOffset = 0;
   } else {
-    if (pMemFile->currPageOffset + pHandle->extRowBytes >= pMemFile->pageSize) {
+    SSortMemFileRegion regionNew = {0};
+    SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, pMemFile->currRegionId);
+    regionNew.fileOffset = pRegion->fileOffset + pRegion->regionSize;
+    regionNew.bufRegOffset = 0;
+    taosArrayPush(pMemFile->aFileRegions, &regionNew);
+    ++pMemFile->currRegionId;
+    pMemFile->currRegionOffset = 0;
+    pMemFile->writeFileOffset = regionNew.fileOffset;
+  }
+  return TSDB_CODE_SUCCESS;
+}
 
-      ++pMemFile->currPageId;
-      pMemFile->currPageOffset = 0;
+static int32_t tsortCloseRegion(SSortHandle* pHandle) {
+  SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
+  SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, pMemFile->currRegionId);
+  pRegion->regionSize = pMemFile->currRegionOffset;
+  int32_t writeBytes = pRegion->regionSize - (pMemFile->writeFileOffset - pRegion->fileOffset);
+  if (writeBytes > 0) {
+    int ret = tsortSeekFile(pMemFile->pTdFile, pMemFile->writeFileOffset, SEEK_SET);
+    if (ret == 0) {
+      ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile);
+    }
+    if (ret != 1) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return terrno;
+    }
+    pMemFile->bRegionDirty = false;
+  }
+  return TSDB_CODE_SUCCESS;
+}
 
-      if (pMemFile->currPageId - pMemFile->startPageId >= pMemFile->numWritePages) {
-        int ret = tsortSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
-        if (ret == 0) {
-          ret = fwrite(pMemFile->writePageBuf, pMemFile->pageSize * pMemFile->numWritePages, 1, pMemFile->pTdFile);
-        }
-        if (ret !=  1) {
-          terrno = TAOS_SYSTEM_ERROR(errno);
-          return terrno;
-        }          
-        pMemFile->startPageId = pMemFile->currPageId;
+static int32_t tsortFinalizeRegions(SSortHandle* pHandle) {
+  SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
+  size_t numRegions = taosArrayGetSize(pMemFile->aFileRegions);
+  ASSERT(numRegions == (pMemFile->currRegionId + 1));
+  if (numRegions == 0) return TSDB_CODE_SUCCESS;
+  int32_t blockReadBytes = (pMemFile->cacheSize / numRegions + 4095) & ~4095;
+  pMemFile->blockSize = blockReadBytes;
+
+  for (int32_t i = 0; i < numRegions; ++i) {
+    SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, i);
+    pRegion->bufRegOffset = 0;
+  }
+  taosMemoryFree(pMemFile->writeBuf);
+  pMemFile->writeBuf = NULL;
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* pBlock, int32_t rowIdx, int32_t* pRegionId, int32_t* pOffset, int32_t* pLength) {
+  SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
+  SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, pMemFile->currRegionId);
+  {
+    if (pMemFile->currRegionOffset + pHandle->extRowBytes >= pMemFile->writeBufSize) {
+      int ret = tsortSeekFile(pMemFile->pTdFile, pMemFile->writeFileOffset, SEEK_SET);
+      int32_t writeBytes = pMemFile->currRegionOffset - (pMemFile->writeFileOffset - pRegion->fileOffset);
+      if (ret == 0) {
+        ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile);
       }
+      if (ret !=  1) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        return terrno;
+      }
+      pMemFile->writeFileOffset = pRegion->fileOffset + pMemFile->currRegionOffset;
     }
   }
-
-  *pPageId = pMemFile->currPageId;
-  *pOffset = pMemFile->currPageOffset;
-  int32_t offsetPages = (pMemFile->currPageId - pMemFile->startPageId) * pMemFile->pageSize;
-  int32_t blockLen = blockRowToBuf(pBlock, rowIdx, 
-                  pMemFile->writePageBuf + offsetPages + pMemFile->currPageOffset);
+  *pRegionId = pMemFile->currRegionId;
+  *pOffset = pMemFile->currRegionOffset;
+  int32_t writeBufOffset = pMemFile->currRegionOffset - (pMemFile->writeFileOffset - pRegion->fileOffset);
+  int32_t blockLen = blockRowToBuf(pBlock, rowIdx, pMemFile->writeBuf + writeBufOffset);
   *pLength = blockLen;
-  pMemFile->currPageOffset += blockLen;
-  pMemFile->bDirty = true;
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t saveDirtyPagesToExtRowsMemFile(SSortHandle* pHandle) {
-  SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
-  if (!pMemFile->bDirty) {
-    return TSDB_CODE_SUCCESS;
-  }
-  int ret = tsortSeekFile(pMemFile->pTdFile, ((int64_t)pMemFile->startPageId) * pMemFile->pageSize, SEEK_SET);
-  int32_t numWriteBytes = pMemFile->pageSize * (pMemFile->currPageId - pMemFile->startPageId + 1);
-  if (ret == 0) {
-    ret = fwrite(pMemFile->writePageBuf, numWriteBytes, 1, pMemFile->pTdFile);
-  }
-  if (ret != 1) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return terrno;
-  }
-  pMemFile->bDirty = false;
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t freeExtRowMemFileWriteBuf(SSortHandle* pHandle) {
-  SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
-
-  if (pMemFile == NULL) return TSDB_CODE_SUCCESS;
-
-  taosMemoryFree(pMemFile->writePageBuf);
-  pMemFile->writePageBuf = NULL;
+  pMemFile->currRegionOffset += blockLen;
+  pMemFile->bRegionDirty = true;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -1225,12 +1199,15 @@ static void appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSource
   char* pData = colDataGetData(pSrcTsCol, *rowIndex);
   colDataSetVal(pTsCol, pBlock->info.rows, pData, false);
 
-  SColumnInfoData* pPageIdCol = taosArrayGet(pBlock->pDataBlock, 1);
-  colDataSetInt32(pPageIdCol, pBlock->info.rows, &pageId);
+  SColumnInfoData* pRegionIdCol = taosArrayGet(pBlock->pDataBlock, 1);
+  colDataSetInt32(pRegionIdCol, pBlock->info.rows, &pageId);
 
   SColumnInfoData* pOffsetCol = taosArrayGet(pBlock->pDataBlock, 2);
   colDataSetInt32(pOffsetCol, pBlock->info.rows, &offset);
 
+  SColumnInfoData* pLengthCol = taosArrayGet(pBlock->pDataBlock, 3);
+  colDataSetInt32(pLengthCol, pBlock->info.rows, &length);
+
   pBlock->info.rows += 1;
   *rowIndex += 1;
 }
@@ -1240,10 +1217,12 @@ static void initRowIdSort(SSortHandle* pHandle) {
   SSDataBlock* pSortInput = createDataBlock();
   SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1);
   blockDataAppendColInfo(pSortInput, &tsCol);
-  SColumnInfoData pageIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
-  blockDataAppendColInfo(pSortInput, &pageIdCol);
+  SColumnInfoData regionIdCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
+  blockDataAppendColInfo(pSortInput, &regionIdCol);
   SColumnInfoData  offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
   blockDataAppendColInfo(pSortInput, &offsetCol);
+  SColumnInfoData  lengthCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 4);
+  blockDataAppendColInfo(pSortInput, &lengthCol);
 
   blockDataDestroy(pHandle->pDataBlock);
   pHandle->pDataBlock = pSortInput;
@@ -1267,9 +1246,8 @@ static void initRowIdSort(SSortHandle* pHandle) {
   return;
 }
 
-int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsPageSize, int32_t extRowsMemSize) {
+int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsMemSize) {
   pHandle->extRowBytes = blockDataGetRowSize(pHandle->pDataBlock) + taosArrayGetSize(pHandle->pDataBlock->pDataBlock) + sizeof(int32_t);
-  pHandle->extRowsPageSize = extRowsPageSize;
   pHandle->extRowsMemSize = extRowsMemSize;
   SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);
   pHandle->extRowsOrderInfo = *pOrder;
@@ -1552,7 +1530,6 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   while (1) {
     SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param);
 
-    int64_t p = taosGetTimestampUs();
     bool bExtractedBlock = false;
     bool bSkipBlock = false;
     if (pBlk != NULL && pHandle->mergeLimit > 0) {
@@ -1594,6 +1571,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
       tSimpleHashClear(mUidBlk);
 
       int64_t p = taosGetTimestampUs();
+      if (pHandle->bSortByRowId) {
+        tsortOpenRegion(pHandle);
+      }
       code = sortBlocksToExtSource(pHandle, aBlkSort, aExtSrc);
 
       if (code != TSDB_CODE_SUCCESS) {
@@ -1603,7 +1583,9 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
         taosArrayClear(aBlkSort);
         break;	
       }
-
+      if (pHandle->bSortByRowId) {
+        tsortCloseRegion(pHandle);
+      }
       int64_t el = taosGetTimestampUs() - p;
       pHandle->sortElapsed += el;
 
@@ -1641,8 +1623,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   taosArrayDestroy(aExtSrc);
   tSimpleHashCleanup(mTableNumRows);
   if (pHandle->bSortByRowId) {
-    saveDirtyPagesToExtRowsMemFile(pHandle);
-    freeExtRowMemFileWriteBuf(pHandle);
+    tsortFinalizeRegions(pHandle);
   }
   pHandle->type = SORT_SINGLESOURCE_SORT;
   return code;

From eba7b2fa82db2b1a93971990ff846597953a80e2 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 4 Mar 2024 15:58:25 +0800
Subject: [PATCH 136/201] fix(test/last_row): remove random columns' checking

---
 tests/script/tsim/parser/last_both.sim       |   2 +-
 tests/script/tsim/parser/last_both_query.sim | 190 +++++++++----------
 2 files changed, 96 insertions(+), 96 deletions(-)

diff --git a/tests/script/tsim/parser/last_both.sim b/tests/script/tsim/parser/last_both.sim
index 7b6c40c127..e01a966744 100644
--- a/tests/script/tsim/parser/last_both.sim
+++ b/tests/script/tsim/parser/last_both.sim
@@ -6,7 +6,7 @@ sql connect
 print ======================== dnode1 start
 $db = testdb
 sql drop database if exists $db
-sql create database $db cachemodel 'both' minrows 10 stt_trigger 1 
+sql create database $db cachemodel 'none' minrows 10 stt_trigger 1
 sql use $db
 
 sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int)
diff --git a/tests/script/tsim/parser/last_both_query.sim b/tests/script/tsim/parser/last_both_query.sim
index 775fd482ea..5f86412199 100644
--- a/tests/script/tsim/parser/last_both_query.sim
+++ b/tests/script/tsim/parser/last_both_query.sim
@@ -357,95 +357,95 @@ if $data45 != 5 then
   return -1
 endi
 
-#sql select last_row(*), id from st2 group by id order by id
-#print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
-#print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
-#print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
-#print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
-#print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49
-#
-#if $rows != 5 then
-#  return -1
-#endi
-#if $data00 != @21-05-12 10:10:12.000@ then
-#  return -1
-#endi
-#if $data01 != 6 then
-#  return -1
-#endi
-#if $data02 != NULL then
-#  print $data02
-#  return -1
-#endi
-#if $data03 != NULL then
-#  return -1
-#endi
-#if $data04 != @70-01-01 07:59:57.000@ then
-#  return -1
-#endi
-#if $data05 != 1 then
-#  return -1
-#endi
-#if $data10 != @21-05-11 10:12:23.000@ then
-#  return -1
-#endi
-#if $data11 != 22 then
-#  return -1
-#endi
-#if $data12 != 23.000000000 then
-#  print $data02
-#  return -1
-#endi
-#if $data13 != NULL then
-#  return -1
-#endi
-#if $data14 != @70-01-01 07:59:58.-04@ then
-#  return -1
-#endi
-#if $data15 != 2 then
-#  return -1
-#endi
-#if $data20 != @21-05-10 10:12:24.000@ then
-#  return -1
-#endi
-#if $data21 != 24 then
-#  return -1
-#endi
-#if $data22 != NULL then
-#  print expect NULL actual: $data22
-#  return -1
-#endi
-#if $data23 != 25 then
-#  return -1
-#endi
-#if $data24 != @70-01-01 07:59:57.-04@ then =
-#  return -1
-#endi
-#if $data25 != 3 then
-#  return -1
-#endi
-#if $data30 != @21-05-11 10:12:25.000@ then
-#  return -1
-#endi
-#if $data31 != 26 then
-#  return -1
-#endi
-#if $data32 != NULL then
-#  print $data02
-#  return -1
-#endi
-#if $data33 != 27 then
-#  return -1
-#endi
-#if $data34 != @70-01-01 07:59:56.-04@ then
-#  return -1
-#endi
-#if $data35 != 4 then
-#  return -1
-#endi
-#if $data40 != @21-05-11 10:12:29.000@ then
-#  return -1
-#endi
+sql select last_row(*), id from st2 group by id order by id
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49
+
+if $rows != 5 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != NULL then
+  print $data02
+  return -1
+endi
+if $data03 != NULL then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+if $data05 != 1 then
+  return -1
+endi
+if $data10 != @21-05-11 10:12:23.000@ then
+  return -1
+endi
+if $data11 != 22 then
+  return -1
+endi
+if $data12 != 23.000000000 then
+  print $data02
+  return -1
+endi
+if $data13 != NULL then
+  return -1
+endi
+if $data14 != @70-01-01 07:59:58.-04@ then
+  return -1
+endi
+if $data15 != 2 then
+  return -1
+endi
+if $data20 != @21-05-10 10:12:24.000@ then
+  return -1
+endi
+if $data21 != 24 then
+  return -1
+endi
+if $data22 != NULL then
+  print expect NULL actual: $data22
+  return -1
+endi
+if $data23 != 25 then
+  return -1
+endi
+if $data24 != @70-01-01 07:59:57.-04@ then =
+  return -1
+endi
+if $data25 != 3 then
+  return -1
+endi
+if $data30 != @21-05-11 10:12:25.000@ then
+  return -1
+endi
+if $data31 != 26 then
+  return -1
+endi
+if $data32 != NULL then
+  print $data02
+  return -1
+endi
+if $data33 != 27 then
+  return -1
+endi
+if $data34 != @70-01-01 07:59:56.-04@ then
+  return -1
+endi
+if $data35 != 4 then
+  return -1
+endi
+if $data40 != @21-05-11 10:12:29.000@ then
+  return -1
+endi
 #if $data41 != NULL then
 #  return -1
 #endi
@@ -453,15 +453,15 @@ endi
 #  print $data02
 #  return -1
 #endi
-#if $data43 != NULL then
-#  return -1
-#endi
+if $data43 != NULL then
+  return -1
+endi
 #if $data44 != NULL then
 #  return -1
 #endi
-#if $data45 != 5 then
-#  return -1
-#endi
+if $data45 != 5 then
+  return -1
+endi
 
 print "test tbn"
 sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp)

From aaa1c1d4a8dc2097058724f5a2a488d3b1392136 Mon Sep 17 00:00:00 2001
From: facetosea <25808407@qq.com>
Date: Mon, 4 Mar 2024 09:35:08 +0800
Subject: [PATCH 137/201] fix: memleak

---
 source/libs/executor/src/tsort.c | 38 ++++++++++++++------------------
 1 file changed, 16 insertions(+), 22 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 9ff903cdb9..10220426a3 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1199,6 +1199,18 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   return code;
 }
 
+static void freeSSortSource(SSortSource* source) {
+  if (NULL == source) return;
+  if (source->param && !source->onlyRef) {
+    taosMemoryFree(source->param);
+  }
+  if (!source->onlyRef && source->src.pBlock) {
+    blockDataDestroy(source->src.pBlock);
+    source->src.pBlock = NULL;
+  }
+  taosMemoryFree(source);
+}
+
 static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
   int32_t code = 0;
   size_t  sortBufSize = pHandle->numOfPages * pHandle->pageSize;
@@ -1231,14 +1243,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
 
     code = blockDataMerge(pHandle->pDataBlock, pBlock);
     if (code != TSDB_CODE_SUCCESS) {
-      if (source->param && !source->onlyRef) {
-        taosMemoryFree(source->param);
-      }
-      if (!source->onlyRef && source->src.pBlock) {
-        blockDataDestroy(source->src.pBlock);
-        source->src.pBlock = NULL;
-      }
-      taosMemoryFree(source);
+      freeSSortSource(source);
       return code;
     }
 
@@ -1248,15 +1253,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
       int64_t p = taosGetTimestampUs();
       code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
       if (code != 0) {
-        if (source->param && !source->onlyRef) {
-          taosMemoryFree(source->param);
-        }
-        if (!source->onlyRef && source->src.pBlock) {
-          blockDataDestroy(source->src.pBlock);
-          source->src.pBlock = NULL;
-        }
-
-        taosMemoryFree(source);
+        freeSSortSource(source);
         return code;
       }
 
@@ -1265,16 +1262,13 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
       if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows);
       code = doAddToBuf(pHandle->pDataBlock, pHandle);
       if (code != TSDB_CODE_SUCCESS) {
+        freeSSortSource(source);
         return code;
       }
     }
   }
 
-  if (source->param && !source->onlyRef) {
-    taosMemoryFree(source->param);
-  }
-
-  taosMemoryFree(source);
+  freeSSortSource(source);
 
   if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) {
     size_t size = blockDataGetSize(pHandle->pDataBlock);

From 80fd3e044546a16221816829e768ceaf095671cb Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 4 Mar 2024 19:31:47 +0800
Subject: [PATCH 138/201] fix(stream): remove related fill-history if task in
 stop status.

---
 include/libs/stream/tstream.h              |  2 +-
 source/dnode/vnode/src/tqCommon/tqCommon.c | 36 +++++++++++++++-------
 source/libs/stream/src/streamTask.c        | 14 +--------
 3 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index cf980400d6..0df6b76cf7 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -782,7 +782,7 @@ bool    streamTaskIsAllUpstreamClosed(SStreamTask* pTask);
 bool    streamTaskSetSchedStatusWait(SStreamTask* pTask);
 int8_t  streamTaskSetSchedStatusActive(SStreamTask* pTask);
 int8_t  streamTaskSetSchedStatusInactive(SStreamTask* pTask);
-int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt, bool metaLock);
+int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt);
 
 int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
 int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event);
diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index a2d45062b9..8c66ecc6a9 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -142,8 +142,10 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
   if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
     ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id));
     if (ppHTask == NULL || *ppHTask == NULL) {
-      tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already",
-              vgId, req.taskId);
+      tqError(
+          "vgId:%d failed to acquire fill-history task:0x%x when handling update, may have been dropped already, rel "
+          "stream task:0x%x",
+          vgId, (uint32_t)pTask->hTaskInfo.id.taskId, req.taskId);
       CLEAR_RELATED_FILLHISTORY_TASK(pTask);
     } else {
       tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr);
@@ -612,23 +614,35 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
 
 int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) {
   SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
+  int32_t              vgId = pMeta->vgId;
+  STaskId              hTaskId = {0};
 
-  int32_t vgId = pMeta->vgId;
   tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId);
 
-  SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
-  if (pTask != NULL) {
-    // drop the related fill-history task firstly
+  streamMetaWLock(pMeta);
+
+  STaskId       id = {.streamId = pReq->streamId, .taskId = pReq->taskId};
+  SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
+  if ((ppTask != NULL) && ((*ppTask) != NULL)) {
+    streamMetaAcquireOneTask(*ppTask);
+    SStreamTask* pTask = *ppTask;
+
     if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
-      STaskId* pHTaskId = &pTask->hTaskInfo.id;
-      streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId);
-      tqDebug("s-task:0x%x vgId:%d drop fill-history task:0x%x firstly", pReq->taskId, vgId,
-              (int32_t)pHTaskId->taskId);
+      hTaskId.streamId = pTask->hTaskInfo.id.streamId;
+      hTaskId.taskId = pTask->hTaskInfo.id.taskId;
+      streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt);
     }
+
     streamMetaReleaseTask(pMeta, pTask);
   }
 
-  streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt, true);
+  streamMetaWUnLock(pMeta);
+
+  // drop the related fill-history task firstly
+  if (hTaskId.taskId != 0 && hTaskId.streamId != 0) {
+    streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
+    tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
+  }
 
   // drop the stream task now
   streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId);
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 9639921c77..68b4ba2296 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -766,21 +766,13 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) {
   return status;
 }
 
-int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool metaLock) {
-  if (pTask == NULL) {
-    return TSDB_CODE_SUCCESS;
-  }
-
+int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) {
   SStreamMeta* pMeta = pTask->pMeta;
   STaskId      sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId};
   if (pTask->info.fillHistory == 0) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if (metaLock) {
-    streamMetaWLock(pMeta);
-  }
-
   SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId));
   if (ppStreamTask != NULL) {
     stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr,
@@ -798,10 +790,6 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool
     taosThreadMutexUnlock(&(*ppStreamTask)->lock);
   }
 
-  if (metaLock) {
-    streamMetaWUnLock(pMeta);
-  }
-
   return TSDB_CODE_SUCCESS;
 }
 

From ad1780dbdc8211a7fec7414ffecb904c6aa52a89 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 4 Mar 2024 19:32:22 +0800
Subject: [PATCH 139/201] fix(stream): reset the pData after transferring
 state.

---
 source/libs/stream/inc/streamInt.h  | 1 +
 source/libs/stream/src/streamTask.c | 4 +---
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h
index 87f63b48ed..d0055d5400 100644
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@@ -99,6 +99,7 @@ void    streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration);
 int32_t streamDispatchStreamBlock(SStreamTask* pTask);
 void    destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups);
 int32_t getNumOfDispatchBranch(SStreamTask* pTask);
+void    clearBufferedDispatchMsg(SStreamTask* pTask);
 
 int32_t           streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock);
 SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg);
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 68b4ba2296..8be5b94096 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -412,9 +412,7 @@ void tFreeStreamTask(SStreamTask* pTask) {
   pTask->pReadyMsgList = taosArrayDestroy(pTask->pReadyMsgList);
 
   if (pTask->msgInfo.pData != NULL) {
-    destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask));
-    pTask->msgInfo.pData = NULL;
-    pTask->msgInfo.dispatchMsgType = 0;
+    clearBufferedDispatchMsg(pTask);
   }
 
   if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) {

From a38161299b2f64db97459d8f38e572e4624fc4ad Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 4 Mar 2024 19:51:01 +0800
Subject: [PATCH 140/201] fix(stream): add clear msgData buf impl.

---
 source/libs/stream/src/streamDispatch.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index 9702096b60..dc790b5b2d 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -315,6 +315,16 @@ int32_t getNumOfDispatchBranch(SStreamTask* pTask) {
              : taosArrayGetSize(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos);
 }
 
+void clearBufferedDispatchMsg(SStreamTask* pTask) {
+  SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo;
+  if (pMsgInfo->pData != NULL) {
+    destroyDispatchMsg(pMsgInfo->pData, getNumOfDispatchBranch(pTask));
+  }
+
+  pMsgInfo->pData = NULL;
+  pMsgInfo->dispatchMsgType = 0;
+}
+
 static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pData) {
   int32_t code = 0;
   int32_t numOfBlocks = taosArrayGetSize(pData->blocks);
@@ -678,8 +688,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
     // todo deal with only partially success dispatch case
     atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0);
     if (terrno == TSDB_CODE_APP_IS_STOPPING) {  // in case of this error, do not retry anymore
-      destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask));
-      pTask->msgInfo.pData = NULL;
+      clearBufferedDispatchMsg(pTask);
       return code;
     }
 
@@ -938,15 +947,12 @@ void streamClearChkptReadyMsg(SStreamTask* pTask) {
 // this message has been sent successfully, let's try next one.
 static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId) {
   stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData);
-  destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask));
-
   bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER);
   if (delayDispatch) {
     pTask->chkInfo.dispatchCheckpointTrigger = true;
   }
 
-  pTask->msgInfo.pData = NULL;
-  pTask->msgInfo.dispatchMsgType = 0;
+  clearBufferedDispatchMsg(pTask);
 
   int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs;
 
@@ -1086,7 +1092,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
     } else {  // this message has been sent successfully, let's try next one.
       pTask->msgInfo.retryCount = 0;
 
-      // transtate msg has been sent to downstream successfully. let's transfer the fill-history task state
+      // trans-state msg has been sent to downstream successfully. let's transfer the fill-history task state
       if (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__TRANS_STATE) {
         stDebug("s-task:%s dispatch transtate msgId:%d to downstream successfully, start to transfer state", id, msgId);
         ASSERT(pTask->info.fillHistory == 1);
@@ -1095,6 +1101,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
         if (code != TSDB_CODE_SUCCESS) {  // todo: do nothing if error happens
         }
 
+        clearBufferedDispatchMsg(pTask);
+
         // now ready for next data output
         atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL);
       } else {

From 09b991d97ad6672a215fd345b08aa414f254a1fc Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 4 Mar 2024 12:24:00 +0000
Subject: [PATCH 141/201] fix mem leak

---
 source/libs/transport/src/transSvr.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 5a1ef31b7d..f47a688e6f 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -159,7 +159,7 @@ static void uvStartSendResp(SSvrMsg* msg);
 
 static void uvNotifyLinkBrokenToApp(SSvrConn* conn);
 
-static FORCE_INLINE void destroySmsg(SSvrMsg* smsg);
+static FORCE_INLINE void      destroySmsg(SSvrMsg* smsg);
 static FORCE_INLINE SSvrConn* createConn(void* hThrd);
 static FORCE_INLINE void      destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
 static FORCE_INLINE void      destroyConnRegArg(SSvrConn* conn);
@@ -527,6 +527,10 @@ void uvOnSendCb(uv_write_t* req, int status) {
       if (!transQueueEmpty(&conn->srvMsgs)) {
         msg = (SSvrMsg*)transQueueGet(&conn->srvMsgs, 0);
         if (msg->type == Register && conn->status == ConnAcquire) {
+          if (conn->regArg.init) {
+            transFreeMsg(conn->regArg.msg.pCont);
+            conn->regArg.init = 0;
+          }
           conn->regArg.notifyCount = 0;
           conn->regArg.init = 1;
           conn->regArg.msg = msg->msg;
@@ -1350,6 +1354,11 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) {
       return;
     }
     transQueuePop(&conn->srvMsgs);
+
+    if (conn->regArg.init) {
+      transFreeMsg(conn->regArg.msg.pCont);
+      conn->regArg.init = 0;
+    }
     conn->regArg.notifyCount = 0;
     conn->regArg.init = 1;
     conn->regArg.msg = msg->msg;

From 2ac803bf3c5975185f25be9fb62350dc945ce935 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Mon, 4 Mar 2024 22:16:41 +0800
Subject: [PATCH 142/201] fix(stream): clear and set task status.

---
 source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index 8c66ecc6a9..6a5bd444ef 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -630,9 +630,9 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
     if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
       hTaskId.streamId = pTask->hTaskInfo.id.streamId;
       hTaskId.taskId = pTask->hTaskInfo.id.taskId;
-      streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt);
     }
 
+    streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt);
     streamMetaReleaseTask(pMeta, pTask);
   }
 

From af6d6b69c3380385a6effe88d0c557cc4735e693 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 5 Mar 2024 11:25:43 +0800
Subject: [PATCH 143/201] fix:add excluded msg for delete in tmq

---
 include/common/tmsg.h                   | 1 -
 include/libs/scheduler/scheduler.h      | 1 +
 source/client/inc/clientInt.h           | 5 +++--
 source/client/src/clientImpl.c          | 9 ++++++---
 source/client/src/clientMain.c          | 6 +++---
 source/client/src/clientRawBlockWrite.c | 4 +---
 source/common/src/tmsg.c                | 1 -
 source/libs/scheduler/inc/schInt.h      | 1 +
 source/libs/scheduler/src/schJob.c      | 1 +
 source/libs/scheduler/src/schRemote.c   | 2 +-
 10 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index e400698f69..4fa244a07c 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -3816,7 +3816,6 @@ typedef struct {
   int8_t   source;
 } SVDeleteReq;
 
-extern int8_t deleteFromTaosx;
 int32_t tSerializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq);
 int32_t tDeserializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq);
 
diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h
index 958d63349d..952af3c443 100644
--- a/include/libs/scheduler/scheduler.h
+++ b/include/libs/scheduler/scheduler.h
@@ -78,6 +78,7 @@ typedef struct SSchedulerReq {
   void*              chkKillParam;
   SExecResult*       pExecRes;
   void**             pFetchRes;
+  int8_t             source;
 } SSchedulerReq;
 
 int32_t schedulerInit(void);
diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index f73d121b15..6cc2a62236 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -284,6 +284,7 @@ typedef struct SRequestObj {
   void*                pWrapper;
   SMetaData            parseMeta;
   char*                effectiveUser;
+  int8_t               source;
 } SRequestObj;
 
 typedef struct SSyncQueryParam {
@@ -306,10 +307,10 @@ void    doFreeReqResultInfo(SReqResultInfo* pResInfo);
 int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq);
 void    syncCatalogFn(SMetaData* pResult, void* param, int32_t code);
 
-TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly);
+TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source);
 TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, int64_t reqid);
 
-void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly);
+void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source);
 void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly,
                                  int64_t reqid);
 void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param);
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 971ed407f9..aa18ef1c94 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -729,6 +729,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
          .chkKillFp = chkRequestKilled,
          .chkKillParam = (void*)pRequest->self,
          .pExecRes = &res,
+         .source = pRequest->source,
   };
 
   int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob);
@@ -1198,6 +1199,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
            .chkKillFp = chkRequestKilled,
            .chkKillParam = (void*)pRequest->self,
            .pExecRes = NULL,
+           .source = pRequest->source,
     };
     code = schedulerExecJob(&req, &pRequest->body.queryJob);
     taosArrayDestroy(pNodeList);
@@ -2461,7 +2463,7 @@ void syncQueryFn(void* param, void* res, int32_t code) {
   tsem_post(&pParam->sem);
 }
 
-void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) {
+void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source) {
   if (sql == NULL || NULL == fp) {
     terrno = TSDB_CODE_INVALID_PARA;
     if (fp) {
@@ -2487,6 +2489,7 @@ void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp,
     return;
   }
 
+  pRequest->source = source;
   pRequest->body.queryFp = fp;
   doAsyncQuery(pRequest, false);
 }
@@ -2521,7 +2524,7 @@ void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_
   doAsyncQuery(pRequest, false);
 }
 
-TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
+TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source) {
   if (NULL == taos) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     return NULL;
@@ -2536,7 +2539,7 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
   }
   tsem_init(&param->sem, 0, 0);
 
-  taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
+  taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, source);
   tsem_wait(&param->sem);
 
   SRequestObj* pRequest = NULL;
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index f6aef5aa26..eacad0f18f 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -401,7 +401,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
   return pResInfo->userFields;
 }
 
-TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false); }
+TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false, TD_REQ_FROM_APP); }
 TAOS_RES *taos_query_with_reqid(TAOS *taos, const char *sql, int64_t reqid) {
   return taosQueryImplWithReqid(taos, sql, false, reqid);
 }
@@ -827,7 +827,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
 }
 
 int taos_validate_sql(TAOS *taos, const char *sql) {
-  TAOS_RES *pObj = taosQueryImpl(taos, sql, true);
+  TAOS_RES *pObj = taosQueryImpl(taos, sql, true, TD_REQ_FROM_APP);
 
   int code = taos_errno(pObj);
 
@@ -1125,7 +1125,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
 void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
   int64_t connId = *(int64_t *)taos;
   tscDebug("taos_query_a start with sql:%s", sql);
-  taosAsyncQueryImpl(connId, sql, fp, param, false);
+  taosAsyncQueryImpl(connId, sql, fp, param, false, TD_REQ_FROM_APP);
   tscDebug("taos_query_a end with sql:%s", sql);
 }
 
diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c
index 3857e47f57..adc8c361cd 100644
--- a/source/client/src/clientRawBlockWrite.c
+++ b/source/client/src/clientRawBlockWrite.c
@@ -1256,9 +1256,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
   snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName,
            req.tsColName, req.skey, req.tsColName, req.ekey);
 
-  deleteFromTaosx = TD_REQ_FROM_TAOX;
-  TAOS_RES*    res = taos_query(taos, sql);
-  deleteFromTaosx = TD_REQ_FROM_APP;
+  TAOS_RES*    res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX);
   SRequestObj* pRequest = (SRequestObj*)res;
   code = pRequest->code;
   if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_PAR_GET_META_ERROR) {
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index e7414c6004..467300352c 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -7148,7 +7148,6 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
   return 0;
 }
 
-int8_t deleteFromTaosx = TD_REQ_FROM_APP;
 int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
   int32_t headLen = sizeof(SMsgHead);
   if (buf != NULL) {
diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h
index 1c90e61ea3..276b668717 100644
--- a/source/libs/scheduler/inc/schInt.h
+++ b/source/libs/scheduler/inc/schInt.h
@@ -304,6 +304,7 @@ typedef struct SSchJob {
   SSchResInfo          userRes;
   char                *sql;
   SQueryProfileSummary summary;
+  int8_t               source;
 } SSchJob;
 
 typedef struct SSchTaskCtx {
diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c
index e50ec64d54..48aab63ba3 100644
--- a/source/libs/scheduler/src/schJob.c
+++ b/source/libs/scheduler/src/schJob.c
@@ -746,6 +746,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
   pJob->chkKillParam = pReq->chkKillParam;
   pJob->userRes.execFp = pReq->execFp;
   pJob->userRes.cbParam = pReq->cbParam;
+  pJob->source = pReq->source;
 
   if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) {
     qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId);
diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c
index f6465316d2..39273ffa50 100644
--- a/source/libs/scheduler/src/schRemote.c
+++ b/source/libs/scheduler/src/schRemote.c
@@ -1086,7 +1086,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
       req.sqlLen = strlen(pJob->sql);
       req.sql = (char *)pJob->sql;
       req.msg = pTask->msg;
-      req.source = deleteFromTaosx;
+      req.source = pJob->source;
       msgSize = tSerializeSVDeleteReq(NULL, 0, &req);
       msg = taosMemoryCalloc(1, msgSize);
       if (NULL == msg) {

From cdfbbc298535858df1a8a65de45803cee3855b6e Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 5 Mar 2024 13:28:34 +0800
Subject: [PATCH 144/201] fix(stream): set the dest epset from mnode, and set
 the retry error code for trans.

---
 source/dnode/mnode/impl/inc/mndStream.h     |  2 +-
 source/dnode/mnode/impl/src/mndStream.c     |  2 +-
 source/dnode/mnode/impl/src/mndStreamUtil.c | 16 ++++++++++++----
 source/libs/stream/src/streamTask.c         |  2 +-
 4 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h
index 57fd187da3..5307ff4b05 100644
--- a/source/dnode/mnode/impl/inc/mndStream.h
+++ b/source/dnode/mnode/impl/inc/mndStream.h
@@ -111,7 +111,7 @@ STrans  *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const
 int32_t  mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
 SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
 void     killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo);
-int32_t  mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
+int32_t  mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
 
 SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId);
 int32_t     extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId);
diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index 3ef2f64df7..fca757006d 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -1811,7 +1811,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
     mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid,
            pStream->name, pTrans->id);
 
-    int32_t code = mndStreamSetUpdateEpsetAction(pStream, pChangeInfo, pTrans);
+    int32_t code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans);
 
     // todo: not continue, drop all and retry again
     if (code != TSDB_CODE_SUCCESS) {
diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c
index 1ae85a2cc6..2b8fcee9fd 100644
--- a/source/dnode/mnode/impl/src/mndStreamUtil.c
+++ b/source/dnode/mnode/impl/src/mndStreamUtil.c
@@ -462,14 +462,22 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) {
+static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) {
   void   *pBuf = NULL;
   int32_t len = 0;
   streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
 
   doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
 
-  int32_t code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0);
+  SEpSet  epset = {0};
+  bool    hasEpset = false;
+  int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
+  if (code != TSDB_CODE_SUCCESS || !hasEpset) {
+    terrno = code;
+    return code;
+  }
+
+  code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID);
   if (code != TSDB_CODE_SUCCESS) {
     taosMemoryFree(pBuf);
   }
@@ -478,14 +486,14 @@ static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroup
 }
 
 // build trans to update the epset
-int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
+int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
   mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid);
   taosWLockLatch(&pStream->lock);
 
   SStreamTaskIter *pIter = createStreamTaskIter(pStream);
   while (streamTaskIterNextTask(pIter)) {
     SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
-    int32_t      code = doSetUpdateTaskAction(pTrans, pTask, pInfo);
+    int32_t      code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo);
     if (code != TSDB_CODE_SUCCESS) {
       destroyStreamTaskIter(pIter);
       taosWUnLockLatch(&pStream->lock);
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 8be5b94096..6fd96f651e 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -39,7 +39,7 @@ static int32_t doUpdateTaskEpset(SStreamTask* pTask, int32_t nodeId, SEpSet* pEp
     stDebug("s-task:0x%x (vgId:%d) self node epset is updated %s", pTask->id.taskId, nodeId, buf);
   }
 
-  // check for the dispath info and the upstream task info
+  // check for the dispatch info and the upstream task info
   int32_t level = pTask->info.taskLevel;
   if (level == TASK_LEVEL__SOURCE) {
     streamTaskUpdateDownstreamInfo(pTask, nodeId, pEpSet);

From 254afc070c8050cfd3fb5f8fddff4eff7841ffec Mon Sep 17 00:00:00 2001
From: Shungang Li <sgli@taosdata.com>
Date: Tue, 5 Mar 2024 15:20:11 +0800
Subject: [PATCH 145/201] fix: dynmaic config supportvnodes in ent

---
 source/common/src/tglobal.c            |  2 +-
 source/dnode/mnode/impl/src/mndDnode.c | 28 +++++---------------------
 2 files changed, 6 insertions(+), 24 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 55df80ca44..fde8313228 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -586,7 +586,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   tsNumOfSupportVnodes = tsNumOfCores * 2;
   tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
-  if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
   if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c
index 79a5f5fd83..befb6d3521 100644
--- a/source/dnode/mnode/impl/src/mndDnode.c
+++ b/source/dnode/mnode/impl/src/mndDnode.c
@@ -610,7 +610,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
         for(int32_t j = 0; j < tagSize; j++){
           SJson* item = tjsonGetArrayItem(arrayTag, j);
 
-          *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); 
+          *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN);
           tjsonGetStringValue(item, "name", *(labels + j));
 
           *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN);
@@ -626,7 +626,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
         for(int32_t j = 0; j < metricLen; j++){
           SJson *item = tjsonGetArrayItem(metrics, j);
 
-          char name[MONITOR_METRIC_NAME_LEN] = {0}; 
+          char name[MONITOR_METRIC_NAME_LEN] = {0};
           tjsonGetStringValue(item, "name", name);
 
           double value = 0;
@@ -636,7 +636,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
           tjsonGetDoubleValue(item, "type", &type);
 
           int32_t metricNameLen = strlen(name) + strlen(tableName) + 2;
-          char* metricName = taosMemoryMalloc(metricNameLen); 
+          char* metricName = taosMemoryMalloc(metricNameLen);
           memset(metricName, 0, metricNameLen);
           sprintf(metricName, "%s:%s", tableName, name);
 
@@ -669,7 +669,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
           else{
             mTrace("get metric from registry:%p", metric);
           }
-          
+
           if(type == 0){
             taos_counter_add(metric, value, (const char**)sample_labels);
           }
@@ -689,7 +689,7 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
         taosMemoryFreeClear(labels);
       }
     }
-    
+
   }
 
   code = 0;
@@ -1409,24 +1409,6 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
   if (strcasecmp(cfgReq.config, "resetlog") == 0) {
     strcpy(dcfgReq.config, "resetlog");
 #ifdef TD_ENTERPRISE
-  } else if (strncasecmp(cfgReq.config, "supportvnodes", 13) == 0) {
-    int32_t optLen = strlen("supportvnodes");
-    int32_t flag = -1;
-    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
-    if (code < 0) return code;
-
-    if (flag < 0 || flag > 4096) {
-      mError("dnode:%d, failed to config supportVnodes since value:%d. Valid range: [0, 4096]", cfgReq.dnodeId, flag);
-      terrno = TSDB_CODE_OUT_OF_RANGE;
-      goto _err_out;
-    }
-    if (flag == 0) {
-      flag = tsNumOfCores * 2;
-    }
-    flag = TMAX(flag, 2);
-
-    strcpy(dcfgReq.config, "supportvnodes");
-    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
   } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) {
     int32_t optLen = strlen("s3blocksize");
     int32_t flag = -1;

From 35c5dca341820f37ac9d519ddbd66651bd757f4a Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 5 Mar 2024 07:42:29 +0000
Subject: [PATCH 146/201] fix count error

---
 source/libs/transport/src/transCli.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index 6ae72eac14..2e35aecd08 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -584,8 +584,8 @@ void* destroyConnPool(SCliThrd* pThrd) {
 
 static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) {
   void*      pool = pThrd->pool;
-  SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1);
   STrans*    pTranInst = pThrd->pTransInst;
+  SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1);
   if (plist == NULL) {
     SConnList list = {0};
     taosHashPut((SHashObj*)pool, key, strlen(key) + 1, (void*)&list, sizeof(list));
@@ -867,17 +867,18 @@ static void cliDestroyConn(SCliConn* conn, bool clear) {
   QUEUE_INIT(&conn->q);
 
   conn->broken = true;
+  if (conn->list == NULL) {
+    conn->list = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr));
+  }
 
-  if (conn->list != NULL) {
-    SConnList* connList = conn->list;
-    connList->list->numOfConn--;
-    connList->size--;
-  } else {
-    if (pThrd->pool) {
-      SConnList* connList = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr) + 1);
-      if (connList != NULL) connList->list->numOfConn--;
+  if (conn->list) {
+    SConnList* list = conn->list;
+    list->list->numOfConn--;
+    if (conn->status == ConnInPool) {
+      list->size--;
     }
   }
+
   conn->list = NULL;
   pThrd->newConnCount--;
 

From 45e779d464cf41fb60aa7e51bf940d6255c0171f Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Tue, 5 Mar 2024 15:48:58 +0800
Subject: [PATCH 147/201] refactor: do some internal refactor.

---
 source/libs/stream/src/streamTask.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index 6fd96f651e..4dc01ee9b4 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -622,6 +622,7 @@ void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDo
 void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet) {
   char buf[512] = {0};
   EPSET_TO_STR(pEpSet, buf);
+  int32_t id = pTask->id.taskId;
 
   int8_t type = pTask->outputInfo.type;
   if (type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
@@ -633,8 +634,8 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE
 
       if (pVgInfo->vgId == nodeId) {
         epsetAssign(&pVgInfo->epSet, pEpSet);
-        stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", pTask->id.taskId,
-                pVgInfo->taskId, nodeId, buf);
+        stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pVgInfo->taskId, nodeId,
+                buf);
         break;
       }
     }
@@ -642,8 +643,8 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE
     STaskDispatcherFixed* pDispatcher = &pTask->outputInfo.fixedDispatcher;
     if (pDispatcher->nodeId == nodeId) {
       epsetAssign(&pDispatcher->epSet, pEpSet);
-      stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpSet:%s", pTask->id.taskId,
-              pDispatcher->taskId, nodeId, buf);
+      stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pDispatcher->taskId, nodeId,
+              buf);
     }
   } else {
     // do nothing

From 4d1b556bc50d51584464042d4dbad9b8d8fae1d5 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 5 Mar 2024 18:54:07 +0800
Subject: [PATCH 148/201] load operator checkpoint

---
 source/libs/executor/src/streamcountwindowoperator.c | 4 ++--
 source/libs/executor/src/streameventwindowoperator.c | 4 ++--
 source/libs/executor/src/streamtimewindowoperator.c  | 8 ++++----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c
index f9c7b51316..d9ce10d38a 100644
--- a/source/libs/executor/src/streamcountwindowoperator.c
+++ b/source/libs/executor/src/streamcountwindowoperator.c
@@ -694,6 +694,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->recvGetAll = false;
 
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT;
+  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, true,
+                  OP_NOT_OPENED, pInfo, pTaskInfo);
   // for stream
   void*   buff = NULL;
   int32_t len = 0;
@@ -704,8 +706,6 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
     doStreamCountDecodeOpState(buff, len, pOperator, true);
     taosMemoryFree(buff);
   }
-  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, true,
-                  OP_NOT_OPENED, pInfo, pTaskInfo);
   pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAgg, NULL, destroyStreamCountAggOperatorInfo,
                                          optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
   setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState);
diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c
index 4a2fe2416f..10b4a5c344 100644
--- a/source/libs/executor/src/streameventwindowoperator.c
+++ b/source/libs/executor/src/streameventwindowoperator.c
@@ -735,6 +735,8 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->reCkBlock = false;
   pInfo->recvGetAll = false;
 
+  setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED,
+                  pInfo, pTaskInfo);
   // for stream
   void*   buff = NULL;
   int32_t len = 0;
@@ -746,8 +748,6 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
     taosMemoryFree(buff);
   }
 
-  setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED,
-                  pInfo, pTaskInfo);
   pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo,
                                          optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
   setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c
index 2838d005ab..00877ae1b1 100644
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@@ -2992,6 +2992,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   pInfo->recvGetAll = false;
 
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
+  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
+                  OP_NOT_OPENED, pInfo, pTaskInfo);
   // for stream
   void*   buff = NULL;
   int32_t len = 0;
@@ -3002,8 +3004,6 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
     doStreamSessionDecodeOpState(buff, len, pOperator, true);
     taosMemoryFree(buff);
   }
-  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
-                  OP_NOT_OPENED, pInfo, pTaskInfo);
   pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo,
                                          optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
   setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
@@ -3873,6 +3873,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
   pInfo->recvGetAll = false;
 
+  setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
+                  pInfo, pTaskInfo);
   // for stream
   void*   buff = NULL;
   int32_t len = 0;
@@ -3884,8 +3886,6 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
     taosMemoryFree(buff);
   }
 
-  setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
-                  pInfo, pTaskInfo);
   pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo,
                                          optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
   setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);

From 57baceeeaf2bc939f1f2e743c05140c005878d90 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 5 Mar 2024 19:14:20 +0800
Subject: [PATCH 149/201] load operator checkpoint

---
 source/libs/executor/src/streamcountwindowoperator.c | 8 +++++---
 source/libs/executor/src/streamtimewindowoperator.c  | 1 -
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c
index d9ce10d38a..720734431f 100644
--- a/source/libs/executor/src/streamcountwindowoperator.c
+++ b/source/libs/executor/src/streamcountwindowoperator.c
@@ -390,10 +390,12 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
   buf = taosDecodeFixedI32(buf, &mapSize);
   for (int32_t i = 0; i < mapSize; i++) {
     SSessionKey       key = {0};
-    SResultWindowInfo winfo = {0};
+    SCountWindowInfo curWin = {0};
     buf = decodeSSessionKey(buf, &key);
-    buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
-    tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
+    SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
+    setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
+    buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
+    tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
   }
 
   // 2.twAggSup
diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c
index 00877ae1b1..3b62cc80e3 100644
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@@ -2533,7 +2533,6 @@ int32_t encodeSResultWindowInfo(void** buf, SResultWindowInfo* key, int32_t outL
 
 void* decodeSResultWindowInfo(void* buf, SResultWindowInfo* key, int32_t outLen) {
   buf = taosDecodeFixedBool(buf, &key->isOutput);
-  key->pStatePos->pRowBuff = NULL;
   buf = decodeSSessionKey(buf, &key->sessionWin);
   return buf;
 }

From bd430a031ab147f3759e9f02bd327dabd4979af4 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 5 Mar 2024 19:37:54 +0800
Subject: [PATCH 150/201] load operator checkpoint

---
 source/libs/executor/src/streameventwindowoperator.c | 3 +++
 source/libs/executor/src/streamtimewindowoperator.c  | 7 +++++++
 2 files changed, 10 insertions(+)

diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c
index 10b4a5c344..11d8d1487a 100644
--- a/source/libs/executor/src/streameventwindowoperator.c
+++ b/source/libs/executor/src/streameventwindowoperator.c
@@ -406,6 +406,7 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
   if (!pInfo) {
     return buf;
   }
+  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
 
   // 4.checksum
   int32_t dataLen = len - sizeof(uint32_t);
@@ -423,6 +424,8 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
     SSessionKey       key = {0};
     SResultWindowInfo winfo = {0};
     buf = decodeSSessionKey(buf, &key);
+    pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap,
+                                                        (void**)&winfo.pStatePos, &pAggSup->resultRowSize);
     buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
     tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
   }
diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c
index 3b62cc80e3..7a1bb2729c 100644
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@@ -2590,6 +2590,7 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
   if (!pInfo) {
     return buf;
   }
+  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
 
   // 5.checksum
   if (isParent) {
@@ -2608,6 +2609,8 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
     SSessionKey       key = {0};
     SResultWindowInfo winfo = {0};
     buf = decodeSSessionKey(buf, &key);
+    pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap,
+                                                        (void**)&winfo.pStatePos, &pAggSup->resultRowSize);
     buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
     tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
   }
@@ -3537,6 +3540,7 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
   if (!pInfo) {
     return buf;
   }
+  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
 
   // 5.checksum
   if (isParent) {
@@ -3555,6 +3559,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
     SSessionKey       key = {0};
     SResultWindowInfo winfo = {0};
     buf = decodeSSessionKey(buf, &key);
+    pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL,
+                                                      pAggSup->stateKeySize, compareStateKey,
+                                                      (void**)&winfo.pStatePos, &pAggSup->resultRowSize);
     buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
     tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
   }

From 7d3bfb0da91abe6236df97d8cdc8d8438a3467ad Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 5 Mar 2024 12:05:31 +0000
Subject: [PATCH 151/201] fix mem leak

---
 source/libs/scheduler/src/schRemote.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c
index 1c0b31109e..be21858e6a 100644
--- a/source/libs/scheduler/src/schRemote.c
+++ b/source/libs/scheduler/src/schRemote.c
@@ -940,6 +940,10 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery
   SCH_ERR_JRET(schGenerateCallBackInfo(pJob, pTask, msg, msgSize, msgType, trans, isHb, &pMsgSendInfo));
   SCH_ERR_JRET(schUpdateSendTargetInfo(pMsgSendInfo, addr, pTask));
 
+  if (isHb && persistHandle && trans->pHandle == 0) {
+    trans->pHandle = rpcAllocHandle();
+  } 
+
   if (pJob && pTask) {
     SCH_TASK_DLOG("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId,
            epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle);

From 3416d684335b92c67f32f0057233ce58a5a004c7 Mon Sep 17 00:00:00 2001
From: yihaoDeng <yhdeng@taosdata.com>
Date: Tue, 5 Mar 2024 21:02:20 +0800
Subject: [PATCH 152/201] add coverity scan

---
 source/libs/stream/src/streamBackendRocksdb.c |  2 +
 source/libs/stream/test/backendTest.cpp       | 56 +++++++++++++------
 2 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 910fd93989..5dafad8951 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -906,6 +906,7 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) {
   return 0;
 }
 
+#ifdef BUILD_NO_CALL
 static int32_t chkpIdComp(const void* a, const void* b) {
   int64_t x = *(int64_t*)a;
   int64_t y = *(int64_t*)b;
@@ -964,6 +965,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) {
   taosMemoryFree(chkpPath);
   return 0;
 }
+#endif
 
 #ifdef BUILD_NO_CALL
 int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) {
diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp
index a949748eb5..c9b981e5f9 100644
--- a/source/libs/stream/test/backendTest.cpp
+++ b/source/libs/stream/test/backendTest.cpp
@@ -29,7 +29,7 @@ class BackendEnv : public ::testing::Test {
 
 void *backendCreate() {
   const char *streamPath = "/tmp";
-  void       *p = NULL;
+  void *      p = NULL;
 
   // char *absPath = NULL;
   // // SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(streamPath, -1, 2);
@@ -52,7 +52,7 @@ SStreamState *stateCreate(const char *path) {
 }
 void *backendOpen() {
   streamMetaInit();
-  const char   *path = "/tmp/backend";
+  const char *  path = "/tmp/backend";
   SStreamState *p = stateCreate(path);
   ASSERT(p != NULL);
 
@@ -79,7 +79,7 @@ void *backendOpen() {
 
     const char *val = "value data";
     int32_t     len = 0;
-    char       *newVal = NULL;
+    char *      newVal = NULL;
     streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
     ASSERT(len == strlen(val));
   }
@@ -100,7 +100,7 @@ void *backendOpen() {
 
     const char *val = "value data";
     int32_t     len = 0;
-    char       *newVal = NULL;
+    char *      newVal = NULL;
     int32_t     code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
     ASSERT(code != 0);
   }
@@ -130,7 +130,7 @@ void *backendOpen() {
 
   winkey.groupId = 0;
   winkey.ts = tsArray[0];
-  char   *val = NULL;
+  char *  val = NULL;
   int32_t len = 0;
 
   pCurr = streamStateSeekKeyNext_rocksdb(p, &winkey);
@@ -157,7 +157,7 @@ void *backendOpen() {
     key.ts = tsArray[i];
     key.exprIdx = i;
 
-    char   *val = NULL;
+    char *  val = NULL;
     int32_t len = 0;
     streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len);
     ASSERT(len == strlen("Value"));
@@ -168,7 +168,7 @@ void *backendOpen() {
     key.ts = tsArray[i];
     key.exprIdx = i;
 
-    char   *val = NULL;
+    char *  val = NULL;
     int32_t len = 0;
     streamStateFuncDel_rocksdb(p, &key);
   }
@@ -213,7 +213,7 @@ void *backendOpen() {
   {
     SSessionKey key;
     memset(&key, 0, sizeof(key));
-    char   *val = NULL;
+    char *  val = NULL;
     int32_t vlen = 0;
     code = streamStateSessionGetKVByCur_rocksdb(pCurr, &key, (void **)&val, &vlen);
     ASSERT(code == 0);
@@ -260,7 +260,7 @@ void *backendOpen() {
     SWinKey key = {0};  // {.groupId = (uint64_t)(i), .ts = tsArray[i]};
     key.groupId = (uint64_t)(i);
     key.ts = tsArray[i];
-    char   *val = NULL;
+    char *  val = NULL;
     int32_t vlen = 0;
     ASSERT(streamStateFillGet_rocksdb(p, &key, (void **)&val, &vlen) == 0);
     taosMemoryFreeClear(val);
@@ -272,7 +272,7 @@ void *backendOpen() {
     SStreamStateCur *pCurr = streamStateFillGetCur_rocksdb(p, &key);
     ASSERT(pCurr != NULL);
 
-    char   *val = NULL;
+    char *  val = NULL;
     int32_t vlen = 0;
     ASSERT(0 == streamStateFillGetKVByCur_rocksdb(pCurr, &key, (const void **)&val, &vlen));
     ASSERT(vlen == strlen("Value"));
@@ -296,7 +296,7 @@ void *backendOpen() {
     SWinKey key = {0};  // {.groupId = (uint64_t)(i), .ts = tsArray[i]};
     key.groupId = (uint64_t)(i);
     key.ts = tsArray[i];
-    char   *val = NULL;
+    char *  val = NULL;
     int32_t vlen = 0;
     ASSERT(streamStateFillDel_rocksdb(p, &key) == 0);
     taosMemoryFreeClear(val);
@@ -338,7 +338,7 @@ void *backendOpen() {
     char key[128] = {0};
     sprintf(key, "tbname_%d", i);
 
-    char   *val = NULL;
+    char *  val = NULL;
     int32_t len = 0;
     code = streamDefaultGet_rocksdb(p, key, (void **)&val, &len);
     ASSERT(code == 0);
@@ -354,7 +354,7 @@ TEST_F(BackendEnv, checkOpen) {
   SStreamState *p = (SStreamState *)backendOpen();
   int64_t       tsStart = taosGetTimestampMs();
   {
-    void   *pBatch = streamStateCreateBatch();
+    void *  pBatch = streamStateCreateBatch();
     int32_t size = 0;
     for (int i = 0; i < size; i++) {
       char key[128] = {0};
@@ -368,7 +368,7 @@ TEST_F(BackendEnv, checkOpen) {
     streamStateDestroyBatch(pBatch);
   }
   {
-    void   *pBatch = streamStateCreateBatch();
+    void *  pBatch = streamStateCreateBatch();
     int32_t size = 0;
     char    valBuf[256] = {0};
     for (int i = 0; i < size; i++) {
@@ -385,7 +385,7 @@ TEST_F(BackendEnv, checkOpen) {
   // do checkpoint 2
   taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2);
   {
-    void   *pBatch = streamStateCreateBatch();
+    void *  pBatch = streamStateCreateBatch();
     int32_t size = 0;
     char    valBuf[256] = {0};
     for (int i = 0; i < size; i++) {
@@ -407,12 +407,22 @@ TEST_F(BackendEnv, checkOpen) {
   // taosMkDir(dump);
   taosMulMkDir(dump);
   SBkdMgt *mgt = bkdMgtCreate((char *)path);
-  SArray  *result = taosArrayInit(4, sizeof(void *));
+  SArray * result = taosArrayInit(4, sizeof(void *));
   bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump);
 
+  taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 4);
+
+  taosArrayClear(result);
+  bkdMgtGetDelta(mgt, p->pTdbState->idstr, 4, result, (char *)dump);
   bkdMgtDestroy(mgt);
   streamStateClose((SStreamState *)p, true);
+  // {
+  //   taosRemoveDir("/tmp/backend");
+  //   const char *  path = "/tmp/backend";
+  //   SStreamState *p = stateCreate(path);
+  // }
   taosRemoveDir(path);
+  // streamStateClose((SStreamState *)p, true);
 }
 
 TEST_F(BackendEnv, backendChkp) { const char *path = "/tmp"; }
@@ -430,6 +440,20 @@ TEST_F(BackendEnv, backendUtil) {
     ASSERT_EQ(nextPow2((uint32_t)(kvDict[i].k)), kvDict[i].v);
   }
 }
+TEST_F(BackendEnv, oldBackendInit) {
+  const char *path = "/tmp/backend1";
+  taosMulMkDir(path);
+  {
+    SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(path, 10, 10);
+    streamBackendCleanup((void *)p);
+  }
+  {
+    SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(path, 10, 10);
+    streamBackendCleanup((void *)p);
+  }
+
+  taosRemoveDir(path);
+}
 
 int main(int argc, char **argv) {
   testing::InitGoogleTest(&argc, argv);

From 4f8ff5b1a679f58eee318e3bed65ee6057ffe3d0 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Wed, 6 Mar 2024 09:44:14 +0800
Subject: [PATCH 153/201] feat: remove seek when write to file and use a small
 write buf

---
 source/libs/executor/src/tsort.c | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 38ab506918..822c997cf0 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -1068,7 +1068,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
   pMemFile->currRegionId = -1;
   pMemFile->currRegionOffset = -1;
 
-  pMemFile->writeBufSize = 64 * 1024 * 1024;
+  pMemFile->writeBufSize = 4 * 1024 * 1024;
   pMemFile->writeBuf = taosMemoryMalloc(pMemFile->writeBufSize);
   pMemFile->writeFileOffset = -1;
   pMemFile->bRegionDirty = false;
@@ -1130,10 +1130,7 @@ static int32_t tsortCloseRegion(SSortHandle* pHandle) {
   pRegion->regionSize = pMemFile->currRegionOffset;
   int32_t writeBytes = pRegion->regionSize - (pMemFile->writeFileOffset - pRegion->fileOffset);
   if (writeBytes > 0) {
-    int ret = tsortSeekFile(pMemFile->pTdFile, pMemFile->writeFileOffset, SEEK_SET);
-    if (ret == 0) {
-      ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile);
-    }
+    int ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile);
     if (ret != 1) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return terrno;
@@ -1165,11 +1162,8 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p
   SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, pMemFile->currRegionId);
   {
     if (pMemFile->currRegionOffset + pHandle->extRowBytes >= pMemFile->writeBufSize) {
-      int ret = tsortSeekFile(pMemFile->pTdFile, pMemFile->writeFileOffset, SEEK_SET);
       int32_t writeBytes = pMemFile->currRegionOffset - (pMemFile->writeFileOffset - pRegion->fileOffset);
-      if (ret == 0) {
-        ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile);
-      }
+      int ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile);
       if (ret !=  1) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         return terrno;

From 3afec5e224d1ac496a09de4fd6eca793c0e05583 Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Wed, 6 Mar 2024 10:44:36 +0800
Subject: [PATCH 154/201] fix: join table fetch error

---
 source/libs/executor/src/dynqueryctrloperator.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c
index 9e7f1144f8..e7c9c17a0b 100755
--- a/source/libs/executor/src/dynqueryctrloperator.c
+++ b/source/libs/executor/src/dynqueryctrloperator.c
@@ -456,7 +456,8 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS
       code = pInfo->stbJoin.basic.srcScan[1] ? buildSingleTableScanOperatorParam(&pSrcParam1, 1, rightVg, rightUid) : buildExchangeOperatorParam(&pSrcParam1, 1, rightVg, rightUid);
     }
   }
-  
+
+  bool initParam = pSrcParam0 ? true : false;  
   if (TSDB_CODE_SUCCESS == code) {
     code = buildGroupCacheOperatorParam(&pGcParam0, 0, *leftVg, *leftUid, pPost->leftNeedCache, pSrcParam0);
     pSrcParam0 = NULL;
@@ -466,7 +467,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS
     pSrcParam1 = NULL;
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = buildMergeJoinOperatorParam(ppParam, pSrcParam0 ? true : false, pGcParam0, pGcParam1);
+    code = buildMergeJoinOperatorParam(ppParam, initParam, pGcParam0, pGcParam1);
   }
   if (TSDB_CODE_SUCCESS != code) {
     if (pSrcParam0) {

From f092eead38d726863dff778f1ae5b9ee5ba5f096 Mon Sep 17 00:00:00 2001
From: chenhaoran <haoran920c@163.com>
Date: Wed, 6 Mar 2024 17:22:25 +0800
Subject: [PATCH 155/201] test:comment tmqparamstest test with restful

---
 tests/parallel_test/cases.task | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index c32790022a..a2348bdedd 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -115,7 +115,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ins_topics_test.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxTopic.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py
-,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R
+#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py

From 02de597e69aa427f816cdd195ef82a59dc79626d Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 7 Mar 2024 09:00:40 +0800
Subject: [PATCH 156/201] opt ci

---
 tests/script/tsim/stream/state0.sim | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim
index 0d96d0841b..37081ad1ae 100644
--- a/tests/script/tsim/stream/state0.sim
+++ b/tests/script/tsim/stream/state0.sim
@@ -21,6 +21,8 @@ print create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0
 
 sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0   into streamt1 as select  _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a)  c4, min(c) c5, max(id) c from t1 state_window(a);
 
+sleep 1000
+
 sql insert into t1 values(1648791213000,1,2,3,1.0,1);
 sql insert into t1 values(1648791213000,1,2,3,1.0,2);
 $loop_count = 0
@@ -457,6 +459,8 @@ print create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0
 
 sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0   into streamt1 as select  _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a)  c4, min(c) c5, max(id) c from t1 state_window(a);
 
+sleep 1000
+
 sql insert into t1 values(1648791212000,2,2,3,1.0,1);
 sql insert into t1 values(1648791213000,1,2,3,1.0,1);
 sql insert into t1 values(1648791213000,1,2,4,1.0,2);
@@ -504,6 +508,9 @@ sql create table t1(ts timestamp, a int, b int , c int, d double, id int);
 print create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0   into streamt3 as select  _wstart, count(*) c1, sum(b) c3 from t1 state_window(a);
 
 sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0   into streamt3 as select  _wstart, count(*) c1, sum(b) c3 from t1 state_window(a);
+
+sleep 1000
+
 sql insert into t1 values(1648791212000,1,2,3,1.0,1);
 sql insert into t1 values(1648791213000,2,2,3,1.0,1);
 sql insert into t1 values(1648791214000,3,2,4,1.0,2);
@@ -557,6 +564,8 @@ print create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0
 
 sql create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 IGNORE UPDATE 0  into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1);
 
+sleep 1000
+
 sql insert into t1 (ts, c1) values (1668073288209, 11);
 sql insert into t1 (ts, c1) values (1668073288210, 11);
 sql insert into t1 (ts, c1) values (1668073288211, 11);
@@ -745,6 +754,9 @@ sql create table b (c timestamp, d int, e int , f int, g double);
 print create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0  into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a);
 
 sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0  into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a);
+
+sleep 1000
+
 sql insert into b values(1648791213000,NULL,NULL,NULL,NULL);
 sql select * from streamt order by c1, c2, c3;
 

From ffc8c8d1485ba3de170e22b82375767955e21f36 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 7 Mar 2024 09:52:56 +0800
Subject: [PATCH 157/201] feat: use tdfile api

---
 source/libs/executor/src/tsort.c | 58 ++++++++++++++++++++------------
 1 file changed, 36 insertions(+), 22 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 822c997cf0..e8eb511ebe 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -67,8 +67,7 @@ typedef struct SSortMemFile {
   int32_t cacheSize;
   int32_t blockSize;
 
-  FILE* pTdFile;
-  // TdFilePtr pTdFile;
+  TdFilePtr pTdFile;
   char memFilePath[PATH_MAX];
 } SSortMemFile;
 
@@ -1027,9 +1026,17 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
   if (pRegion->buf == NULL) {
     pRegion->bufRegOffset = 0;
     pRegion->buf = taosMemoryMalloc(pMemFile->blockSize);
-    tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET);
+    if (pRegion->buf == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    int64_t ret = taosLSeekFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET);
     int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize);
-    fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
+    if (ret >= 0) {
+      ret = taosReadFile(pMemFile->pTdFile, pRegion->buf, readBytes);
+    }
+    if (ret != readBytes) {
+      return TAOS_SYSTEM_ERROR(errno);
+    }
     pRegion->bufLen = readBytes;
   }
   // TODO: ASSERT(pRegion->offset < tupleOffset);
@@ -1037,17 +1044,25 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
     *pFreeRow = false;
     *ppRow = pRegion->buf + tupleOffset - pRegion->bufRegOffset;
   } else {
-      *ppRow = taosMemoryMalloc(rowLen);
-      int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset);
-      memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset,
-             szThisBlock);
-      tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
-      int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen));
-      fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
-      memcpy(*ppRow + szThisBlock, pRegion->buf, rowLen - szThisBlock);
-      *pFreeRow = true;
-      pRegion->bufRegOffset += pRegion->bufLen;
-      pRegion->bufLen = readBytes;
+    *ppRow = taosMemoryMalloc(rowLen);
+    if (*ppRow == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset);
+    memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset,
+            szThisBlock);
+    int64_t ret = taosLSeekFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
+    int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen));
+    if (ret >= 0) {
+      ret = taosReadFile(pMemFile->pTdFile, pRegion->buf, readBytes);
+    }
+    if (ret != readBytes) {
+      return TAOS_SYSTEM_ERROR(errno);        
+    }
+    memcpy(*ppRow + szThisBlock, pRegion->buf, rowLen - szThisBlock);
+    *pFreeRow = true;
+    pRegion->bufRegOffset += pRegion->bufLen;
+    pRegion->bufLen = readBytes;
   }
   //TODO: free region memory
   return TSDB_CODE_SUCCESS;
@@ -1060,7 +1075,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
   SSortMemFile* pMemFile = taosMemoryCalloc(1, sizeof(SSortMemFile));
   pMemFile->cacheSize = pHandle->extRowsMemSize;
   taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
-  pMemFile->pTdFile = fopen(pMemFile->memFilePath, "w+");
+  pMemFile->pTdFile = taosOpenFile(pMemFile->memFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);;
   if (pMemFile->pTdFile == NULL) {
     taosMemoryFree(pMemFile);
     return TAOS_SYSTEM_ERROR(errno);
@@ -1093,8 +1108,7 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
   taosMemoryFree(pMemFile->writeBuf);
   pMemFile->writeBuf = NULL;
 
-  fclose(pMemFile->pTdFile);
-  pMemFile->pTdFile = NULL;
+  taosCloseFile(&pMemFile->pTdFile);
   taosRemoveFile(pMemFile->memFilePath);
   taosMemoryFree(pMemFile);
   pHandle->pExtRowsMemFile = NULL;
@@ -1130,8 +1144,8 @@ static int32_t tsortCloseRegion(SSortHandle* pHandle) {
   pRegion->regionSize = pMemFile->currRegionOffset;
   int32_t writeBytes = pRegion->regionSize - (pMemFile->writeFileOffset - pRegion->fileOffset);
   if (writeBytes > 0) {
-    int ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile);
-    if (ret != 1) {
+    int64_t ret = taosWriteFile(pMemFile->pTdFile, pMemFile->writeBuf, writeBytes);
+    if (ret != writeBytes) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return terrno;
     }
@@ -1163,8 +1177,8 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p
   {
     if (pMemFile->currRegionOffset + pHandle->extRowBytes >= pMemFile->writeBufSize) {
       int32_t writeBytes = pMemFile->currRegionOffset - (pMemFile->writeFileOffset - pRegion->fileOffset);
-      int ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile);
-      if (ret !=  1) {
+      int64_t ret = taosWriteFile(pMemFile->pTdFile, pMemFile->writeBuf, writeBytes);
+      if (ret !=  writeBytes) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         return terrno;
       }

From 53aa45e22577d0a41fcd100f422bc455e89f1daa Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 7 Mar 2024 11:31:02 +0800
Subject: [PATCH 158/201] Revert "feat: use tdfile api"

This reverts commit ffc8c8d1485ba3de170e22b82375767955e21f36.
---
 source/libs/executor/src/tsort.c | 58 ++++++++++++--------------------
 1 file changed, 22 insertions(+), 36 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index e8eb511ebe..822c997cf0 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -67,7 +67,8 @@ typedef struct SSortMemFile {
   int32_t cacheSize;
   int32_t blockSize;
 
-  TdFilePtr pTdFile;
+  FILE* pTdFile;
+  // TdFilePtr pTdFile;
   char memFilePath[PATH_MAX];
 } SSortMemFile;
 
@@ -1026,17 +1027,9 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
   if (pRegion->buf == NULL) {
     pRegion->bufRegOffset = 0;
     pRegion->buf = taosMemoryMalloc(pMemFile->blockSize);
-    if (pRegion->buf == NULL) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    int64_t ret = taosLSeekFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET);
+    tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET);
     int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize);
-    if (ret >= 0) {
-      ret = taosReadFile(pMemFile->pTdFile, pRegion->buf, readBytes);
-    }
-    if (ret != readBytes) {
-      return TAOS_SYSTEM_ERROR(errno);
-    }
+    fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
     pRegion->bufLen = readBytes;
   }
   // TODO: ASSERT(pRegion->offset < tupleOffset);
@@ -1044,25 +1037,17 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
     *pFreeRow = false;
     *ppRow = pRegion->buf + tupleOffset - pRegion->bufRegOffset;
   } else {
-    *ppRow = taosMemoryMalloc(rowLen);
-    if (*ppRow == NULL) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset);
-    memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset,
-            szThisBlock);
-    int64_t ret = taosLSeekFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
-    int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen));
-    if (ret >= 0) {
-      ret = taosReadFile(pMemFile->pTdFile, pRegion->buf, readBytes);
-    }
-    if (ret != readBytes) {
-      return TAOS_SYSTEM_ERROR(errno);        
-    }
-    memcpy(*ppRow + szThisBlock, pRegion->buf, rowLen - szThisBlock);
-    *pFreeRow = true;
-    pRegion->bufRegOffset += pRegion->bufLen;
-    pRegion->bufLen = readBytes;
+      *ppRow = taosMemoryMalloc(rowLen);
+      int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset);
+      memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset,
+             szThisBlock);
+      tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
+      int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen));
+      fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
+      memcpy(*ppRow + szThisBlock, pRegion->buf, rowLen - szThisBlock);
+      *pFreeRow = true;
+      pRegion->bufRegOffset += pRegion->bufLen;
+      pRegion->bufLen = readBytes;
   }
   //TODO: free region memory
   return TSDB_CODE_SUCCESS;
@@ -1075,7 +1060,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
   SSortMemFile* pMemFile = taosMemoryCalloc(1, sizeof(SSortMemFile));
   pMemFile->cacheSize = pHandle->extRowsMemSize;
   taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
-  pMemFile->pTdFile = taosOpenFile(pMemFile->memFilePath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);;
+  pMemFile->pTdFile = fopen(pMemFile->memFilePath, "w+");
   if (pMemFile->pTdFile == NULL) {
     taosMemoryFree(pMemFile);
     return TAOS_SYSTEM_ERROR(errno);
@@ -1108,7 +1093,8 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
   taosMemoryFree(pMemFile->writeBuf);
   pMemFile->writeBuf = NULL;
 
-  taosCloseFile(&pMemFile->pTdFile);
+  fclose(pMemFile->pTdFile);
+  pMemFile->pTdFile = NULL;
   taosRemoveFile(pMemFile->memFilePath);
   taosMemoryFree(pMemFile);
   pHandle->pExtRowsMemFile = NULL;
@@ -1144,8 +1130,8 @@ static int32_t tsortCloseRegion(SSortHandle* pHandle) {
   pRegion->regionSize = pMemFile->currRegionOffset;
   int32_t writeBytes = pRegion->regionSize - (pMemFile->writeFileOffset - pRegion->fileOffset);
   if (writeBytes > 0) {
-    int64_t ret = taosWriteFile(pMemFile->pTdFile, pMemFile->writeBuf, writeBytes);
-    if (ret != writeBytes) {
+    int ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile);
+    if (ret != 1) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return terrno;
     }
@@ -1177,8 +1163,8 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p
   {
     if (pMemFile->currRegionOffset + pHandle->extRowBytes >= pMemFile->writeBufSize) {
       int32_t writeBytes = pMemFile->currRegionOffset - (pMemFile->writeFileOffset - pRegion->fileOffset);
-      int64_t ret = taosWriteFile(pMemFile->pTdFile, pMemFile->writeBuf, writeBytes);
-      if (ret !=  writeBytes) {
+      int ret = fwrite(pMemFile->writeBuf, writeBytes, 1, pMemFile->pTdFile);
+      if (ret !=  1) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         return terrno;
       }

From 80f55abc61895bcf602ae5362d9e9cf54ceac529 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 7 Mar 2024 14:16:10 +0800
Subject: [PATCH 159/201] fix:[TS-4540]seperate block if insert in one sql when
 subscribe query

---
 include/libs/executor/storageapi.h        |  4 +--
 source/dnode/vnode/src/tq/tqRead.c        | 42 ++++++++---------------
 source/dnode/vnode/src/vnd/vnodeInitApi.c |  4 +--
 tests/system-test/7-tmq/tmq_taosx.py      | 38 ++++++++++++++++++++
 4 files changed, 56 insertions(+), 32 deletions(-)

diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h
index e66ffe5521..3433f98d38 100644
--- a/include/libs/executor/storageapi.h
+++ b/include/libs/executor/storageapi.h
@@ -223,10 +223,10 @@ typedef struct SStoreTqReader {
   bool (*tqReaderCurrentBlockConsumed)();
 
   struct SWalReader* (*tqReaderGetWalReader)();  // todo remove it
-  int32_t (*tqReaderRetrieveTaosXBlock)();       // todo remove it
+//  int32_t (*tqReaderRetrieveTaosXBlock)();       // todo remove it
 
   int32_t (*tqReaderSetSubmitMsg)();  // todo remove it
-  bool (*tqReaderNextBlockFilterOut)();
+//  bool (*tqReaderNextBlockFilterOut)();
 } SStoreTqReader;
 
 typedef struct SStoreSnapshotFn {
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 7a737fbb5d..110cac152d 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -368,24 +368,11 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
   }
 }
 
-// todo ignore the error in wal?
 bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
   SWalReader*  pWalReader = pReader->pWalReader;
-  SSDataBlock* pDataBlock = NULL;
 
   uint64_t st = taosGetTimestampMs();
   while (1) {
-    // try next message in wal file
-    if (walNextValidMsg(pWalReader) < 0) {
-      return false;
-    }
-
-    void*   pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
-    int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
-    int64_t ver = pWalReader->pHead->head.version;
-
-    tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver);
-    pReader->nextBlk = 0;
     int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData);
     while (pReader->nextBlk < numOfBlocks) {
       tqTrace("tq reader next data block %d/%d, len:%d %" PRId64, pReader->nextBlk, numOfBlocks, pReader->msg.msgLen,
@@ -400,33 +387,32 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
         tqTrace("tq reader return submit block, uid:%" PRId64, pSubmitTbData->uid);
         SSDataBlock* pRes = NULL;
         int32_t      code = tqRetrieveDataBlock(pReader, &pRes, NULL);
-        if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) {
-          if (pDataBlock == NULL) {
-            pDataBlock = createOneDataBlock(pRes, true);
-          } else {
-            blockDataMerge(pDataBlock, pRes);
-          }
+        if (code == TSDB_CODE_SUCCESS) {
+          return true;
         }
       } else {
         pReader->nextBlk += 1;
         tqTrace("tq reader discard submit block, uid:%" PRId64 ", continue", pSubmitTbData->uid);
       }
     }
+
     tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
     pReader->msg.msgStr = NULL;
 
-    if (pDataBlock != NULL) {
-      blockDataCleanup(pReader->pResBlock);
-      copyDataBlock(pReader->pResBlock, pDataBlock);
-      blockDataDestroy(pDataBlock);
-      return true;
-    } else {
-      qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id);
-    }
-
     if (taosGetTimestampMs() - st > 1000) {
       return false;
     }
+
+    // try next message in wal file
+    if (walNextValidMsg(pWalReader) < 0) {
+      return false;
+    }
+
+    void*   pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
+    int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
+    int64_t ver = pWalReader->pHead->head.version;
+    tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver);
+    pReader->nextBlk = 0;
   }
 }
 
diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c
index aff7e4642b..16abe69def 100644
--- a/source/dnode/vnode/src/vnd/vnodeInitApi.c
+++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c
@@ -128,12 +128,12 @@ void initTqAPI(SStoreTqReader* pTq) {
   pTq->tqReaderCurrentBlockConsumed = tqCurrentBlockConsumed;
 
   pTq->tqReaderGetWalReader = tqGetWalReader;              // todo remove it
-  pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock;  // todo remove it
+//  pTq->tqReaderRetrieveTaosXBlock = tqRetrieveTaosxBlock;  // todo remove it
 
   pTq->tqReaderSetSubmitMsg = tqReaderSetSubmitMsg;  // todo remove it
   pTq->tqGetResultBlock = tqGetResultBlock;
 
-  pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut;
+//  pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut;
   pTq->tqGetResultBlockTime = tqGetResultBlockTime;
 
   pTq->tqGetStreamExecProgress = tqGetStreamExecInfo;
diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py
index c547385d70..a4508bbb9f 100644
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@@ -359,8 +359,46 @@ class TDTestCase:
         finally:
             consumer.close()
 
+    def consume_TS_4540_Test(self):
+        tdSql.execute(f'create database if not exists test')
+        tdSql.execute(f'use test')
+        tdSql.execute(f'CREATE STABLE `test`.`b` ( `time` TIMESTAMP , `task_id` NCHAR(1000) ) TAGS( `key` NCHAR(1000))')
+        tdSql.execute(f"insert into `test`.b1 using `test`.`b`(key) tags('1') (time, task_id) values ('2024-03-04 12:50:01.000', '32') `test`.b2 using `test`.`b`(key) tags('2') (time, task_id) values ('2024-03-04 12:50:01.000', '43') `test`.b3 using `test`.`b`(key) tags('3') (time, task_id) values ('2024-03-04 12:50:01.000', '123456')")
+
+        tdSql.execute(f'create topic tt as select tbname,task_id,key from b')
+        consumer_dict = {
+            "group.id": "g1",
+            "td.connect.user": "root",
+            "td.connect.pass": "taosdata",
+            "auto.offset.reset": "earliest",
+        }
+        consumer = Consumer(consumer_dict)
+
+        try:
+            consumer.subscribe(["tt"])
+        except TmqError:
+            tdLog.exit(f"subscribe error")
+
+        try:
+            while True:
+                res = consumer.poll(1)
+                if not res:
+                    break
+                val = res.value()
+                if val is None:
+                    continue
+                for block in val:
+                    data = block.fetchall()
+                    print(data)
+                    if data != [('b1', '32', '1')] and data != [('b2', '43', '2')] and data != [('b3', '123456', '3')]:
+                        tdLog.exit(f"index = 0 table b1 error")
+
+        finally:
+            consumer.close()
+
     def run(self):
         self.consumeTest()
+        self.consume_TS_4540_Test()
 
         tdSql.prepare()
         self.checkWal1VgroupOnlyMeta()

From f26b7f78f5adbfe6c8ca6ecc85428bb5942c14eb Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 7 Mar 2024 14:23:30 +0800
Subject: [PATCH 160/201] calc stream  tag size

---
 source/libs/parser/src/parTranslater.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 3a622eeea6..491768e9ab 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -8135,7 +8135,9 @@ static int32_t adjustTagsForCreateTable(STranslateContext* pCxt, SCreateStreamSt
     SColumnDefNode* pDef = (SColumnDefNode*)pTagDef;
     if (!dataTypeEqual(&pDef->dataType, &((SExprNode*)pTagExpr)->resType)) {
       SNode*  pFunc = NULL;
-      int32_t code = createCastFunc(pCxt, pTagExpr, pDef->dataType, &pFunc);
+      SDataType defType = pDef->dataType;
+      defType.bytes = calcTypeBytes(defType);
+      int32_t code = createCastFunc(pCxt, pTagExpr, defType, &pFunc);
       if (TSDB_CODE_SUCCESS != code) {
         return code;
       }

From f690e5cfb991f59bf1766d4058b957faf9fa9ee0 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Thu, 7 Mar 2024 14:26:46 +0800
Subject: [PATCH 161/201] fix: add error processing

---
 source/libs/executor/src/tsort.c | 95 ++++++++++++++++++++++++--------
 1 file changed, 73 insertions(+), 22 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 822c997cf0..6d5886730f 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -241,11 +241,19 @@ void destroyTuple(void* t) {
 }
 
 int tsortSeekFile(FILE* file, int64_t offset, int whence) {
-  #ifdef WINDOWS
+#ifdef WINDOWS
   return _fseeki64(file, offset, whence);
-  #else
+#else
   return fseeko(file, offset, whence);
-  #endif
+#endif
+}
+
+int tsortSetAutoDelFile(char* path) {
+#ifdef WINDOWS
+  return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY);
+#else
+  return unlink(path);
+#endif
 }
 
 /**
@@ -1027,9 +1035,16 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
   if (pRegion->buf == NULL) {
     pRegion->bufRegOffset = 0;
     pRegion->buf = taosMemoryMalloc(pMemFile->blockSize);
+    if (pRegion->buf == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
     tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET);
     int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize);
-    fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
+    int ret = fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
+    if (ret != 1) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return terrno;
+    }
     pRegion->bufLen = readBytes;
   }
   // TODO: ASSERT(pRegion->offset < tupleOffset);
@@ -1038,12 +1053,20 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
     *ppRow = pRegion->buf + tupleOffset - pRegion->bufRegOffset;
   } else {
       *ppRow = taosMemoryMalloc(rowLen);
+      if (*ppRow == NULL) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
       int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset);
       memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset,
              szThisBlock);
       tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
       int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen));
-      fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
+      int ret = fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
+      if (ret != 1) {
+        taosMemoryFreeClear(*ppRow);
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        return terrno;
+      }
       memcpy(*ppRow + szThisBlock, pRegion->buf, rowLen - szThisBlock);
       *pFreeRow = true;
       pRegion->bufRegOffset += pRegion->bufLen;
@@ -1057,27 +1080,55 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
   if (pHandle->pExtRowsMemFile != NULL) {
     return TSDB_CODE_SUCCESS;
   }
+  int32_t       code = TSDB_CODE_SUCCESS;
   SSortMemFile* pMemFile = taosMemoryCalloc(1, sizeof(SSortMemFile));
-  pMemFile->cacheSize = pHandle->extRowsMemSize;
-  taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
-  pMemFile->pTdFile = fopen(pMemFile->memFilePath, "w+");
-  if (pMemFile->pTdFile == NULL) {
-    taosMemoryFree(pMemFile);
-    return TAOS_SYSTEM_ERROR(errno);
+  if (pMemFile == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
   }
-  pMemFile->currRegionId = -1;
-  pMemFile->currRegionOffset = -1;
+  if (code == TSDB_CODE_SUCCESS) {
+    taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
+    pMemFile->pTdFile = fopen(pMemFile->memFilePath, "w+");
+    if (pMemFile->pTdFile == NULL) {
+      code = terrno = TAOS_SYSTEM_ERROR(errno);
+    }
+  }
+  if (code == TSDB_CODE_SUCCESS) {
+    tsortSetAutoDelFile(pMemFile->memFilePath);
 
-  pMemFile->writeBufSize = 4 * 1024 * 1024;
-  pMemFile->writeBuf = taosMemoryMalloc(pMemFile->writeBufSize);
-  pMemFile->writeFileOffset = -1;
-  pMemFile->bRegionDirty = false;
+    pMemFile->currRegionId = -1;
+    pMemFile->currRegionOffset = -1;
 
-  pMemFile->aFileRegions = taosArrayInit(64, sizeof(SSortMemFileRegion));
-
-  pHandle->pExtRowsMemFile = pMemFile;
-  return TSDB_CODE_SUCCESS;
-} 
+    pMemFile->writeBufSize = 4 * 1024 * 1024;
+    pMemFile->writeFileOffset = -1;
+    pMemFile->bRegionDirty = false;
+    
+    pMemFile->writeBuf = taosMemoryMalloc(pMemFile->writeBufSize);
+    if (pMemFile->writeBuf == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  if (code == TSDB_CODE_SUCCESS) {
+    pMemFile->cacheSize = pHandle->extRowsMemSize;
+    pMemFile->aFileRegions = taosArrayInit(64, sizeof(SSortMemFileRegion));
+    if (pMemFile->aFileRegions == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  if (code == TSDB_CODE_SUCCESS) {
+    pHandle->pExtRowsMemFile = pMemFile;
+  } else {
+    if (pMemFile) {
+      if (pMemFile->aFileRegions) taosMemoryFreeClear(pMemFile->aFileRegions);
+      if (pMemFile->writeBuf) taosMemoryFreeClear(pMemFile->writeBuf);
+      if (pMemFile->pTdFile) {
+        fclose(pMemFile->pTdFile);
+        pMemFile->pTdFile = NULL;
+      }
+      taosMemoryFreeClear(pMemFile);
+    }
+  }
+  return code;
+}
 
 static int32_t destroySortMemFile(SSortHandle* pHandle) {
   if (pHandle->pExtRowsMemFile == NULL) return TSDB_CODE_SUCCESS;

From 69741d6e10c06cc1745aceba6ef6ddec64cb1303 Mon Sep 17 00:00:00 2001
From: dmchen <cademfly@hotmail.com>
Date: Thu, 7 Mar 2024 06:33:45 +0000
Subject: [PATCH 162/201] fix/TS-4535

---
 source/dnode/vnode/src/inc/vnodeInt.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index e610161544..9fdb4993bd 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -106,7 +106,7 @@ typedef struct SQueryNode         SQueryNode;
 #define VND_INFO_FNAME     "vnode.json"
 #define VND_INFO_FNAME_TMP "vnode_tmp.json"
 
-#define VNODE_METRIC_SQL_COUNT "taos_sql_req:count"
+#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count"
 
 #define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type"
 #define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id"

From b1167a4410a17035879f4d7442d3b1bd2571e931 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 7 Mar 2024 17:18:22 +0800
Subject: [PATCH 163/201] fix:memory leak

---
 tests/parallel_test/cases.task | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index a2348bdedd..2ab001621a 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -228,7 +228,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py
-,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py
+,,n,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_replay.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py
 ,,n,system-test,python3 ./test.py -f 7-tmq/tmq_offset.py

From 7c319adf46e9bbbde24c5ff98afea028d1ef2b8e Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 7 Mar 2024 17:59:02 +0800
Subject: [PATCH 164/201] fix:[TS-4544]do not get vgroup if subscribe query

---
 source/libs/parser/src/parTranslater.c |  2 +-
 tests/system-test/7-tmq/tmq_taosx.py   | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 3a622eeea6..291c04d8c4 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -4547,7 +4547,7 @@ static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (TSDB_CODE_SUCCESS == code) {
     code = getQueryTimeRange(pCxt, pSelect->pWhere, &pSelect->timeRange);
   }
-  if (pSelect->pWhere != NULL) {
+  if (pSelect->pWhere != NULL && pCxt->pParseCxt->topicQuery == false) {
     setTableVgroupsFromEqualTbnameCond(pCxt, pSelect);
   }
   return code;
diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py
index c547385d70..b8f3c944b0 100644
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@@ -359,8 +359,34 @@ class TDTestCase:
         finally:
             consumer.close()
 
+    def consume_ts_4544(self):
+        tdSql.execute(f'create database if not exists d1')
+        tdSql.execute(f'use d1')
+        tdSql.execute(f'create table stt(ts timestamp, i int) tags(t int)')
+        tdSql.execute(f'insert into tt1 using stt tags(1) values(now, 1) (now+1s, 2)')
+        tdSql.execute(f'insert into tt2 using stt tags(2) values(now, 1) (now+1s, 2)')
+        tdSql.execute(f'insert into tt3 using stt tags(3) values(now, 1) (now+1s, 2)')
+        tdSql.execute(f'insert into tt1 using stt tags(1) values(now+5s, 11) (now+10s, 12)')
+
+        tdSql.execute(f'create topic topic_in as select * from stt where tbname in ("tt2")')
+        consumer_dict = {
+            "group.id": "g1",
+            "td.connect.user": "root",
+            "td.connect.pass": "taosdata",
+            "auto.offset.reset": "earliest",
+        }
+        consumer = Consumer(consumer_dict)
+
+        try:
+            consumer.subscribe(["topic_in"])
+        except TmqError:
+            tdLog.exit(f"subscribe error")
+
+        consumer.close()
+
     def run(self):
         self.consumeTest()
+        self.consume_ts_4544()
 
         tdSql.prepare()
         self.checkWal1VgroupOnlyMeta()

From 37071e0cfbc6186a8b657af7a0ebf21b14a537d3 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Thu, 7 Mar 2024 18:43:35 +0800
Subject: [PATCH 165/201] do some stream fill refactor

---
 source/libs/executor/src/filloperator.c       | 1142 ----------------
 source/libs/executor/src/streamfilloperator.c | 1181 +++++++++++++++++
 2 files changed, 1181 insertions(+), 1142 deletions(-)
 create mode 100644 source/libs/executor/src/streamfilloperator.c

diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c
index ecc2526dd9..e1aa75d828 100644
--- a/source/libs/executor/src/filloperator.c
+++ b/source/libs/executor/src/filloperator.c
@@ -32,12 +32,6 @@
 #include "operator.h"
 #include "querytask.h"
 
-
-#define FILL_POS_INVALID 0
-#define FILL_POS_START   1
-#define FILL_POS_MID     2
-#define FILL_POS_END     3
-
 typedef struct STimeRange {
   TSKEY    skey;
   TSKEY    ekey;
@@ -474,1139 +468,3 @@ _error:
   taosMemoryFreeClear(pOperator);
   return NULL;
 }
-
-TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) {
-  STimeWindow win = {.skey = ts, .ekey = ts};
-  getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);
-  return win.skey;
-}
-
-TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) {
-  STimeWindow win = {.skey = ts, .ekey = ts};
-  getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC);
-  return win.skey;
-}
-
-void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) {
-  colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull);
-}
-
-SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) {
-  if (!pRaw || !pRaw->pRowVal) {
-    return NULL;
-  }
-  char*            pData = (char*)pRaw->pRowVal;
-  SResultCellData* pCell = pRaw->pRowVal;
-  for (int32_t i = 0; i < index; i++) {
-    pData += (pCell->bytes + sizeof(SResultCellData));
-    pCell = (SResultCellData*)pData;
-  }
-  return pCell;
-}
-
-void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) {
-  for (int32_t i = start; i < end; i++) {
-    destroyExprInfo(pFillCol[i].pExpr, 1);
-    taosVariantDestroy(&pFillCol[i].fillVal);
-  }
-  taosMemoryFreeClear(pFillCol[start].pExpr);
-  taosMemoryFree(pFillCol);
-  return NULL;
-}
-
-void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
-  pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols);
-  tSimpleHashCleanup(pFillSup->pResMap);
-  pFillSup->pResMap = NULL;
-  cleanupExprSupp(&pFillSup->notFillExprSup);
-  if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) {
-    taosMemoryFree(pFillSup->cur.pRowVal);
-  }
-  taosMemoryFree(pFillSup->prev.pRowVal);
-  taosMemoryFree(pFillSup->next.pRowVal);
-  taosMemoryFree(pFillSup->nextNext.pRowVal);
-
-  taosMemoryFree(pFillSup);
-  return NULL;
-}
-
-void destroySPoint(void* ptr) {
-  SPoint* point = (SPoint*) ptr;
-  taosMemoryFreeClear(point->val);
-}
-
-void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) {
-  taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint);
-  taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint);
-  taosMemoryFree(pFillLinear);
-  return NULL;
-}
-void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) {
-  if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F ||
-      pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) {
-    taosMemoryFreeClear(pFillInfo->pResRow->pRowVal);
-    taosMemoryFreeClear(pFillInfo->pResRow);
-  }
-  pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo);
-  taosArrayDestroy(pFillInfo->delRanges);
-  taosMemoryFree(pFillInfo);
-  return NULL;
-}
-
-static void destroyStreamFillOperatorInfo(void* param) {
-  SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param;
-  pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo);
-  pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup);
-  pInfo->pRes = blockDataDestroy(pInfo->pRes);
-  pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock);
-  pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);
-  pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList);
-  taosMemoryFree(pInfo);
-}
-
-static void resetFillWindow(SResultRowData* pRowData) {
-  pRowData->key = INT64_MIN;
-  taosMemoryFreeClear(pRowData->pRowVal);
-}
-
-void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) {
-  if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) {
-    resetFillWindow(&pFillSup->cur);
-  } else {
-    pFillSup->cur.key = INT64_MIN;
-    pFillSup->cur.pRowVal = NULL;
-  }
-  resetFillWindow(&pFillSup->prev);
-  resetFillWindow(&pFillSup->next);
-  resetFillWindow(&pFillSup->nextNext);
-}
-
-void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
-  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
-
-  void* pState = pOperator->pTaskInfo->streamInfo.pState;
-  resetPrevAndNextWindow(pFillSup, pState, pAPI);
-
-  SWinKey key = {.ts = ts, .groupId = groupId};
-  int32_t curVLen = 0;
-
-  int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen);
-  ASSERT(code == TSDB_CODE_SUCCESS);
-  pFillSup->cur.key = key.ts;
-}
-
-void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
-  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
-  void* pState = pOperator->pTaskInfo->streamInfo.pState;
-  resetPrevAndNextWindow(pFillSup, pState, pAPI);
-
-  SWinKey key = {.ts = ts, .groupId = groupId};
-  void*   curVal = NULL;
-  int32_t curVLen = 0;
-  int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen);
-  ASSERT(code == TSDB_CODE_SUCCESS);
-  pFillSup->cur.key = key.ts;
-  pFillSup->cur.pRowVal = curVal;
-
-  SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key);
-  SWinKey          preKey = {.ts = INT64_MIN, .groupId = groupId};
-  void*            preVal = NULL;
-  int32_t          preVLen = 0;
-  code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen);
-
-  if (code == TSDB_CODE_SUCCESS) {
-    pFillSup->prev.key = preKey.ts;
-    pFillSup->prev.pRowVal = preVal;
-
-    code = pAPI->stateStore.streamStateCurNext(pState, pCur);
-    ASSERT(code == TSDB_CODE_SUCCESS);
-
-    code = pAPI->stateStore.streamStateCurNext(pState, pCur);
-    if (code != TSDB_CODE_SUCCESS) {
-      pAPI->stateStore.streamStateFreeCur(pCur);
-      pCur = NULL;
-    }
-  } else {
-    pAPI->stateStore.streamStateFreeCur(pCur);
-    pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key);
-  }
-
-  SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId};
-  void*   nextVal = NULL;
-  int32_t nextVLen = 0;
-  code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen);
-  if (code == TSDB_CODE_SUCCESS) {
-    pFillSup->next.key = nextKey.ts;
-    pFillSup->next.pRowVal = nextVal;
-    if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) {
-      code = pAPI->stateStore.streamStateCurNext(pState, pCur);
-      if (code == TSDB_CODE_SUCCESS) {
-        SWinKey nextNextKey = {.groupId = groupId};
-        void*   nextNextVal = NULL;
-        int32_t nextNextVLen = 0;
-        code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen);
-        if (code == TSDB_CODE_SUCCESS) {
-          pFillSup->nextNext.key = nextNextKey.ts;
-          pFillSup->nextNext.pRowVal = nextNextVal;
-        }
-      }
-    }
-  }
-  pAPI->stateStore.streamStateFreeCur(pCur);
-}
-
-static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; }
-static bool hasNextWindow(SStreamFillSupporter* pFillSup) { return pFillSup->next.key != INT64_MIN; }
-static bool hasNextNextWindow(SStreamFillSupporter* pFillSup) {
-  return pFillSup->nextNext.key != INT64_MIN;
-  return false;
-}
-
-static void transBlockToResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SResultRowData* pRowVal) {
-  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-  for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
-    SResultCellData* pCell = getResultCell(pRowVal, i);
-    if (!colDataIsNull_s(pColData, rowId)) {
-      pCell->isNull = false;
-      pCell->type = pColData->info.type;
-      pCell->bytes = pColData->info.bytes;
-      char* val = colDataGetData(pColData, rowId);
-      if (IS_VAR_DATA_TYPE(pCell->type)) {
-        memcpy(pCell->pData, val, varDataTLen(val));
-      } else {
-        memcpy(pCell->pData, val, pCell->bytes);
-      }
-    } else {
-      pCell->isNull = true;
-    }
-  }
-  pRowVal->key = ts;
-}
-
-static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pRowVal, SArray* pDelta,
-                          SFillColInfo* pFillCol, int32_t numOfCol, int32_t winCount, int32_t order) {
-  for (int32_t i = 0; i < numOfCol; i++) {
-    if (!pFillCol[i].notFillCol) {
-      int32_t          slotId = GET_DEST_SLOT_ID(pFillCol + i);
-      SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
-      char*            var = colDataGetData(pCol, rowId);
-      double           start = 0;
-      GET_TYPED_DATA(start, double, pCol->info.type, var);
-      SResultCellData* pCell = getResultCell(pRowVal, slotId);
-      double           end = 0;
-      GET_TYPED_DATA(end, double, pCell->type, pCell->pData);
-      double delta = 0;
-      if (order == TSDB_ORDER_ASC) {
-        delta = (end - start) / winCount;
-      } else {
-        delta = (start - end) / winCount;
-      }
-      taosArraySet(pDelta, slotId, &delta);
-    }
-  }
-}
-
-static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol,
-                             int32_t numOfCol) {
-  for (int32_t i = 0; i < numOfCol; i++) {
-    if (!pFillCol[i].notFillCol) {
-      int32_t          slotId = GET_DEST_SLOT_ID(pFillCol + i);
-      SResultCellData* pECell = getResultCell(pEndRow, slotId);
-      SPoint* pPoint = taosArrayGet(pEndPoins, slotId);
-      pPoint->key = pEndRow->key;
-      memcpy(pPoint->val, pECell->pData, pECell->bytes);
-    }
-  }
-}
-
-static void setFillInfoStart(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) {
-  ts = taosTimeAdd(ts, pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
-  pFillInfo->start = ts;
-}
-
-static void setFillInfoEnd(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) {
-  ts = taosTimeAdd(ts, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision);
-  pFillInfo->end = ts;
-}
-
-static void setFillKeyInfo(TSKEY start, TSKEY end, SInterval* pInterval, SStreamFillInfo* pFillInfo) {
-  setFillInfoStart(start, pInterval, pFillInfo);
-  pFillInfo->current = pFillInfo->start;
-  setFillInfoEnd(end, pInterval, pFillInfo);
-}
-
-void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) {
-  if (!hasPrevWindow(pFillSup) || !hasNextWindow(pFillSup)) {
-    pFillInfo->needFill = false;
-    return;
-  }
-
-  TSKEY realStart = taosTimeAdd(pFillSup->prev.key, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
-                                pFillSup->interval.precision);
-
-  pFillInfo->needFill = true;
-  pFillInfo->start = realStart;
-  pFillInfo->current = pFillInfo->start;
-  pFillInfo->end = end;
-  pFillInfo->pos = FILL_POS_INVALID;
-  switch (pFillInfo->type) {
-    case TSDB_FILL_NULL:
-    case TSDB_FILL_NULL_F:
-    case TSDB_FILL_SET_VALUE:
-    case TSDB_FILL_SET_VALUE_F:
-      break;
-    case TSDB_FILL_PREV:
-      pFillInfo->pResRow = &pFillSup->prev;
-      break;
-    case TSDB_FILL_NEXT:
-      pFillInfo->pResRow = &pFillSup->next;
-      break;
-    case TSDB_FILL_LINEAR: {
-      setFillKeyInfo(pFillSup->prev.key, pFillSup->next.key, &pFillSup->interval, pFillInfo);
-      pFillInfo->pLinearInfo->hasNext = false;
-      pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
-      calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
-                       pFillSup->numOfAllCols);
-      pFillInfo->pResRow = &pFillSup->prev;
-      pFillInfo->pLinearInfo->winIndex = 0;
-    } break;
-    default:
-      ASSERT(0);
-      break;
-  }
-}
-
-void copyNotFillExpData(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) {
-  for (int32_t i = pFillSup->numOfFillCols; i < pFillSup->numOfAllCols; ++i) {
-    SFillColInfo*    pFillCol = pFillSup->pAllColInfo + i;
-    int32_t          slotId = GET_DEST_SLOT_ID(pFillCol);
-    SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId);
-    SResultCellData* pCurCell = getResultCell(&pFillSup->cur, slotId);
-    pCell->isNull = pCurCell->isNull;
-    if (!pCurCell->isNull) {
-      memcpy(pCell->pData, pCurCell->pData, pCell->bytes);
-    }
-  }
-}
-
-void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillSupporter* pFillSup,
-                      SStreamFillInfo* pFillInfo) {
-  pFillInfo->preRowKey = pFillSup->cur.key;
-  if (!hasPrevWindow(pFillSup) && !hasNextWindow(pFillSup)) {
-    pFillInfo->needFill = false;
-    pFillInfo->pos = FILL_POS_START;
-    return;
-  }
-  TSKEY prevWKey = INT64_MIN;
-  TSKEY nextWKey = INT64_MIN;
-  if (hasPrevWindow(pFillSup)) {
-    prevWKey = pFillSup->prev.key;
-  }
-  if (hasNextWindow(pFillSup)) {
-    nextWKey = pFillSup->next.key;
-  }
-
-  pFillInfo->needFill = true;
-  pFillInfo->pos = FILL_POS_INVALID;
-  switch (pFillInfo->type) {
-    case TSDB_FILL_NULL:
-    case TSDB_FILL_NULL_F:
-    case TSDB_FILL_SET_VALUE:
-    case TSDB_FILL_SET_VALUE_F: {
-      if (pFillSup->prev.key == pFillInfo->preRowKey) {
-        resetFillWindow(&pFillSup->prev);
-      }
-      if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) {
-        if (pFillSup->next.key == pFillInfo->nextRowKey) {
-          pFillInfo->preRowKey = INT64_MIN;
-          setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
-          pFillInfo->pos = FILL_POS_END;
-        } else {
-          pFillInfo->needFill = false;
-          pFillInfo->pos = FILL_POS_START;
-        }
-      } else if (hasPrevWindow(pFillSup)) {
-        setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
-        pFillInfo->pos = FILL_POS_END;
-      } else {
-        setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo);
-        pFillInfo->pos = FILL_POS_START;
-      }
-      copyNotFillExpData(pFillSup, pFillInfo);
-    } break;
-    case TSDB_FILL_PREV: {
-      if (hasNextWindow(pFillSup) && ((pFillSup->next.key != pFillInfo->nextRowKey) ||
-                                      (pFillSup->next.key == pFillInfo->nextRowKey && hasNextNextWindow(pFillSup)) ||
-                                      (pFillSup->next.key == pFillInfo->nextRowKey && !hasPrevWindow(pFillSup)))) {
-        setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo);
-        pFillInfo->pos = FILL_POS_START;
-        resetFillWindow(&pFillSup->prev);
-        pFillSup->prev.key = pFillSup->cur.key;
-        pFillSup->prev.pRowVal = pFillSup->cur.pRowVal;
-      } else if (hasPrevWindow(pFillSup)) {
-        setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
-        pFillInfo->pos = FILL_POS_END;
-        pFillInfo->preRowKey = INT64_MIN;
-      }
-      pFillInfo->pResRow = &pFillSup->prev;
-    } break;
-    case TSDB_FILL_NEXT: {
-      if (hasPrevWindow(pFillSup)) {
-        setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
-        pFillInfo->pos = FILL_POS_END;
-        resetFillWindow(&pFillSup->next);
-        pFillSup->next.key = pFillSup->cur.key;
-        pFillSup->next.pRowVal = pFillSup->cur.pRowVal;
-        pFillInfo->preRowKey = INT64_MIN;
-      } else {
-        ASSERT(hasNextWindow(pFillSup));
-        setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo);
-        pFillInfo->pos = FILL_POS_START;
-      }
-      pFillInfo->pResRow = &pFillSup->next;
-    } break;
-    case TSDB_FILL_LINEAR: {
-      pFillInfo->pLinearInfo->winIndex = 0;
-      if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) {
-        setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
-        pFillInfo->pos = FILL_POS_MID;
-        pFillInfo->pLinearInfo->nextEnd = nextWKey;
-        calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
-                         pFillSup->numOfAllCols);
-        pFillInfo->pResRow = &pFillSup->prev;
-
-        calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pNextEndPoints, pFillSup->pAllColInfo,
-                         pFillSup->numOfAllCols);
-        pFillInfo->pLinearInfo->hasNext = true;
-      } else if (hasPrevWindow(pFillSup)) {
-        setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
-        pFillInfo->pos = FILL_POS_END;
-        pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
-        calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
-                         pFillSup->numOfAllCols);
-        pFillInfo->pResRow = &pFillSup->prev;
-        pFillInfo->pLinearInfo->hasNext = false;
-      } else {
-        ASSERT(hasNextWindow(pFillSup));
-        setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo);
-        pFillInfo->pos = FILL_POS_START;
-        pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
-        calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
-                         pFillSup->numOfAllCols);
-        pFillInfo->pResRow = &pFillSup->cur;
-        pFillInfo->pLinearInfo->hasNext = false;
-      }
-    } break;
-    default:
-      ASSERT(0);
-      break;
-  }
-  ASSERT(pFillInfo->pos != FILL_POS_INVALID);
-}
-
-static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId) {
-  SWinKey key = {.groupId = groupId, .ts = ts};
-  if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) {
-    return false;
-  }
-  tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0);
-  return true;
-}
-
-static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) {
-  if (pBlock->info.rows >= pBlock->info.capacity) {
-    return false;
-  }
-  uint64_t groupId = pBlock->info.id.groupId;
-  if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) {
-    return true;
-  }
-  for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
-    SFillColInfo*    pFillCol = pFillSup->pAllColInfo + i;
-    int32_t          slotId = GET_DEST_SLOT_ID(pFillCol);
-    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId);
-    SFillInfo        tmpInfo = {
-               .currentKey = ts,
-               .order = TSDB_ORDER_ASC,
-               .interval = pFillSup->interval,
-    };
-    bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows);
-    if (!filled) {
-      SResultCellData* pCell = getResultCell(pResRow, slotId);
-      setRowCell(pColData, pBlock->info.rows, pCell);
-    }
-  }
-  pBlock->info.rows++;
-  return true;
-}
-
-static bool hasRemainCalc(SStreamFillInfo* pFillInfo) {
-  if (pFillInfo->current != INT64_MIN && pFillInfo->current <= pFillInfo->end) {
-    return true;
-  }
-  return false;
-}
-
-static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) {
-  while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) {
-    STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current};
-    if (inWinRange(&pFillSup->winRange, &st)) {
-      buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock);
-    }
-    pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
-                                     pFillSup->interval.precision);
-  }
-}
-
-static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) {
-  while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) {
-    uint64_t groupId = pBlock->info.id.groupId;
-    SWinKey  key = {.groupId = groupId, .ts = pFillInfo->current};
-    STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current};
-    if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) {
-      pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
-                                       pFillSup->interval.precision);
-      pFillInfo->pLinearInfo->winIndex++;
-      continue;
-    }
-    pFillInfo->pLinearInfo->winIndex++;
-    for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
-      SFillColInfo* pFillCol = pFillSup->pAllColInfo + i;
-      SFillInfo     tmp = {
-              .currentKey = pFillInfo->current,
-              .order = TSDB_ORDER_ASC,
-              .interval = pFillSup->interval,
-      };
-
-      int32_t          slotId = GET_DEST_SLOT_ID(pFillCol);
-      SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId);
-      int16_t          type = pColData->info.type;
-      SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId);
-      int32_t          index = pBlock->info.rows;
-      if (pFillCol->notFillCol) {
-        bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index);
-        if (!filled) {
-          setRowCell(pColData, index, pCell);
-        }
-      } else {
-        if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) {
-          colDataSetNULL(pColData, index);
-          continue;
-        }
-        SPoint* pEnd = taosArrayGet(pFillInfo->pLinearInfo->pEndPoints, slotId);
-        double  vCell = 0;
-        SPoint start = {0};
-        start.key = pFillInfo->pResRow->key;
-        start.val = pCell->pData;
-
-        SPoint cur = {0};
-        cur.key = pFillInfo->current;
-        cur.val = taosMemoryCalloc(1, pCell->bytes);
-        taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type);
-        colDataSetVal(pColData, index, (const char*)cur.val, false);
-        destroySPoint(&cur);
-      }
-    }
-    pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
-                                     pFillSup->interval.precision);
-    pBlock->info.rows++;
-  }
-}
-
-static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) {
-  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
-
-  SWinKey key = {.groupId = groupId, .ts = pRow->key};
-  int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len);
-  qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 "  code:%d", key.ts, key.groupId, code);
-  ASSERT(code == TSDB_CODE_SUCCESS);
-}
-
-static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) {
-  if (pFillInfo->needFill == false) {
-    buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes);
-    return;
-  }
-
-  if (pFillInfo->pos == FILL_POS_START) {
-    if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) {
-      pFillInfo->pos = FILL_POS_INVALID;
-    }
-  }
-  if (pFillInfo->type != TSDB_FILL_LINEAR) {
-    doStreamFillNormal(pFillSup, pFillInfo, pRes);
-  } else {
-    doStreamFillLinear(pFillSup, pFillInfo, pRes);
-
-    if (pFillInfo->pos == FILL_POS_MID) {
-      if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) {
-        pFillInfo->pos = FILL_POS_INVALID;
-      }
-    }
-
-    if (pFillInfo->current > pFillInfo->end && pFillInfo->pLinearInfo->hasNext) {
-      pFillInfo->pLinearInfo->hasNext = false;
-      pFillInfo->pLinearInfo->winIndex = 0;
-      taosArraySwap(pFillInfo->pLinearInfo->pEndPoints, pFillInfo->pLinearInfo->pNextEndPoints);
-      pFillInfo->pResRow = &pFillSup->cur;
-      setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo);
-      doStreamFillLinear(pFillSup, pFillInfo, pRes);
-    }
-  }
-  if (pFillInfo->pos == FILL_POS_END) {
-    if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) {
-      pFillInfo->pos = FILL_POS_INVALID;
-    }
-  }
-}
-
-void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol,
-                           int32_t rowId, uint64_t groupId, int32_t rowSize) {
-  TSKEY ts = tsCol[rowId];
-  pFillInfo->nextRowKey = ts;
-  SResultRowData tmpNextRow = {.key = ts};
-  tmpNextRow.pRowVal = taosMemoryCalloc(1, rowSize);
-  transBlockToResultRow(pBlock, rowId, ts, &tmpNextRow);
-  keepResultInDiscBuf(pOperator, groupId, &tmpNextRow, rowSize);
-  taosMemoryFreeClear(tmpNextRow.pRowVal);
-}
-
-static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo,
-                          SSDataBlock* pBlock, TSKEY* tsCol, int32_t rowId, SSDataBlock* pRes) {
-  uint64_t groupId = pBlock->info.id.groupId;
-  getWindowFromDiscBuf(pOperator, tsCol[rowId], groupId, pFillSup);
-  if (pFillSup->prev.key == pFillInfo->preRowKey) {
-    resetFillWindow(&pFillSup->prev);
-  }
-  setFillValueInfo(pBlock, tsCol[rowId], rowId, pFillSup, pFillInfo);
-  doStreamFillRange(pFillInfo, pFillSup, pRes);
-}
-
-static void doStreamFillImpl(SOperatorInfo* pOperator) {
-  SStreamFillOperatorInfo* pInfo = pOperator->info;
-  SExecTaskInfo*           pTaskInfo = pOperator->pTaskInfo;
-  SStreamFillSupporter*    pFillSup = pInfo->pFillSup;
-  SStreamFillInfo*         pFillInfo = pInfo->pFillInfo;
-  SSDataBlock*             pBlock = pInfo->pSrcBlock;
-  uint64_t                 groupId = pBlock->info.id.groupId;
-  SSDataBlock*             pRes = pInfo->pRes;
-  SColumnInfoData*         pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol);
-  TSKEY*                   tsCol = (TSKEY*)pTsCol->pData;
-  pRes->info.id.groupId = groupId;
-  pInfo->srcRowIndex++;
-
-  if (pInfo->srcRowIndex == 0) {
-    keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize);
-    pInfo->srcRowIndex++;
-  }
-
-  while (pInfo->srcRowIndex < pBlock->info.rows) {
-    TSKEY ts = tsCol[pInfo->srcRowIndex];
-    keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize);
-    doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes);
-    if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) {
-      blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol);
-      return;
-    }
-    pInfo->srcRowIndex++;
-  }
-  doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes);
-  blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol);
-  blockDataCleanup(pInfo->pSrcBlock);
-}
-
-static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) {
-  SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;
-  void* pState = pOp->pTaskInfo->streamInfo.pState;
-
-  SSDataBlock*     pBlock = delRes;
-  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
-  SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
-  SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
-  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
-  SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
-  SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
-  SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
-  colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false);
-  colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false);
-  colDataSetNULL(pUidCol, pBlock->info.rows);
-  colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false);
-  colDataSetNULL(pCalStartCol, pBlock->info.rows);
-  colDataSetNULL(pCalEndCol, pBlock->info.rows);
-
-  SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
-
-  void* tbname = NULL;
-  pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname);
-  if (tbname == NULL) {
-    colDataSetNULL(pTableCol, pBlock->info.rows);
-  } else {
-    char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
-    STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
-    colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
-    pAPI->stateStore.streamStateFreeVal(tbname);
-  }
-
-  pBlock->info.rows++;
-}
-
-static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId,
-                              SSDataBlock* delRes) {
-  SStreamFillOperatorInfo* pInfo = pOperator->info;
-  SStreamFillSupporter*    pFillSup = pInfo->pFillSup;
-  if (hasPrevWindow(pFillSup)) {
-    TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval);
-    buildDeleteRange(pOperator, start, endTs, groupId, delRes);
-  } else if (hasNextWindow(pFillSup)) {
-    TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval);
-    buildDeleteRange(pOperator, startTs, end, groupId, delRes);
-  } else {
-    buildDeleteRange(pOperator, startTs, endTs, groupId, delRes);
-  }
-}
-
-static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) {
-  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
-  SStreamFillOperatorInfo* pInfo = pOperator->info;
-  getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup);
-  setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo);
-  SWinKey key = {.ts = startTs, .groupId = groupId};
-  if (!pInfo->pFillInfo->needFill) {
-    pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
-    buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes);
-  } else {
-    STimeRange tw = {
-        .skey = startTs,
-        .ekey = endTs,
-        .groupId = groupId,
-    };
-    taosArrayPush(pInfo->pFillInfo->delRanges, &tw);
-    while (key.ts <= endTs) {
-      key.ts = taosTimeAdd(key.ts, pInfo->pFillSup->interval.sliding, pInfo->pFillSup->interval.slidingUnit,
-                           pInfo->pFillSup->interval.precision);
-      tSimpleHashPut(pInfo->pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0);
-    }
-  }
-}
-
-static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
-  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
-
-  SStreamFillOperatorInfo* pInfo = pOperator->info;
-  SStreamFillInfo*         pFillInfo = pInfo->pFillInfo;
-  int32_t                  size = taosArrayGetSize(pFillInfo->delRanges);
-  tSimpleHashClear(pInfo->pFillSup->pResMap);
-  for (; pFillInfo->delIndex < size; pFillInfo->delIndex++) {
-    STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex);
-    if (pInfo->pRes->info.id.groupId != 0 && pInfo->pRes->info.id.groupId != range->groupId) {
-      return;
-    }
-    getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup);
-    setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo);
-    if (pInfo->pFillInfo->needFill) {
-      doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes);
-      pInfo->pRes->info.id.groupId = range->groupId;
-    }
-    SWinKey key = {.ts = range->skey, .groupId = range->groupId};
-    pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
-  }
-}
-
-static void doDeleteFillResult(SOperatorInfo* pOperator) {
-  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
-
-  SStreamFillOperatorInfo* pInfo = pOperator->info;
-  SStreamFillInfo*         pFillInfo = pInfo->pFillInfo;
-  SSDataBlock*             pBlock = pInfo->pSrcDelBlock;
-
-  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
-  TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
-  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
-  uint64_t*        groupIds = (uint64_t*)pGroupCol->pData;
-  while (pInfo->srcDelRowIndex < pBlock->info.rows) {
-    TSKEY            ts = tsStarts[pInfo->srcDelRowIndex];
-    TSKEY            endTs = ts;
-    uint64_t         groupId = groupIds[pInfo->srcDelRowIndex];
-    SWinKey          key = {.ts = ts, .groupId = groupId};
-    SStreamStateCur* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key);
-
-    if (!pCur) {
-      pInfo->srcDelRowIndex++;
-      continue;
-    }
-
-    SWinKey nextKey = {.groupId = groupId, .ts = ts};
-    while (pInfo->srcDelRowIndex < pBlock->info.rows) {
-      TSKEY    delTs = tsStarts[pInfo->srcDelRowIndex];
-      uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex];
-      int32_t  code = TSDB_CODE_SUCCESS;
-      if (groupId != delGroupId) {
-        break;
-      }
-      if (delTs > nextKey.ts) {
-        break;
-      }
-
-      SWinKey delKey = {.groupId = delGroupId, .ts = delTs};
-      if (delTs == nextKey.ts) {
-        code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur);
-        if (code == TSDB_CODE_SUCCESS) {
-          code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL);
-        }
-        // ts will be deleted later
-        if (delTs != ts) {
-          pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey);
-          pAPI->stateStore.streamStateFreeCur(pCur);
-          pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey);
-        }
-        endTs = TMAX(delTs, nextKey.ts - 1);
-        if (code != TSDB_CODE_SUCCESS) {
-          break;
-        }
-      }
-      pInfo->srcDelRowIndex++;
-    }
-
-    pAPI->stateStore.streamStateFreeCur(pCur);
-    doDeleteFillResultImpl(pOperator, ts, endTs, groupId);
-  }
-
-  pFillInfo->current = pFillInfo->end + 1;
-}
-
-static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) {
-  tSimpleHashClear(pInfo->pFillSup->pResMap);
-  pInfo->pFillSup->hasDelete = false;
-  taosArrayClear(pInfo->pFillInfo->delRanges);
-  pInfo->pFillInfo->delIndex = 0;
-}
-
-static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) {
-  SStreamFillOperatorInfo* pInfo = pOperator->info;
-  SExprSupp*               pSup = &pOperator->exprSupp;
-
-  blockDataCleanup(pDstBlock);
-  blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows);
-  setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
-  projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL);
-
-  pDstBlock->info.rows = 0;
-  pSup = &pInfo->pFillSup->notFillExprSup;
-  setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
-  projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL);
-  pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId;
-
-  blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol);
-}
-
-static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
-  SStreamFillOperatorInfo* pInfo = pOperator->info;
-  SExecTaskInfo*           pTaskInfo = pOperator->pTaskInfo;
-
-  if (pOperator->status == OP_EXEC_DONE) {
-    return NULL;
-  }
-  blockDataCleanup(pInfo->pRes);
-  if (hasRemainCalc(pInfo->pFillInfo) ||
-      (pInfo->pFillInfo->pos != FILL_POS_INVALID && pInfo->pFillInfo->needFill == true)) {
-    doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes);
-    if (pInfo->pRes->info.rows > 0) {
-      printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
-      return pInfo->pRes;
-    }
-  }
-  if (pOperator->status == OP_RES_TO_RETURN) {
-    doDeleteFillFinalize(pOperator);
-    if (pInfo->pRes->info.rows > 0) {
-      printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
-      return pInfo->pRes;
-    }
-    setOperatorCompleted(pOperator);
-    resetStreamFillInfo(pInfo);
-    return NULL;
-  }
-
-  SSDataBlock*   fillResult = NULL;
-  SOperatorInfo* downstream = pOperator->pDownstream[0];
-  while (1) {
-    if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows || pInfo->pSrcBlock->info.rows == 0) {
-      // If there are delete datablocks, we receive  them first.
-      SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
-      if (pBlock == NULL) {
-        pOperator->status = OP_RES_TO_RETURN;
-        pInfo->pFillInfo->preRowKey = INT64_MIN;
-        if (pInfo->pRes->info.rows > 0) {
-          printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
-          return pInfo->pRes;
-        }
-        break;
-      }
-      printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
-
-      if (pInfo->pFillInfo->curGroupId != pBlock->info.id.groupId) {
-        pInfo->pFillInfo->curGroupId = pBlock->info.id.groupId;
-        pInfo->pFillInfo->preRowKey = INT64_MIN;
-      }
-
-      pInfo->pFillSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
-      if (pInfo->pFillSup->winRange.ekey <= 0) {
-        pInfo->pFillSup->winRange.ekey = INT64_MAX;
-      }
-
-      switch (pBlock->info.type) {
-        case STREAM_RETRIEVE:
-          return pBlock;
-        case STREAM_DELETE_RESULT: {
-          pInfo->pSrcDelBlock = pBlock;
-          pInfo->srcDelRowIndex = 0;
-          blockDataCleanup(pInfo->pDelRes);
-          pInfo->pFillSup->hasDelete = true;
-          doDeleteFillResult(pOperator);
-          if (pInfo->pDelRes->info.rows > 0) {
-            printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
-            return pInfo->pDelRes;
-          }
-          continue;
-        } break;
-        case STREAM_NORMAL:
-        case STREAM_INVALID: 
-        case STREAM_PULL_DATA: {
-          doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock);
-          memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
-          pInfo->srcRowIndex = -1;
-        } break;
-        case STREAM_CHECKPOINT:
-        case STREAM_CREATE_CHILD_TABLE: {
-          return pBlock;
-        } break;
-        default:
-          ASSERTS(false, "invalid SSDataBlock type");
-      }
-    }
-
-    doStreamFillImpl(pOperator);
-    doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo);
-    memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
-    pOperator->resultInfo.totalRows += pInfo->pRes->info.rows;
-    if (pInfo->pRes->info.rows > 0) {
-      break;
-    }
-  }
-  if (pOperator->status == OP_RES_TO_RETURN) {
-    doDeleteFillFinalize(pOperator);
-  }
-
-  if (pInfo->pRes->info.rows == 0) {
-    setOperatorCompleted(pOperator);
-    resetStreamFillInfo(pInfo);
-    return NULL;
-  }
-
-  pOperator->resultInfo.totalRows += pInfo->pRes->info.rows;
-  printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
-  return pInfo->pRes;
-}
-
-static int32_t initResultBuf(SStreamFillSupporter* pFillSup) {
-  pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols;
-  for (int i = 0; i < pFillSup->numOfAllCols; i++) {
-    SFillColInfo* pCol = &pFillSup->pAllColInfo[i];
-    SResSchema*   pSchema = &pCol->pExpr->base.resSchema;
-    pFillSup->rowSize += pSchema->bytes;
-  }
-  pFillSup->next.key = INT64_MIN;
-  pFillSup->nextNext.key = INT64_MIN;
-  pFillSup->prev.key = INT64_MIN;
-  pFillSup->cur.key = INT64_MIN;
-  pFillSup->next.pRowVal = NULL;
-  pFillSup->nextNext.pRowVal = NULL;
-  pFillSup->prev.pRowVal = NULL;
-  pFillSup->cur.pRowVal = NULL;
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval,
-                                               SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) {
-  SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter));
-  if (!pFillSup) {
-    return NULL;
-  }
-  pFillSup->numOfFillCols = numOfFillCols;
-  int32_t    numOfNotFillCols = 0;
-  SExprInfo* noFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols);
-  pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, noFillExprInfo, numOfNotFillCols,
-                                            (const SNodeListNode*)(pPhyFillNode->pValues));
-  pFillSup->type = convertFillType(pPhyFillNode->mode);
-  pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols;
-  pFillSup->interval = *pInterval;
-  pFillSup->pAPI = pAPI;
-
-  int32_t code = initResultBuf(pFillSup);
-  if (code != TSDB_CODE_SUCCESS) {
-    destroyStreamFillSupporter(pFillSup);
-    return NULL;
-  }
-
-  SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols);
-  code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore);
-  if (code != TSDB_CODE_SUCCESS) {
-    destroyStreamFillSupporter(pFillSup);
-    return NULL;
-  }
-
-  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
-  pFillSup->pResMap = tSimpleHashInit(16, hashFn);
-  pFillSup->hasDelete = false;
-  return pFillSup;
-}
-
-SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) {
-  SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo));
-  pFillInfo->start = INT64_MIN;
-  pFillInfo->current = INT64_MIN;
-  pFillInfo->end = INT64_MIN;
-  pFillInfo->preRowKey = INT64_MIN;
-  pFillInfo->needFill = false;
-  pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo));
-  pFillInfo->pLinearInfo->hasNext = false;
-  pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
-  pFillInfo->pLinearInfo->pEndPoints = NULL;
-  pFillInfo->pLinearInfo->pNextEndPoints = NULL;
-  if (pFillSup->type == TSDB_FILL_LINEAR) {
-    pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint));
-    pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint));
-    for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) {
-      SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i);
-      SPoint value = {0};
-      value.val = taosMemoryCalloc(1, pColData->info.bytes);
-      taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value);
-
-      value.val = taosMemoryCalloc(1, pColData->info.bytes);
-      taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value);
-    }
-  }
-  pFillInfo->pLinearInfo->winIndex = 0;
-
-  pFillInfo->pResRow = NULL;
-  if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_SET_VALUE_F ||
-      pFillSup->type == TSDB_FILL_NULL || pFillSup->type == TSDB_FILL_NULL_F) {
-    pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData));
-    pFillInfo->pResRow->key = INT64_MIN;
-    pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize);
-    for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
-      SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i);
-      SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i);
-      pCell->bytes = pColData->info.bytes;
-      pCell->type = pColData->info.type;
-    }
-  }
-
-  pFillInfo->type = pFillSup->type;
-  pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange));
-  pFillInfo->delIndex = 0;
-  pFillInfo->curGroupId = 0;
-  return pFillInfo;
-}
-
-SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode,
-                                            SExecTaskInfo* pTaskInfo) {
-  SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo));
-  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
-  if (pInfo == NULL || pOperator == NULL) {
-    goto _error;
-  }
-
-  SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval;
-  int32_t    numOfFillCols = 0;
-  SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols);
-  pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI);
-  if (!pInfo->pFillSup) {
-    goto _error;
-  }
-
-  initResultSizeInfo(&pOperator->resultInfo, 4096);
-  pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
-  pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
-  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
-  blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity);
-
-  pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes);
-  if (!pInfo->pFillInfo) {
-    goto _error;
-  }
-
-  if (pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE || pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE_F) {
-    for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) {
-      SFillColInfo*    pFillCol = pInfo->pFillSup->pAllColInfo + i;
-      int32_t          slotId = GET_DEST_SLOT_ID(pFillCol);
-      SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId);
-      SVariant*        pVar = &(pFillCol->fillVal);
-      if (pCell->type == TSDB_DATA_TYPE_FLOAT) {
-        float v = 0;
-        GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
-        SET_TYPED_DATA(pCell->pData, pCell->type, v);
-      } else if (IS_FLOAT_TYPE(pCell->type)) {
-        double v = 0;
-        GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
-        SET_TYPED_DATA(pCell->pData, pCell->type, v);
-      } else if (IS_INTEGER_TYPE(pCell->type)) {
-        int64_t v = 0;
-        GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
-        SET_TYPED_DATA(pCell->pData, pCell->type, v);
-      } else {
-        pCell->isNull = true;
-      }
-    }
-  } else if (pInfo->pFillInfo->type == TSDB_FILL_NULL || pInfo->pFillInfo->type == TSDB_FILL_NULL_F) {
-    for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) {
-      SFillColInfo*    pFillCol = pInfo->pFillSup->pAllColInfo + i;
-      int32_t          slotId = GET_DEST_SLOT_ID(pFillCol);
-      SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId);
-      pCell->isNull = true;
-    }
-  }
-
-  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
-  blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity);
-
-  pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId;
-  pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId;
-
-  int32_t numOfOutputCols = 0;
-  int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc,
-                                     &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  pInfo->srcRowIndex = -1;
-  setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo,
-                  pTaskInfo);
-  pOperator->fpSet =
-      createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
-  setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
-
-  code = appendDownstream(pOperator, &downstream, 1);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-  return pOperator;
-
-_error:
-  destroyStreamFillOperatorInfo(pInfo);
-  taosMemoryFreeClear(pOperator);
-  pTaskInfo->code = code;
-  return NULL;
-}
diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c
new file mode 100644
index 0000000000..b54802e2c6
--- /dev/null
+++ b/source/libs/executor/src/streamfilloperator.c
@@ -0,0 +1,1181 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "filter.h"
+#include "os.h"
+#include "query.h"
+#include "taosdef.h"
+#include "tmsg.h"
+#include "ttypes.h"
+
+#include "executorInt.h"
+#include "tcommon.h"
+#include "thash.h"
+#include "ttime.h"
+
+#include "function.h"
+#include "querynodes.h"
+#include "tdatablock.h"
+#include "tfill.h"
+#include "operator.h"
+#include "querytask.h"
+
+
+#define FILL_POS_INVALID 0
+#define FILL_POS_START   1
+#define FILL_POS_MID     2
+#define FILL_POS_END     3
+
+typedef struct STimeRange {
+  TSKEY    skey;
+  TSKEY    ekey;
+  uint64_t groupId;
+} STimeRange;
+
+TSKEY getNextWindowTs(TSKEY ts, SInterval* pInterval) {
+  STimeWindow win = {.skey = ts, .ekey = ts};
+  getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);
+  return win.skey;
+}
+
+TSKEY getPrevWindowTs(TSKEY ts, SInterval* pInterval) {
+  STimeWindow win = {.skey = ts, .ekey = ts};
+  getNextTimeWindow(pInterval, &win, TSDB_ORDER_DESC);
+  return win.skey;
+}
+
+void setRowCell(SColumnInfoData* pCol, int32_t rowId, const SResultCellData* pCell) {
+  colDataSetVal(pCol, rowId, pCell->pData, pCell->isNull);
+}
+
+SResultCellData* getResultCell(SResultRowData* pRaw, int32_t index) {
+  if (!pRaw || !pRaw->pRowVal) {
+    return NULL;
+  }
+  char*            pData = (char*)pRaw->pRowVal;
+  SResultCellData* pCell = pRaw->pRowVal;
+  for (int32_t i = 0; i < index; i++) {
+    pData += (pCell->bytes + sizeof(SResultCellData));
+    pCell = (SResultCellData*)pData;
+  }
+  return pCell;
+}
+
+void* destroyFillColumnInfo(SFillColInfo* pFillCol, int32_t start, int32_t end) {
+  for (int32_t i = start; i < end; i++) {
+    destroyExprInfo(pFillCol[i].pExpr, 1);
+    taosVariantDestroy(&pFillCol[i].fillVal);
+  }
+  taosMemoryFreeClear(pFillCol[start].pExpr);
+  taosMemoryFree(pFillCol);
+  return NULL;
+}
+
+void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) {
+  pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols);
+  tSimpleHashCleanup(pFillSup->pResMap);
+  pFillSup->pResMap = NULL;
+  cleanupExprSupp(&pFillSup->notFillExprSup);
+  if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) {
+    taosMemoryFree(pFillSup->cur.pRowVal);
+  }
+  taosMemoryFree(pFillSup->prev.pRowVal);
+  taosMemoryFree(pFillSup->next.pRowVal);
+  taosMemoryFree(pFillSup->nextNext.pRowVal);
+
+  taosMemoryFree(pFillSup);
+  return NULL;
+}
+
+void destroySPoint(void* ptr) {
+  SPoint* point = (SPoint*) ptr;
+  taosMemoryFreeClear(point->val);
+}
+
+void* destroyStreamFillLinearInfo(SStreamFillLinearInfo* pFillLinear) {
+  taosArrayDestroyEx(pFillLinear->pEndPoints, destroySPoint);
+  taosArrayDestroyEx(pFillLinear->pNextEndPoints, destroySPoint);
+  taosMemoryFree(pFillLinear);
+  return NULL;
+}
+void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) {
+  if (pFillInfo->type == TSDB_FILL_SET_VALUE || pFillInfo->type == TSDB_FILL_SET_VALUE_F ||
+      pFillInfo->type == TSDB_FILL_NULL || pFillInfo->type == TSDB_FILL_NULL_F) {
+    taosMemoryFreeClear(pFillInfo->pResRow->pRowVal);
+    taosMemoryFreeClear(pFillInfo->pResRow);
+  }
+  pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo);
+  taosArrayDestroy(pFillInfo->delRanges);
+  taosMemoryFree(pFillInfo);
+  return NULL;
+}
+
+static void destroyStreamFillOperatorInfo(void* param) {
+  SStreamFillOperatorInfo* pInfo = (SStreamFillOperatorInfo*)param;
+  pInfo->pFillInfo = destroyStreamFillInfo(pInfo->pFillInfo);
+  pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup);
+  pInfo->pRes = blockDataDestroy(pInfo->pRes);
+  pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock);
+  pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);
+  pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList);
+  taosMemoryFree(pInfo);
+}
+
+static void resetFillWindow(SResultRowData* pRowData) {
+  pRowData->key = INT64_MIN;
+  taosMemoryFreeClear(pRowData->pRowVal);
+}
+
+void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, void* pState, SStorageAPI* pAPI) {
+  if (pFillSup->cur.pRowVal != pFillSup->prev.pRowVal && pFillSup->cur.pRowVal != pFillSup->next.pRowVal) {
+    resetFillWindow(&pFillSup->cur);
+  } else {
+    pFillSup->cur.key = INT64_MIN;
+    pFillSup->cur.pRowVal = NULL;
+  }
+  resetFillWindow(&pFillSup->prev);
+  resetFillWindow(&pFillSup->next);
+  resetFillWindow(&pFillSup->nextNext);
+}
+
+void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
+  void* pState = pOperator->pTaskInfo->streamInfo.pState;
+  resetPrevAndNextWindow(pFillSup, pState, pAPI);
+
+  SWinKey key = {.ts = ts, .groupId = groupId};
+  int32_t curVLen = 0;
+
+  int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen);
+  ASSERT(code == TSDB_CODE_SUCCESS);
+  pFillSup->cur.key = key.ts;
+}
+
+void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+  void* pState = pOperator->pTaskInfo->streamInfo.pState;
+  resetPrevAndNextWindow(pFillSup, pState, pAPI);
+
+  SWinKey key = {.ts = ts, .groupId = groupId};
+  void*   curVal = NULL;
+  int32_t curVLen = 0;
+  int32_t code = pAPI->stateStore.streamStateFillGet(pState, &key, (void**)&curVal, &curVLen);
+  ASSERT(code == TSDB_CODE_SUCCESS);
+  pFillSup->cur.key = key.ts;
+  pFillSup->cur.pRowVal = curVal;
+
+  SStreamStateCur* pCur = pAPI->stateStore.streamStateFillSeekKeyPrev(pState, &key);
+  SWinKey          preKey = {.ts = INT64_MIN, .groupId = groupId};
+  void*            preVal = NULL;
+  int32_t          preVLen = 0;
+  code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen);
+
+  if (code == TSDB_CODE_SUCCESS) {
+    pFillSup->prev.key = preKey.ts;
+    pFillSup->prev.pRowVal = preVal;
+
+    code = pAPI->stateStore.streamStateCurNext(pState, pCur);
+    ASSERT(code == TSDB_CODE_SUCCESS);
+
+    code = pAPI->stateStore.streamStateCurNext(pState, pCur);
+    if (code != TSDB_CODE_SUCCESS) {
+      pAPI->stateStore.streamStateFreeCur(pCur);
+      pCur = NULL;
+    }
+  } else {
+    pAPI->stateStore.streamStateFreeCur(pCur);
+    pCur = pAPI->stateStore.streamStateFillSeekKeyNext(pState, &key);
+  }
+
+  SWinKey nextKey = {.ts = INT64_MIN, .groupId = groupId};
+  void*   nextVal = NULL;
+  int32_t nextVLen = 0;
+  code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen);
+  if (code == TSDB_CODE_SUCCESS) {
+    pFillSup->next.key = nextKey.ts;
+    pFillSup->next.pRowVal = nextVal;
+    if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) {
+      code = pAPI->stateStore.streamStateCurNext(pState, pCur);
+      if (code == TSDB_CODE_SUCCESS) {
+        SWinKey nextNextKey = {.groupId = groupId};
+        void*   nextNextVal = NULL;
+        int32_t nextNextVLen = 0;
+        code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen);
+        if (code == TSDB_CODE_SUCCESS) {
+          pFillSup->nextNext.key = nextNextKey.ts;
+          pFillSup->nextNext.pRowVal = nextNextVal;
+        }
+      }
+    }
+  }
+  pAPI->stateStore.streamStateFreeCur(pCur);
+}
+
+static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; }
+static bool hasNextWindow(SStreamFillSupporter* pFillSup) { return pFillSup->next.key != INT64_MIN; }
+static bool hasNextNextWindow(SStreamFillSupporter* pFillSup) {
+  return pFillSup->nextNext.key != INT64_MIN;
+  return false;
+}
+
+static void transBlockToResultRow(const SSDataBlock* pBlock, int32_t rowId, TSKEY ts, SResultRowData* pRowVal) {
+  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
+    SResultCellData* pCell = getResultCell(pRowVal, i);
+    if (!colDataIsNull_s(pColData, rowId)) {
+      pCell->isNull = false;
+      pCell->type = pColData->info.type;
+      pCell->bytes = pColData->info.bytes;
+      char* val = colDataGetData(pColData, rowId);
+      if (IS_VAR_DATA_TYPE(pCell->type)) {
+        memcpy(pCell->pData, val, varDataTLen(val));
+      } else {
+        memcpy(pCell->pData, val, pCell->bytes);
+      }
+    } else {
+      pCell->isNull = true;
+    }
+  }
+  pRowVal->key = ts;
+}
+
+static void calcDeltaData(SSDataBlock* pBlock, int32_t rowId, SResultRowData* pRowVal, SArray* pDelta,
+                          SFillColInfo* pFillCol, int32_t numOfCol, int32_t winCount, int32_t order) {
+  for (int32_t i = 0; i < numOfCol; i++) {
+    if (!pFillCol[i].notFillCol) {
+      int32_t          slotId = GET_DEST_SLOT_ID(pFillCol + i);
+      SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
+      char*            var = colDataGetData(pCol, rowId);
+      double           start = 0;
+      GET_TYPED_DATA(start, double, pCol->info.type, var);
+      SResultCellData* pCell = getResultCell(pRowVal, slotId);
+      double           end = 0;
+      GET_TYPED_DATA(end, double, pCell->type, pCell->pData);
+      double delta = 0;
+      if (order == TSDB_ORDER_ASC) {
+        delta = (end - start) / winCount;
+      } else {
+        delta = (start - end) / winCount;
+      }
+      taosArraySet(pDelta, slotId, &delta);
+    }
+  }
+}
+
+static void calcRowDeltaData(SResultRowData* pEndRow, SArray* pEndPoins, SFillColInfo* pFillCol,
+                             int32_t numOfCol) {
+  for (int32_t i = 0; i < numOfCol; i++) {
+    if (!pFillCol[i].notFillCol) {
+      int32_t          slotId = GET_DEST_SLOT_ID(pFillCol + i);
+      SResultCellData* pECell = getResultCell(pEndRow, slotId);
+      SPoint* pPoint = taosArrayGet(pEndPoins, slotId);
+      pPoint->key = pEndRow->key;
+      memcpy(pPoint->val, pECell->pData, pECell->bytes);
+    }
+  }
+}
+
+static void setFillInfoStart(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) {
+  ts = taosTimeAdd(ts, pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
+  pFillInfo->start = ts;
+}
+
+static void setFillInfoEnd(TSKEY ts, SInterval* pInterval, SStreamFillInfo* pFillInfo) {
+  ts = taosTimeAdd(ts, pInterval->sliding * -1, pInterval->slidingUnit, pInterval->precision);
+  pFillInfo->end = ts;
+}
+
+static void setFillKeyInfo(TSKEY start, TSKEY end, SInterval* pInterval, SStreamFillInfo* pFillInfo) {
+  setFillInfoStart(start, pInterval, pFillInfo);
+  pFillInfo->current = pFillInfo->start;
+  setFillInfoEnd(end, pInterval, pFillInfo);
+}
+
+void setDeleteFillValueInfo(TSKEY start, TSKEY end, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) {
+  if (!hasPrevWindow(pFillSup) || !hasNextWindow(pFillSup)) {
+    pFillInfo->needFill = false;
+    return;
+  }
+
+  TSKEY realStart = taosTimeAdd(pFillSup->prev.key, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
+                                pFillSup->interval.precision);
+
+  pFillInfo->needFill = true;
+  pFillInfo->start = realStart;
+  pFillInfo->current = pFillInfo->start;
+  pFillInfo->end = end;
+  pFillInfo->pos = FILL_POS_INVALID;
+  switch (pFillInfo->type) {
+    case TSDB_FILL_NULL:
+    case TSDB_FILL_NULL_F:
+    case TSDB_FILL_SET_VALUE:
+    case TSDB_FILL_SET_VALUE_F:
+      break;
+    case TSDB_FILL_PREV:
+      pFillInfo->pResRow = &pFillSup->prev;
+      break;
+    case TSDB_FILL_NEXT:
+      pFillInfo->pResRow = &pFillSup->next;
+      break;
+    case TSDB_FILL_LINEAR: {
+      setFillKeyInfo(pFillSup->prev.key, pFillSup->next.key, &pFillSup->interval, pFillInfo);
+      pFillInfo->pLinearInfo->hasNext = false;
+      pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
+      calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
+                       pFillSup->numOfAllCols);
+      pFillInfo->pResRow = &pFillSup->prev;
+      pFillInfo->pLinearInfo->winIndex = 0;
+    } break;
+    default:
+      ASSERT(0);
+      break;
+  }
+}
+
+void copyNotFillExpData(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo) {
+  for (int32_t i = pFillSup->numOfFillCols; i < pFillSup->numOfAllCols; ++i) {
+    SFillColInfo*    pFillCol = pFillSup->pAllColInfo + i;
+    int32_t          slotId = GET_DEST_SLOT_ID(pFillCol);
+    SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId);
+    SResultCellData* pCurCell = getResultCell(&pFillSup->cur, slotId);
+    pCell->isNull = pCurCell->isNull;
+    if (!pCurCell->isNull) {
+      memcpy(pCell->pData, pCurCell->pData, pCell->bytes);
+    }
+  }
+}
+
+void setFillValueInfo(SSDataBlock* pBlock, TSKEY ts, int32_t rowId, SStreamFillSupporter* pFillSup,
+                      SStreamFillInfo* pFillInfo) {
+  pFillInfo->preRowKey = pFillSup->cur.key;
+  if (!hasPrevWindow(pFillSup) && !hasNextWindow(pFillSup)) {
+    pFillInfo->needFill = false;
+    pFillInfo->pos = FILL_POS_START;
+    return;
+  }
+  TSKEY prevWKey = INT64_MIN;
+  TSKEY nextWKey = INT64_MIN;
+  if (hasPrevWindow(pFillSup)) {
+    prevWKey = pFillSup->prev.key;
+  }
+  if (hasNextWindow(pFillSup)) {
+    nextWKey = pFillSup->next.key;
+  }
+
+  pFillInfo->needFill = true;
+  pFillInfo->pos = FILL_POS_INVALID;
+  switch (pFillInfo->type) {
+    case TSDB_FILL_NULL:
+    case TSDB_FILL_NULL_F:
+    case TSDB_FILL_SET_VALUE:
+    case TSDB_FILL_SET_VALUE_F: {
+      if (pFillSup->prev.key == pFillInfo->preRowKey) {
+        resetFillWindow(&pFillSup->prev);
+      }
+      if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) {
+        if (pFillSup->next.key == pFillInfo->nextRowKey) {
+          pFillInfo->preRowKey = INT64_MIN;
+          setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
+          pFillInfo->pos = FILL_POS_END;
+        } else {
+          pFillInfo->needFill = false;
+          pFillInfo->pos = FILL_POS_START;
+        }
+      } else if (hasPrevWindow(pFillSup)) {
+        setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
+        pFillInfo->pos = FILL_POS_END;
+      } else {
+        setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo);
+        pFillInfo->pos = FILL_POS_START;
+      }
+      copyNotFillExpData(pFillSup, pFillInfo);
+    } break;
+    case TSDB_FILL_PREV: {
+      if (hasNextWindow(pFillSup) && ((pFillSup->next.key != pFillInfo->nextRowKey) ||
+                                      (pFillSup->next.key == pFillInfo->nextRowKey && hasNextNextWindow(pFillSup)) ||
+                                      (pFillSup->next.key == pFillInfo->nextRowKey && !hasPrevWindow(pFillSup)))) {
+        setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo);
+        pFillInfo->pos = FILL_POS_START;
+        resetFillWindow(&pFillSup->prev);
+        pFillSup->prev.key = pFillSup->cur.key;
+        pFillSup->prev.pRowVal = pFillSup->cur.pRowVal;
+      } else if (hasPrevWindow(pFillSup)) {
+        setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
+        pFillInfo->pos = FILL_POS_END;
+        pFillInfo->preRowKey = INT64_MIN;
+      }
+      pFillInfo->pResRow = &pFillSup->prev;
+    } break;
+    case TSDB_FILL_NEXT: {
+      if (hasPrevWindow(pFillSup)) {
+        setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
+        pFillInfo->pos = FILL_POS_END;
+        resetFillWindow(&pFillSup->next);
+        pFillSup->next.key = pFillSup->cur.key;
+        pFillSup->next.pRowVal = pFillSup->cur.pRowVal;
+        pFillInfo->preRowKey = INT64_MIN;
+      } else {
+        ASSERT(hasNextWindow(pFillSup));
+        setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo);
+        pFillInfo->pos = FILL_POS_START;
+      }
+      pFillInfo->pResRow = &pFillSup->next;
+    } break;
+    case TSDB_FILL_LINEAR: {
+      pFillInfo->pLinearInfo->winIndex = 0;
+      if (hasPrevWindow(pFillSup) && hasNextWindow(pFillSup)) {
+        setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
+        pFillInfo->pos = FILL_POS_MID;
+        pFillInfo->pLinearInfo->nextEnd = nextWKey;
+        calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
+                         pFillSup->numOfAllCols);
+        pFillInfo->pResRow = &pFillSup->prev;
+
+        calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pNextEndPoints, pFillSup->pAllColInfo,
+                         pFillSup->numOfAllCols);
+        pFillInfo->pLinearInfo->hasNext = true;
+      } else if (hasPrevWindow(pFillSup)) {
+        setFillKeyInfo(prevWKey, ts, &pFillSup->interval, pFillInfo);
+        pFillInfo->pos = FILL_POS_END;
+        pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
+        calcRowDeltaData(&pFillSup->cur, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
+                         pFillSup->numOfAllCols);
+        pFillInfo->pResRow = &pFillSup->prev;
+        pFillInfo->pLinearInfo->hasNext = false;
+      } else {
+        ASSERT(hasNextWindow(pFillSup));
+        setFillKeyInfo(ts, nextWKey, &pFillSup->interval, pFillInfo);
+        pFillInfo->pos = FILL_POS_START;
+        pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
+        calcRowDeltaData(&pFillSup->next, pFillInfo->pLinearInfo->pEndPoints, pFillSup->pAllColInfo,
+                         pFillSup->numOfAllCols);
+        pFillInfo->pResRow = &pFillSup->cur;
+        pFillInfo->pLinearInfo->hasNext = false;
+      }
+    } break;
+    default:
+      ASSERT(0);
+      break;
+  }
+  ASSERT(pFillInfo->pos != FILL_POS_INVALID);
+}
+
+static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t groupId) {
+  SWinKey key = {.groupId = groupId, .ts = ts};
+  if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) {
+    return false;
+  }
+  tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0);
+  return true;
+}
+
+static bool buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) {
+  if (pBlock->info.rows >= pBlock->info.capacity) {
+    return false;
+  }
+  uint64_t groupId = pBlock->info.id.groupId;
+  if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) {
+    return true;
+  }
+  for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
+    SFillColInfo*    pFillCol = pFillSup->pAllColInfo + i;
+    int32_t          slotId = GET_DEST_SLOT_ID(pFillCol);
+    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId);
+    SFillInfo        tmpInfo = {
+               .currentKey = ts,
+               .order = TSDB_ORDER_ASC,
+               .interval = pFillSup->interval,
+    };
+    bool filled = fillIfWindowPseudoColumn(&tmpInfo, pFillCol, pColData, pBlock->info.rows);
+    if (!filled) {
+      SResultCellData* pCell = getResultCell(pResRow, slotId);
+      setRowCell(pColData, pBlock->info.rows, pCell);
+    }
+  }
+  pBlock->info.rows++;
+  return true;
+}
+
+static bool hasRemainCalc(SStreamFillInfo* pFillInfo) {
+  if (pFillInfo->current != INT64_MIN && pFillInfo->current <= pFillInfo->end) {
+    return true;
+  }
+  return false;
+}
+
+static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) {
+  while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) {
+    STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current};
+    if (inWinRange(&pFillSup->winRange, &st)) {
+      buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock);
+    }
+    pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
+                                     pFillSup->interval.precision);
+  }
+}
+
+static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) {
+  while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) {
+    uint64_t groupId = pBlock->info.id.groupId;
+    SWinKey  key = {.groupId = groupId, .ts = pFillInfo->current};
+    STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current};
+    if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) {
+      pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
+                                       pFillSup->interval.precision);
+      pFillInfo->pLinearInfo->winIndex++;
+      continue;
+    }
+    pFillInfo->pLinearInfo->winIndex++;
+    for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
+      SFillColInfo* pFillCol = pFillSup->pAllColInfo + i;
+      SFillInfo     tmp = {
+              .currentKey = pFillInfo->current,
+              .order = TSDB_ORDER_ASC,
+              .interval = pFillSup->interval,
+      };
+
+      int32_t          slotId = GET_DEST_SLOT_ID(pFillCol);
+      SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, slotId);
+      int16_t          type = pColData->info.type;
+      SResultCellData* pCell = getResultCell(pFillInfo->pResRow, slotId);
+      int32_t          index = pBlock->info.rows;
+      if (pFillCol->notFillCol) {
+        bool filled = fillIfWindowPseudoColumn(&tmp, pFillCol, pColData, index);
+        if (!filled) {
+          setRowCell(pColData, index, pCell);
+        }
+      } else {
+        if (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_BOOL || pCell->isNull) {
+          colDataSetNULL(pColData, index);
+          continue;
+        }
+        SPoint* pEnd = taosArrayGet(pFillInfo->pLinearInfo->pEndPoints, slotId);
+        double  vCell = 0;
+        SPoint start = {0};
+        start.key = pFillInfo->pResRow->key;
+        start.val = pCell->pData;
+
+        SPoint cur = {0};
+        cur.key = pFillInfo->current;
+        cur.val = taosMemoryCalloc(1, pCell->bytes);
+        taosGetLinearInterpolationVal(&cur, pCell->type, &start, pEnd, pCell->type);
+        colDataSetVal(pColData, index, (const char*)cur.val, false);
+        destroySPoint(&cur);
+      }
+    }
+    pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
+                                     pFillSup->interval.precision);
+    pBlock->info.rows++;
+  }
+}
+
+static void keepResultInDiscBuf(SOperatorInfo* pOperator, uint64_t groupId, SResultRowData* pRow, int32_t len) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
+  SWinKey key = {.groupId = groupId, .ts = pRow->key};
+  int32_t code = pAPI->stateStore.streamStateFillPut(pOperator->pTaskInfo->streamInfo.pState, &key, pRow->pRowVal, len);
+  qDebug("===stream===fill operator save key ts:%" PRId64 " group id:%" PRIu64 "  code:%d", key.ts, key.groupId, code);
+  ASSERT(code == TSDB_CODE_SUCCESS);
+}
+
+static void doStreamFillRange(SStreamFillInfo* pFillInfo, SStreamFillSupporter* pFillSup, SSDataBlock* pRes) {
+  if (pFillInfo->needFill == false) {
+    buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes);
+    return;
+  }
+
+  if (pFillInfo->pos == FILL_POS_START) {
+    if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) {
+      pFillInfo->pos = FILL_POS_INVALID;
+    }
+  }
+  if (pFillInfo->type != TSDB_FILL_LINEAR) {
+    doStreamFillNormal(pFillSup, pFillInfo, pRes);
+  } else {
+    doStreamFillLinear(pFillSup, pFillInfo, pRes);
+
+    if (pFillInfo->pos == FILL_POS_MID) {
+      if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) {
+        pFillInfo->pos = FILL_POS_INVALID;
+      }
+    }
+
+    if (pFillInfo->current > pFillInfo->end && pFillInfo->pLinearInfo->hasNext) {
+      pFillInfo->pLinearInfo->hasNext = false;
+      pFillInfo->pLinearInfo->winIndex = 0;
+      taosArraySwap(pFillInfo->pLinearInfo->pEndPoints, pFillInfo->pLinearInfo->pNextEndPoints);
+      pFillInfo->pResRow = &pFillSup->cur;
+      setFillKeyInfo(pFillSup->cur.key, pFillInfo->pLinearInfo->nextEnd, &pFillSup->interval, pFillInfo);
+      doStreamFillLinear(pFillSup, pFillInfo, pRes);
+    }
+  }
+  if (pFillInfo->pos == FILL_POS_END) {
+    if (buildFillResult(&pFillSup->cur, pFillSup, pFillSup->cur.key, pRes)) {
+      pFillInfo->pos = FILL_POS_INVALID;
+    }
+  }
+}
+
+void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock, TSKEY* tsCol,
+                           int32_t rowId, uint64_t groupId, int32_t rowSize) {
+  TSKEY ts = tsCol[rowId];
+  pFillInfo->nextRowKey = ts;
+  SResultRowData tmpNextRow = {.key = ts};
+  tmpNextRow.pRowVal = taosMemoryCalloc(1, rowSize);
+  transBlockToResultRow(pBlock, rowId, ts, &tmpNextRow);
+  keepResultInDiscBuf(pOperator, groupId, &tmpNextRow, rowSize);
+  taosMemoryFreeClear(tmpNextRow.pRowVal);
+}
+
+static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo,
+                          SSDataBlock* pBlock, TSKEY* tsCol, int32_t rowId, SSDataBlock* pRes) {
+  uint64_t groupId = pBlock->info.id.groupId;
+  getWindowFromDiscBuf(pOperator, tsCol[rowId], groupId, pFillSup);
+  if (pFillSup->prev.key == pFillInfo->preRowKey) {
+    resetFillWindow(&pFillSup->prev);
+  }
+  setFillValueInfo(pBlock, tsCol[rowId], rowId, pFillSup, pFillInfo);
+  doStreamFillRange(pFillInfo, pFillSup, pRes);
+}
+
+static void doStreamFillImpl(SOperatorInfo* pOperator) {
+  SStreamFillOperatorInfo* pInfo = pOperator->info;
+  SExecTaskInfo*           pTaskInfo = pOperator->pTaskInfo;
+  SStreamFillSupporter*    pFillSup = pInfo->pFillSup;
+  SStreamFillInfo*         pFillInfo = pInfo->pFillInfo;
+  SSDataBlock*             pBlock = pInfo->pSrcBlock;
+  uint64_t                 groupId = pBlock->info.id.groupId;
+  SSDataBlock*             pRes = pInfo->pRes;
+  SColumnInfoData*         pTsCol = taosArrayGet(pInfo->pSrcBlock->pDataBlock, pInfo->primaryTsCol);
+  TSKEY*                   tsCol = (TSKEY*)pTsCol->pData;
+  pRes->info.id.groupId = groupId;
+  pInfo->srcRowIndex++;
+
+  if (pInfo->srcRowIndex == 0) {
+    keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize);
+    pInfo->srcRowIndex++;
+  }
+
+  while (pInfo->srcRowIndex < pBlock->info.rows) {
+    TSKEY ts = tsCol[pInfo->srcRowIndex];
+    keepBlockRowInDiscBuf(pOperator, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex, groupId, pFillSup->rowSize);
+    doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes);
+    if (pInfo->pRes->info.rows == pInfo->pRes->info.capacity) {
+      blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol);
+      return;
+    }
+    pInfo->srcRowIndex++;
+  }
+  doFillResults(pOperator, pFillSup, pFillInfo, pBlock, tsCol, pInfo->srcRowIndex - 1, pRes);
+  blockDataUpdateTsWindow(pRes, pInfo->primaryTsCol);
+  blockDataCleanup(pInfo->pSrcBlock);
+}
+
+static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) {
+  SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;
+  void* pState = pOp->pTaskInfo->streamInfo.pState;
+
+  SSDataBlock*     pBlock = delRes;
+  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
+  SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
+  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+  SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
+  SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
+  SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
+  colDataSetVal(pStartCol, pBlock->info.rows, (const char*)&start, false);
+  colDataSetVal(pEndCol, pBlock->info.rows, (const char*)&end, false);
+  colDataSetNULL(pUidCol, pBlock->info.rows);
+  colDataSetVal(pGroupCol, pBlock->info.rows, (const char*)&groupId, false);
+  colDataSetNULL(pCalStartCol, pBlock->info.rows);
+  colDataSetNULL(pCalEndCol, pBlock->info.rows);
+
+  SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
+
+  void* tbname = NULL;
+  pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname);
+  if (tbname == NULL) {
+    colDataSetNULL(pTableCol, pBlock->info.rows);
+  } else {
+    char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
+    STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
+    colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
+    pAPI->stateStore.streamStateFreeVal(tbname);
+  }
+
+  pBlock->info.rows++;
+}
+
+static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId,
+                              SSDataBlock* delRes) {
+  SStreamFillOperatorInfo* pInfo = pOperator->info;
+  SStreamFillSupporter*    pFillSup = pInfo->pFillSup;
+  if (hasPrevWindow(pFillSup)) {
+    TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval);
+    buildDeleteRange(pOperator, start, endTs, groupId, delRes);
+  } else if (hasNextWindow(pFillSup)) {
+    TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval);
+    buildDeleteRange(pOperator, startTs, end, groupId, delRes);
+  } else {
+    buildDeleteRange(pOperator, startTs, endTs, groupId, delRes);
+  }
+}
+
+static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+  SStreamFillOperatorInfo* pInfo = pOperator->info;
+  getWindowFromDiscBuf(pOperator, startTs, groupId, pInfo->pFillSup);
+  setDeleteFillValueInfo(startTs, endTs, pInfo->pFillSup, pInfo->pFillInfo);
+  SWinKey key = {.ts = startTs, .groupId = groupId};
+  if (!pInfo->pFillInfo->needFill) {
+    pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
+    buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes);
+  } else {
+    STimeRange tw = {
+        .skey = startTs,
+        .ekey = endTs,
+        .groupId = groupId,
+    };
+    taosArrayPush(pInfo->pFillInfo->delRanges, &tw);
+    while (key.ts <= endTs) {
+      key.ts = taosTimeAdd(key.ts, pInfo->pFillSup->interval.sliding, pInfo->pFillSup->interval.slidingUnit,
+                           pInfo->pFillSup->interval.precision);
+      tSimpleHashPut(pInfo->pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0);
+    }
+  }
+}
+
+static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
+  SStreamFillOperatorInfo* pInfo = pOperator->info;
+  SStreamFillInfo*         pFillInfo = pInfo->pFillInfo;
+  int32_t                  size = taosArrayGetSize(pFillInfo->delRanges);
+  tSimpleHashClear(pInfo->pFillSup->pResMap);
+  for (; pFillInfo->delIndex < size; pFillInfo->delIndex++) {
+    STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex);
+    if (pInfo->pRes->info.id.groupId != 0 && pInfo->pRes->info.id.groupId != range->groupId) {
+      return;
+    }
+    getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup);
+    setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo);
+    if (pInfo->pFillInfo->needFill) {
+      doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes);
+      pInfo->pRes->info.id.groupId = range->groupId;
+    }
+    SWinKey key = {.ts = range->skey, .groupId = range->groupId};
+    pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
+  }
+}
+
+static void doDeleteFillResult(SOperatorInfo* pOperator) {
+  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
+
+  SStreamFillOperatorInfo* pInfo = pOperator->info;
+  SStreamFillInfo*         pFillInfo = pInfo->pFillInfo;
+  SSDataBlock*             pBlock = pInfo->pSrcDelBlock;
+
+  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  TSKEY*           tsStarts = (TSKEY*)pStartCol->pData;
+  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+  uint64_t*        groupIds = (uint64_t*)pGroupCol->pData;
+  while (pInfo->srcDelRowIndex < pBlock->info.rows) {
+    TSKEY            ts = tsStarts[pInfo->srcDelRowIndex];
+    TSKEY            endTs = ts;
+    uint64_t         groupId = groupIds[pInfo->srcDelRowIndex];
+    SWinKey          key = {.ts = ts, .groupId = groupId};
+    SStreamStateCur* pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &key);
+
+    if (!pCur) {
+      pInfo->srcDelRowIndex++;
+      continue;
+    }
+
+    SWinKey nextKey = {.groupId = groupId, .ts = ts};
+    while (pInfo->srcDelRowIndex < pBlock->info.rows) {
+      TSKEY    delTs = tsStarts[pInfo->srcDelRowIndex];
+      uint64_t delGroupId = groupIds[pInfo->srcDelRowIndex];
+      int32_t  code = TSDB_CODE_SUCCESS;
+      if (groupId != delGroupId) {
+        break;
+      }
+      if (delTs > nextKey.ts) {
+        break;
+      }
+
+      SWinKey delKey = {.groupId = delGroupId, .ts = delTs};
+      if (delTs == nextKey.ts) {
+        code = pAPI->stateStore.streamStateCurNext(pOperator->pTaskInfo->streamInfo.pState, pCur);
+        if (code == TSDB_CODE_SUCCESS) {
+          code = pAPI->stateStore.streamStateGetGroupKVByCur(pCur, &nextKey, NULL, NULL);
+        }
+        // ts will be deleted later
+        if (delTs != ts) {
+          pAPI->stateStore.streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &delKey);
+          pAPI->stateStore.streamStateFreeCur(pCur);
+          pCur = pAPI->stateStore.streamStateGetAndCheckCur(pOperator->pTaskInfo->streamInfo.pState, &nextKey);
+        }
+        endTs = TMAX(delTs, nextKey.ts - 1);
+        if (code != TSDB_CODE_SUCCESS) {
+          break;
+        }
+      }
+      pInfo->srcDelRowIndex++;
+    }
+
+    pAPI->stateStore.streamStateFreeCur(pCur);
+    doDeleteFillResultImpl(pOperator, ts, endTs, groupId);
+  }
+
+  pFillInfo->current = pFillInfo->end + 1;
+}
+
+static void resetStreamFillInfo(SStreamFillOperatorInfo* pInfo) {
+  tSimpleHashClear(pInfo->pFillSup->pResMap);
+  pInfo->pFillSup->hasDelete = false;
+  taosArrayClear(pInfo->pFillInfo->delRanges);
+  pInfo->pFillInfo->delIndex = 0;
+}
+
+static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pSrcBlock, SSDataBlock* pDstBlock) {
+  SStreamFillOperatorInfo* pInfo = pOperator->info;
+  SExprSupp*               pSup = &pOperator->exprSupp;
+
+  blockDataCleanup(pDstBlock);
+  blockDataEnsureCapacity(pDstBlock, pSrcBlock->info.rows);
+  setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
+  projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL);
+
+  pDstBlock->info.rows = 0;
+  pSup = &pInfo->pFillSup->notFillExprSup;
+  setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
+  projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL);
+  pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId;
+
+  blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol);
+}
+
+static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
+  SStreamFillOperatorInfo* pInfo = pOperator->info;
+  SExecTaskInfo*           pTaskInfo = pOperator->pTaskInfo;
+
+  if (pOperator->status == OP_EXEC_DONE) {
+    return NULL;
+  }
+  blockDataCleanup(pInfo->pRes);
+  if (hasRemainCalc(pInfo->pFillInfo) ||
+      (pInfo->pFillInfo->pos != FILL_POS_INVALID && pInfo->pFillInfo->needFill == true)) {
+    doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes);
+    if (pInfo->pRes->info.rows > 0) {
+      printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+      return pInfo->pRes;
+    }
+  }
+  if (pOperator->status == OP_RES_TO_RETURN) {
+    doDeleteFillFinalize(pOperator);
+    if (pInfo->pRes->info.rows > 0) {
+      printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+      return pInfo->pRes;
+    }
+    setOperatorCompleted(pOperator);
+    resetStreamFillInfo(pInfo);
+    return NULL;
+  }
+
+  SSDataBlock*   fillResult = NULL;
+  SOperatorInfo* downstream = pOperator->pDownstream[0];
+  while (1) {
+    if (pInfo->srcRowIndex >= pInfo->pSrcBlock->info.rows || pInfo->pSrcBlock->info.rows == 0) {
+      // If there are delete datablocks, we receive  them first.
+      SSDataBlock* pBlock = getNextBlockFromDownstream(pOperator, 0);
+      if (pBlock == NULL) {
+        pOperator->status = OP_RES_TO_RETURN;
+        pInfo->pFillInfo->preRowKey = INT64_MIN;
+        if (pInfo->pRes->info.rows > 0) {
+          printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+          return pInfo->pRes;
+        }
+        break;
+      }
+      printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
+
+      if (pInfo->pFillInfo->curGroupId != pBlock->info.id.groupId) {
+        pInfo->pFillInfo->curGroupId = pBlock->info.id.groupId;
+        pInfo->pFillInfo->preRowKey = INT64_MIN;
+      }
+
+      pInfo->pFillSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
+      if (pInfo->pFillSup->winRange.ekey <= 0) {
+        pInfo->pFillSup->winRange.ekey = INT64_MAX;
+      }
+
+      switch (pBlock->info.type) {
+        case STREAM_RETRIEVE:
+          return pBlock;
+        case STREAM_DELETE_RESULT: {
+          pInfo->pSrcDelBlock = pBlock;
+          pInfo->srcDelRowIndex = 0;
+          blockDataCleanup(pInfo->pDelRes);
+          pInfo->pFillSup->hasDelete = true;
+          doDeleteFillResult(pOperator);
+          if (pInfo->pDelRes->info.rows > 0) {
+            printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+            return pInfo->pDelRes;
+          }
+          continue;
+        } break;
+        case STREAM_NORMAL:
+        case STREAM_INVALID: 
+        case STREAM_PULL_DATA: {
+          doApplyStreamScalarCalculation(pOperator, pBlock, pInfo->pSrcBlock);
+          memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
+          pInfo->srcRowIndex = -1;
+        } break;
+        case STREAM_CHECKPOINT:
+        case STREAM_CREATE_CHILD_TABLE: {
+          return pBlock;
+        } break;
+        default:
+          ASSERTS(false, "invalid SSDataBlock type");
+      }
+    }
+
+    doStreamFillImpl(pOperator);
+    doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, &pInfo->matchInfo);
+    memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
+    pOperator->resultInfo.totalRows += pInfo->pRes->info.rows;
+    if (pInfo->pRes->info.rows > 0) {
+      break;
+    }
+  }
+  if (pOperator->status == OP_RES_TO_RETURN) {
+    doDeleteFillFinalize(pOperator);
+  }
+
+  if (pInfo->pRes->info.rows == 0) {
+    setOperatorCompleted(pOperator);
+    resetStreamFillInfo(pInfo);
+    return NULL;
+  }
+
+  pOperator->resultInfo.totalRows += pInfo->pRes->info.rows;
+  printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+  return pInfo->pRes;
+}
+
+static int32_t initResultBuf(SStreamFillSupporter* pFillSup) {
+  pFillSup->rowSize = sizeof(SResultCellData) * pFillSup->numOfAllCols;
+  for (int i = 0; i < pFillSup->numOfAllCols; i++) {
+    SFillColInfo* pCol = &pFillSup->pAllColInfo[i];
+    SResSchema*   pSchema = &pCol->pExpr->base.resSchema;
+    pFillSup->rowSize += pSchema->bytes;
+  }
+  pFillSup->next.key = INT64_MIN;
+  pFillSup->nextNext.key = INT64_MIN;
+  pFillSup->prev.key = INT64_MIN;
+  pFillSup->cur.key = INT64_MIN;
+  pFillSup->next.pRowVal = NULL;
+  pFillSup->nextNext.pRowVal = NULL;
+  pFillSup->prev.pRowVal = NULL;
+  pFillSup->cur.pRowVal = NULL;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNode, SInterval* pInterval,
+                                               SExprInfo* pFillExprInfo, int32_t numOfFillCols, SStorageAPI* pAPI) {
+  SStreamFillSupporter* pFillSup = taosMemoryCalloc(1, sizeof(SStreamFillSupporter));
+  if (!pFillSup) {
+    return NULL;
+  }
+  pFillSup->numOfFillCols = numOfFillCols;
+  int32_t    numOfNotFillCols = 0;
+  SExprInfo* noFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols);
+  pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, noFillExprInfo, numOfNotFillCols,
+                                            (const SNodeListNode*)(pPhyFillNode->pValues));
+  pFillSup->type = convertFillType(pPhyFillNode->mode);
+  pFillSup->numOfAllCols = pFillSup->numOfFillCols + numOfNotFillCols;
+  pFillSup->interval = *pInterval;
+  pFillSup->pAPI = pAPI;
+
+  int32_t code = initResultBuf(pFillSup);
+  if (code != TSDB_CODE_SUCCESS) {
+    destroyStreamFillSupporter(pFillSup);
+    return NULL;
+  }
+
+  SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols);
+  code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore);
+  if (code != TSDB_CODE_SUCCESS) {
+    destroyStreamFillSupporter(pFillSup);
+    return NULL;
+  }
+
+  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
+  pFillSup->pResMap = tSimpleHashInit(16, hashFn);
+  pFillSup->hasDelete = false;
+  return pFillSup;
+}
+
+SStreamFillInfo* initStreamFillInfo(SStreamFillSupporter* pFillSup, SSDataBlock* pRes) {
+  SStreamFillInfo* pFillInfo = taosMemoryCalloc(1, sizeof(SStreamFillInfo));
+  pFillInfo->start = INT64_MIN;
+  pFillInfo->current = INT64_MIN;
+  pFillInfo->end = INT64_MIN;
+  pFillInfo->preRowKey = INT64_MIN;
+  pFillInfo->needFill = false;
+  pFillInfo->pLinearInfo = taosMemoryCalloc(1, sizeof(SStreamFillLinearInfo));
+  pFillInfo->pLinearInfo->hasNext = false;
+  pFillInfo->pLinearInfo->nextEnd = INT64_MIN;
+  pFillInfo->pLinearInfo->pEndPoints = NULL;
+  pFillInfo->pLinearInfo->pNextEndPoints = NULL;
+  if (pFillSup->type == TSDB_FILL_LINEAR) {
+    pFillInfo->pLinearInfo->pEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint));
+    pFillInfo->pLinearInfo->pNextEndPoints = taosArrayInit(pFillSup->numOfAllCols, sizeof(SPoint));
+    for (int32_t i = 0; i < pFillSup->numOfAllCols; i++) {
+      SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i);
+      SPoint value = {0};
+      value.val = taosMemoryCalloc(1, pColData->info.bytes);
+      taosArrayPush(pFillInfo->pLinearInfo->pEndPoints, &value);
+
+      value.val = taosMemoryCalloc(1, pColData->info.bytes);
+      taosArrayPush(pFillInfo->pLinearInfo->pNextEndPoints, &value);
+    }
+  }
+  pFillInfo->pLinearInfo->winIndex = 0;
+
+  pFillInfo->pResRow = NULL;
+  if (pFillSup->type == TSDB_FILL_SET_VALUE || pFillSup->type == TSDB_FILL_SET_VALUE_F ||
+      pFillSup->type == TSDB_FILL_NULL || pFillSup->type == TSDB_FILL_NULL_F) {
+    pFillInfo->pResRow = taosMemoryCalloc(1, sizeof(SResultRowData));
+    pFillInfo->pResRow->key = INT64_MIN;
+    pFillInfo->pResRow->pRowVal = taosMemoryCalloc(1, pFillSup->rowSize);
+    for (int32_t i = 0; i < pFillSup->numOfAllCols; ++i) {
+      SColumnInfoData* pColData = taosArrayGet(pRes->pDataBlock, i);
+      SResultCellData* pCell = getResultCell(pFillInfo->pResRow, i);
+      pCell->bytes = pColData->info.bytes;
+      pCell->type = pColData->info.type;
+    }
+  }
+
+  pFillInfo->type = pFillSup->type;
+  pFillInfo->delRanges = taosArrayInit(16, sizeof(STimeRange));
+  pFillInfo->delIndex = 0;
+  pFillInfo->curGroupId = 0;
+  return pFillInfo;
+}
+
+SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode,
+                                            SExecTaskInfo* pTaskInfo) {
+  SStreamFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFillOperatorInfo));
+  SOperatorInfo*           pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  if (pInfo == NULL || pOperator == NULL) {
+    goto _error;
+  }
+
+  SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval;
+  int32_t    numOfFillCols = 0;
+  SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols);
+  pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI);
+  if (!pInfo->pFillSup) {
+    goto _error;
+  }
+
+  initResultSizeInfo(&pOperator->resultInfo, 4096);
+  pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
+  pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
+  blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
+  blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity);
+
+  pInfo->pFillInfo = initStreamFillInfo(pInfo->pFillSup, pInfo->pRes);
+  if (!pInfo->pFillInfo) {
+    goto _error;
+  }
+
+  if (pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE || pInfo->pFillInfo->type == TSDB_FILL_SET_VALUE_F) {
+    for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) {
+      SFillColInfo*    pFillCol = pInfo->pFillSup->pAllColInfo + i;
+      int32_t          slotId = GET_DEST_SLOT_ID(pFillCol);
+      SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId);
+      SVariant*        pVar = &(pFillCol->fillVal);
+      if (pCell->type == TSDB_DATA_TYPE_FLOAT) {
+        float v = 0;
+        GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
+        SET_TYPED_DATA(pCell->pData, pCell->type, v);
+      } else if (IS_FLOAT_TYPE(pCell->type)) {
+        double v = 0;
+        GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
+        SET_TYPED_DATA(pCell->pData, pCell->type, v);
+      } else if (IS_INTEGER_TYPE(pCell->type)) {
+        int64_t v = 0;
+        GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
+        SET_TYPED_DATA(pCell->pData, pCell->type, v);
+      } else {
+        pCell->isNull = true;
+      }
+    }
+  } else if (pInfo->pFillInfo->type == TSDB_FILL_NULL || pInfo->pFillInfo->type == TSDB_FILL_NULL_F) {
+    for (int32_t i = 0; i < pInfo->pFillSup->numOfAllCols; ++i) {
+      SFillColInfo*    pFillCol = pInfo->pFillSup->pAllColInfo + i;
+      int32_t          slotId = GET_DEST_SLOT_ID(pFillCol);
+      SResultCellData* pCell = getResultCell(pInfo->pFillInfo->pResRow, slotId);
+      pCell->isNull = true;
+    }
+  }
+
+  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
+  blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity);
+
+  pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId;
+  pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId;
+
+  int32_t numOfOutputCols = 0;
+  int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc,
+                                     &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  code = filterInitFromNode((SNode*)pPhyFillNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI.functionStore);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  pInfo->srcRowIndex = -1;
+  setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo,
+                  pTaskInfo);
+  pOperator->fpSet =
+      createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
+  setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
+
+  code = appendDownstream(pOperator, &downstream, 1);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+  return pOperator;
+
+_error:
+  destroyStreamFillOperatorInfo(pInfo);
+  taosMemoryFreeClear(pOperator);
+  pTaskInfo->code = code;
+  return NULL;
+}

From 95358f374e74f0ab9369339521c55f7eb6f5bd87 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Thu, 7 Mar 2024 19:02:12 +0800
Subject: [PATCH 166/201] fix:[TS-4540]fix memory leak

---
 tests/parallel_test/cases.task | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 2ab001621a..481e111715 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -228,7 +228,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py
-,,n,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py
+,,n,system-test,python3 ./test.py -f 7-tmq/tmq_taosx.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_replay.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py
 ,,n,system-test,python3 ./test.py -f 7-tmq/tmq_offset.py

From aef11bf3a82cf8fa0f852cd5ba72abdc0532ebb6 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 8 Mar 2024 00:25:08 +0800
Subject: [PATCH 167/201] fix(stream): add one more status for check.

---
 source/libs/stream/src/streamExec.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c
index 27cd98aac6..cb51d90fba 100644
--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@@ -353,7 +353,8 @@ int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask) {
   if (pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE) {
     ASSERT(status == TASK_STATUS__HALT || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP);
   } else {
-    ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP);
+    ASSERT(status == TASK_STATUS__READY || status == TASK_STATUS__PAUSE || status == TASK_STATUS__DROPPING ||
+           status == TASK_STATUS__STOP);
     int32_t code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
     if (code != TSDB_CODE_SUCCESS) {
       stError("s-task:%s halt stream task:%s failed, code:%s not transfer state to stream task", id,

From 51faf948824c7145f896a5a27f7cc6fe31eef2d8 Mon Sep 17 00:00:00 2001
From: liuyao <38781207+54liuyao@users.noreply.github.com>
Date: Fri, 8 Mar 2024 10:36:48 +0800
Subject: [PATCH 168/201] Update 12-distinguished.md

---
 docs/zh/12-taos-sql/12-distinguished.md | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/docs/zh/12-taos-sql/12-distinguished.md b/docs/zh/12-taos-sql/12-distinguished.md
index 8ae3f900f4..62ceef40c5 100755
--- a/docs/zh/12-taos-sql/12-distinguished.md
+++ b/docs/zh/12-taos-sql/12-distinguished.md
@@ -49,6 +49,7 @@ window_clause: {
   | STATE_WINDOW(col)
   | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [FILL(fill_mod_and_val)]
   | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
+  | COUNT_WINDOW(count_val[, sliding_val])
 }
 ```
 
@@ -180,6 +181,18 @@ select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c
 
 ![TDengine Database 事件窗口示意图](./event_window.webp)
 
+### 计数窗口
+
+计数窗口按固定的数据行数来划分窗口。默认将数据按时间戳排序，再按照count_val的值，将数据划分为多个窗口，然后做聚合计算。count_val表示每个count window包含的最大数据行数，总数据行数不能整除count_val时，最后一个窗口的行数会小于count_val。sliding_val是常量，表示窗口滑动的数量，类似于 interval的SLIDING。
+
+以下面的 SQL 语句为例，计数窗口切分如图所示：
+```sql
+select _wstart, _wend, count(*) from t count_window(4);
+```
+
+![count_window](https://github.com/taosdata/TDengine/assets/38781207/aedea174-2aad-4767-8f7f-9261e0a72d5d)
+
+
 ### 时间戳伪列
 
 窗口聚合查询结果中，如果 SQL 语句中没有指定输出查询结果中的时间戳列，那么最终结果中不会自动包含窗口的时间列信息。如果需要在结果中输出聚合结果所对应的时间窗口信息，需要在 SELECT 子句中使用时间戳相关的伪列: 时间窗口起始时间 (\_WSTART), 时间窗口结束时间 (\_WEND), 时间窗口持续时间 (\_WDURATION), 以及查询整体窗口相关的伪列: 查询窗口起始时间(\_QSTART) 和查询窗口结束时间(\_QEND)。需要注意的是时间窗口起始时间和结束时间均是闭区间，时间窗口持续时间是数据当前时间分辨率下的数值。例如，如果当前数据库的时间分辨率是毫秒，那么结果中 500 就表示当前时间窗口的持续时间是 500毫秒 (500 ms)。

From a42a897c85d662ae0036feb5d80ecaa59527d9a5 Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Fri, 8 Mar 2024 10:47:32 +0800
Subject: [PATCH 169/201] enhance: assert the region offset is keep increasing

---
 source/libs/executor/src/tsort.c | 42 +++++++++++++++-----------------
 1 file changed, 20 insertions(+), 22 deletions(-)

diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 6d5886730f..2c8dc7fb04 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -927,7 +927,7 @@ void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHa
     int32_t regionId = *(int32_t*)tsortGetValue(pTupleHandle, 1);
     int32_t offset = *(int32_t*)tsortGetValue(pTupleHandle, 2);
     int32_t length = *(int32_t*)tsortGetValue(pTupleHandle, 3);
-
+    
     char* buf = NULL;
     bool bFreeRow = false;
     getRowBufFromExtMemFile(pHandle, regionId, offset, length, &buf, &bFreeRow);
@@ -1047,32 +1047,30 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
     }
     pRegion->bufLen = readBytes;
   }
-  // TODO: ASSERT(pRegion->offset < tupleOffset);
+  ASSERT(pRegion->bufRegOffset <= tupleOffset);
   if (pRegion->bufRegOffset + pRegion->bufLen >= tupleOffset + rowLen) {
     *pFreeRow = false;
     *ppRow = pRegion->buf + tupleOffset - pRegion->bufRegOffset;
   } else {
-      *ppRow = taosMemoryMalloc(rowLen);
-      if (*ppRow == NULL) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-      int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset);
-      memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset,
-             szThisBlock);
-      tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
-      int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen));
-      int ret = fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
-      if (ret != 1) {
-        taosMemoryFreeClear(*ppRow);
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        return terrno;
-      }
-      memcpy(*ppRow + szThisBlock, pRegion->buf, rowLen - szThisBlock);
-      *pFreeRow = true;
-      pRegion->bufRegOffset += pRegion->bufLen;
-      pRegion->bufLen = readBytes;
+    *ppRow = taosMemoryMalloc(rowLen);
+    if (*ppRow == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset);
+    memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset, szThisBlock);
+    tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
+    int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen));
+    int     ret = fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
+    if (ret != 1) {
+      taosMemoryFreeClear(*ppRow);
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return terrno;
+    }
+    memcpy(*ppRow + szThisBlock, pRegion->buf, rowLen - szThisBlock);
+    *pFreeRow = true;
+    pRegion->bufRegOffset += pRegion->bufLen;
+    pRegion->bufLen = readBytes;
   }
-  //TODO: free region memory
   return TSDB_CODE_SUCCESS;
 }
 

From 08d3145fdcc4ff6833c5347baaca852a81f5b295 Mon Sep 17 00:00:00 2001
From: liuyao <38781207+54liuyao@users.noreply.github.com>
Date: Fri, 8 Mar 2024 11:00:56 +0800
Subject: [PATCH 170/201] Update 12-distinguished.md

---
 docs/zh/12-taos-sql/12-distinguished.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/zh/12-taos-sql/12-distinguished.md b/docs/zh/12-taos-sql/12-distinguished.md
index 62ceef40c5..f73919a970 100755
--- a/docs/zh/12-taos-sql/12-distinguished.md
+++ b/docs/zh/12-taos-sql/12-distinguished.md
@@ -190,7 +190,8 @@ select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c
 select _wstart, _wend, count(*) from t count_window(4);
 ```
 
-![count_window](https://github.com/taosdata/TDengine/assets/38781207/aedea174-2aad-4767-8f7f-9261e0a72d5d)
+![count_window](https://github.com/taosdata/TDengine/assets/38781207/ff86278e-cbff-477f-a866-6e5cc150b712)
+
 
 
 ### 时间戳伪列

From 848be2ecb39a963b8ff2238bf3c70e45d608fe6b Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Mon, 26 Feb 2024 17:26:56 +0800
Subject: [PATCH 171/201] fix: monitor client config

---
 source/common/src/tglobal.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index c5a26c5c10..3024dc99fd 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -538,8 +538,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
     return -1;
   if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
 
-  if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_CLIENT, CFG_DYN_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1;
   return 0;
 }
 

From 40f430baf18e7de96fa3a45305ad23d67b927973 Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Mon, 26 Feb 2024 17:55:07 +0800
Subject: [PATCH 172/201] fix: dynamic options for client monitor

---
 source/common/src/tglobal.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 3024dc99fd..c9c86ce9b5 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -538,7 +538,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
     return -1;
   if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
 
-  if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_CLIENT, CFG_DYN_SERVER) != 0) return -1;
+  if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1;
   return 0;
 }
@@ -1626,6 +1626,10 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
         tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024);
         uInfo("%s set to %" PRId64, name, tsLogSpace.reserved);
         matched = true;
+      } else if (strcasecmp("monitor", name) == 0) {
+        tsEnableMonitor = pItem->bval;
+        uInfo("%s set to %d", name, tsEnableMonitor);
+        matched = true;
       }
       break;
     }

From c0b8c2f1c03a86be9a756db757f4a13954d0757f Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Mon, 26 Feb 2024 20:39:12 +0800
Subject: [PATCH 173/201] fix: duplicateConfiguration

---
 source/common/src/tglobal.c | 8 ++++----
 source/util/src/tconfig.c   | 9 +++++++++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index c9c86ce9b5..af2eb9cad3 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -538,8 +538,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
     return -1;
   if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
 
-  if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
-  if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
+  if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
   return 0;
 }
 
@@ -691,8 +691,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
-  if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
-  if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
+  if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
+  if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0)
     return -1;
   if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c
index ad3c766510..27a8edc009 100644
--- a/source/util/src/tconfig.c
+++ b/source/util/src/tconfig.c
@@ -449,6 +449,15 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
     return -1;
   }
 
+  int size = pCfg->array->size;
+  for (int32_t i = 0; i < size; ++i) {
+    SConfigItem *existItem = taosArrayGet(pCfg->array, i);
+    char xx[45];
+    if (existItem != NULL && strcmp(existItem->name, pItem->name) == 0) {
+      return 0;
+    }
+  }
+
   int32_t len = strlen(name);
   char    lowcaseName[CFG_NAME_MAX_LEN + 1] = {0};
   strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len));

From 71e4cb92fdbb2911801c7a10229d379bca5cd5b8 Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Tue, 27 Feb 2024 09:49:28 +0800
Subject: [PATCH 174/201] fix: mem leak

---
 source/util/src/tconfig.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c
index 27a8edc009..0253f1a80c 100644
--- a/source/util/src/tconfig.c
+++ b/source/util/src/tconfig.c
@@ -442,13 +442,6 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p
 }
 
 static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
-  pItem->stype = CFG_STYPE_DEFAULT;
-  pItem->name = taosStrdup(name);
-  if (pItem->name == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
   int size = pCfg->array->size;
   for (int32_t i = 0; i < size; ++i) {
     SConfigItem *existItem = taosArrayGet(pCfg->array, i);
@@ -458,6 +451,13 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
     }
   }
 
+  pItem->stype = CFG_STYPE_DEFAULT;
+  pItem->name = taosStrdup(name);
+  if (pItem->name == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
   int32_t len = strlen(name);
   char    lowcaseName[CFG_NAME_MAX_LEN + 1] = {0};
   strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len));

From ec5b59c8751fe5365545459e3aed71d4bc9215a9 Mon Sep 17 00:00:00 2001
From: factosea <285808407@qq.com>
Date: Tue, 27 Feb 2024 21:11:31 +0800
Subject: [PATCH 175/201] fix: add config item

---
 source/util/src/tconfig.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c
index 0253f1a80c..e98c264df5 100644
--- a/source/util/src/tconfig.c
+++ b/source/util/src/tconfig.c
@@ -442,15 +442,6 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p
 }
 
 static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
-  int size = pCfg->array->size;
-  for (int32_t i = 0; i < size; ++i) {
-    SConfigItem *existItem = taosArrayGet(pCfg->array, i);
-    char xx[45];
-    if (existItem != NULL && strcmp(existItem->name, pItem->name) == 0) {
-      return 0;
-    }
-  }
-
   pItem->stype = CFG_STYPE_DEFAULT;
   pItem->name = taosStrdup(name);
   if (pItem->name == NULL) {
@@ -458,6 +449,15 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
     return -1;
   }
 
+  int size = pCfg->array->size;
+  for (int32_t i = 0; i < size; ++i) {
+    SConfigItem *existItem = taosArrayGet(pCfg->array, i);
+    if (existItem != NULL && strcmp(existItem->name, pItem->name) == 0) {
+      taosMemoryFree(pItem->name);
+      return 0;
+    }
+  }
+
   int32_t len = strlen(name);
   char    lowcaseName[CFG_NAME_MAX_LEN + 1] = {0};
   strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len));

From b76831e3fe79ae0da7264d4b547e082ddc030889 Mon Sep 17 00:00:00 2001
From: facetosea <25808407@qq.com>
Date: Fri, 8 Mar 2024 09:41:27 +0800
Subject: [PATCH 176/201] delete duplicate key for server config

---
 source/common/src/tglobal.c | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index af2eb9cad3..af9d2c37df 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -502,7 +502,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
     return -1;
   if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
-  if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
+  if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1;
   if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, CFG_SCOPE_CLIENT,
                   CFG_DYN_NONE) != 0)
     return -1;
@@ -600,18 +600,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
     return -1;
   if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
-  tsNumOfRpcThreads = tsNumOfCores / 2;
-  tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS);
-  if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
-
-  tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 10000);
-  if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0)
-    return -1;
-
-  tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000);
-  if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0)
-    return -1;
-
   tsNumOfCommitThreads = tsNumOfCores / 2;
   tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4);
   if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
@@ -691,9 +679,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
-  if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
-  if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0)
-    return -1;
   if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@@ -707,7 +692,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
-  if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
@@ -814,8 +798,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
     return -1;
   if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
-  if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
-
   // GRANT_CFG_ADD;
   return 0;
 }

From 3de14139742f7cc10b25556c7211fa4b0ad218df Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Fri, 8 Mar 2024 13:38:51 +0800
Subject: [PATCH 177/201] test(stream): add some test cases to inc coverage.

---
 source/common/src/tglobal.c                 |  3 +++
 source/dnode/mnode/impl/src/mndStreamUtil.c |  6 +++++-
 source/util/src/tconfig.c                   |  1 +
 tests/script/sh/deploy.sh                   |  5 +++--
 tests/script/tsim/stream/pauseAndResume.sim | 19 +++++++++++++++++--
 5 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index fde8313228..89d3b88955 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -1353,6 +1353,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
     if (taosAddClientLogCfg(tsCfg) != 0) return -1;
     if (taosAddServerLogCfg(tsCfg) != 0) return -1;
   }
+
   taosAddSystemCfg(tsCfg);
 
   if (taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {
@@ -1379,7 +1380,9 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
     if (taosSetTfsCfg(tsCfg) != 0) return -1;
     if (taosSetS3Cfg(tsCfg) != 0) return -1;
   }
+
   taosSetSystemCfg(tsCfg);
+
   if (taosSetFileHandlesLimit() != 0) return -1;
 
   taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32);
diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c
index 2b8fcee9fd..a124b4052c 100644
--- a/source/dnode/mnode/impl/src/mndStreamUtil.c
+++ b/source/dnode/mnode/impl/src/mndStreamUtil.c
@@ -115,6 +115,7 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
 
     char buf[256] = {0};
     EPSET_TO_STR(&entry.epset, buf);
+
     mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
     taosArrayPush(pVgroupListSnapshot, &entry);
     sdbRelease(pSdb, pVgroup);
@@ -300,7 +301,10 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
     return code;
   }
 
-  mDebug("pause node:%d, epset:%d", pTask->info.nodeId, epset.numOfEps);
+  char buf[256] = {0};
+  EPSET_TO_STR(&epset, buf);
+  mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf);
+
   code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0);
   if (code != 0) {
     taosMemoryFree(pReq);
diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c
index ad3c766510..e072ee2276 100644
--- a/source/util/src/tconfig.c
+++ b/source/util/src/tconfig.c
@@ -457,6 +457,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
     if (pItem->dtype == CFG_DTYPE_STRING) {
       taosMemoryFree(pItem->str);
     }
+
     taosMemoryFree(pItem->name);
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh
index 9af2525c74..c6bd78336c 100755
--- a/tests/script/sh/deploy.sh
+++ b/tests/script/sh/deploy.sh
@@ -113,11 +113,11 @@ echo "firstEp                ${HOSTNAME}:7100"   >> $TAOS_CFG
 echo "secondEp               ${HOSTNAME}:7200"   >> $TAOS_CFG
 echo "fqdn                   ${HOSTNAME}"        >> $TAOS_CFG
 echo "serverPort             ${NODE}"            >> $TAOS_CFG
-echo "supportVnodes          1024"                >> $TAOS_CFG
+echo "supportVnodes          1024"               >> $TAOS_CFG
 echo "statusInterval         1"                  >> $TAOS_CFG
 echo "dataDir                $DATA_DIR"          >> $TAOS_CFG
 echo "logDir                 $LOG_DIR"           >> $TAOS_CFG
-echo "debugFlag              135"                  >> $TAOS_CFG
+echo "debugFlag              135"                >> $TAOS_CFG
 echo "tmrDebugFlag           131"                >> $TAOS_CFG
 echo "uDebugFlag             143"                >> $TAOS_CFG
 echo "rpcDebugFlag           143"                >> $TAOS_CFG
@@ -143,4 +143,5 @@ echo "asyncLog               0"                  >> $TAOS_CFG
 echo "locale                 en_US.UTF-8"        >> $TAOS_CFG
 echo "telemetryReporting     0"                  >> $TAOS_CFG
 echo "querySmaOptimize       1"                  >> $TAOS_CFG
+echo "checkpointInterval     60"                 >> $TAOS_CFG
 echo " "                                         >> $TAOS_CFG  
diff --git a/tests/script/tsim/stream/pauseAndResume.sim b/tests/script/tsim/stream/pauseAndResume.sim
index 5eb9eef010..be89d8e235 100644
--- a/tests/script/tsim/stream/pauseAndResume.sim
+++ b/tests/script/tsim/stream/pauseAndResume.sim
@@ -8,16 +8,19 @@ sql connect
 print ===== step1
 sql drop stream if exists streams1;
 sql drop database if exists test;
-sql create database test  vgroups 10;
+sql create database test vgroups 3;
 sql use test;
 sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
 sql create table ts1 using st tags(1,1,1);
 sql create table ts2 using st tags(2,2,2);
 sql create table ts3 using st tags(3,2,2);
 sql create table ts4 using st tags(4,2,2);
+
 sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt1 as select  _wstart, count(*) c1, sum(a) c3 from st interval(10s);
 sleep 2000
 
+sql_error create stream stream1_same_dst into streamt1 as select _wstart, count(*) c1, sum(a) c3 from st interval(10s);
+
 sql pause stream streams1;
 
 sql insert into ts1 values(1648791213001,1,12,3,1.0);
@@ -102,6 +105,11 @@ if $data11 != 4 then
   goto loop1
 endi
 
+print ===== idle for 70 sec for checkpoint gen
+sleep 70000
+
+print ===== idle 60 sec completed , continue
+
 print ===== step 1 over
 
 print ===== step2
@@ -113,6 +121,12 @@ sql create table t1(ts timestamp, a int, b int , c int, d double);
 
 sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt2 as select  _wstart, count(*) c1, sum(a) c3 from t1 interval(10s);
 
+# duplicate stream
+sql_error create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt2 as select  _wstart, count(*) c1, sum(a) c3 from t1 interval(10s);
+sql create stream if not exists streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 watermark 1d into streamt2 as select  _wstart, count(*) c1, sum(a) c3 from t1 interval(10s);
+
+sleep 1000
+
 sql pause stream streams2;
 
 sql insert into t1 values(1648791213001,1,12,3,1.0);
@@ -237,8 +251,9 @@ print ===== step 2 over
 print ===== step3
 sql drop stream if exists streams3;
 sql drop database if exists test3;
-sql create database test3  vgroups 10;
+sql create database test3  vgroups 3;
 sql use test3;
+
 sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
 sql create table ts1 using st tags(1,1,1);
 sql create table ts2 using st tags(2,2,2);

From 3db2c1edf5e8aa7250d3c7d5dd54dd10435d812f Mon Sep 17 00:00:00 2001
From: shenglian zhou <shenglian_zhou@163.com>
Date: Fri, 8 Mar 2024 14:36:55 +0800
Subject: [PATCH 178/201] feat: add hint smalldata_ts_sort to disable row id
 sort

---
 include/common/ttokendef.h                 |  2 +-
 include/libs/nodes/plannodes.h             |  2 ++
 include/libs/nodes/querynodes.h            |  3 ++-
 source/libs/executor/src/scanoperator.c    |  2 +-
 source/libs/nodes/src/nodesCloneFuncs.c    |  2 ++
 source/libs/nodes/src/nodesCodeFuncs.c     | 16 +++++++++++++++-
 source/libs/nodes/src/nodesMsgFuncs.c      |  6 ++++++
 source/libs/parser/src/parAstCreater.c     | 11 +++++++++++
 source/libs/parser/src/parTokenizer.c      |  1 +
 source/libs/planner/inc/planInt.h          |  3 ++-
 source/libs/planner/src/planLogicCreater.c |  3 ++-
 source/libs/planner/src/planPhysiCreater.c |  1 +
 source/libs/planner/src/planUtil.c         | 14 +++++++++++++-
 13 files changed, 59 insertions(+), 7 deletions(-)

diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h
index 8f89857d33..020be9d447 100644
--- a/include/common/ttokendef.h
+++ b/include/common/ttokendef.h
@@ -380,7 +380,7 @@
 #define TK_SORT_FOR_GROUP   608
 #define TK_PARTITION_FIRST  609
 #define TK_PARA_TABLES_SORT  610
-
+#define TK_SMALLDATA_TS_SORT 611
 
 #define TK_NK_NIL 65535
 
diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h
index 0bc3ce04ef..cbf38102de 100644
--- a/include/libs/nodes/plannodes.h
+++ b/include/libs/nodes/plannodes.h
@@ -122,6 +122,7 @@ typedef struct SScanLogicNode {
   bool          isCountByTag;  // true if selectstmt hasCountFunc & part by tag/tbname
   SArray*       pFuncTypes; // for last, last_row
   bool          paraTablesSort; // for table merge scan
+  bool          smallDataTsSort; // disable row id sort for table merge scan
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -445,6 +446,7 @@ typedef struct STableScanPhysiNode {
   bool           filesetDelimited;
   bool           needCountEmptyTable;
   bool           paraTablesSort;
+  bool           smallDataTsSort;
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;
diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h
index 7ceb7e0278..97ac4ff3b9 100644
--- a/include/libs/nodes/querynodes.h
+++ b/include/libs/nodes/querynodes.h
@@ -128,7 +128,8 @@ typedef enum EHintOption {
   HINT_BATCH_SCAN,
   HINT_SORT_FOR_GROUP,
   HINT_PARTITION_FIRST,
-  HINT_PARA_TABLES_SORT
+  HINT_PARA_TABLES_SORT,
+  HINT_SMALLDATA_TS_SORT,
 } EHintOption;
 
 typedef struct SHintNode {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 1636cd21f0..f44f8756a4 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4349,7 +4349,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   initResultSizeInfo(&pOperator->resultInfo, 1024);
   pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
   blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
-  if (!hasLimit && blockDataGetRowSize(pInfo->pResBlock) >= 256) {
+  if (!hasLimit && blockDataGetRowSize(pInfo->pResBlock) >= 256 && !pTableScanNode->smallDataTsSort) {
     pInfo->bSortRowId = true;
   } else {
     pInfo->bSortRowId = false;
diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c
index 3f5ffcae32..453d927378 100644
--- a/source/libs/nodes/src/nodesCloneFuncs.c
+++ b/source/libs/nodes/src/nodesCloneFuncs.c
@@ -457,6 +457,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
   COPY_SCALAR_FIELD(isCountByTag);
   CLONE_OBJECT_FIELD(pFuncTypes, functParamClone);
   COPY_SCALAR_FIELD(paraTablesSort);
+  COPY_SCALAR_FIELD(smallDataTsSort);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -690,6 +691,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy
   COPY_SCALAR_FIELD(filesetDelimited);
   COPY_SCALAR_FIELD(needCountEmptyTable);
   COPY_SCALAR_FIELD(paraTablesSort);
+  COPY_SCALAR_FIELD(smallDataTsSort);
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c
index 689886c366..019ef6f18b 100644
--- a/source/libs/nodes/src/nodesCodeFuncs.c
+++ b/source/libs/nodes/src/nodesCodeFuncs.c
@@ -699,6 +699,7 @@ static const char* jkScanLogicPlanGroupTags = "GroupTags";
 static const char* jkScanLogicPlanOnlyMetaCtbIdx = "OnlyMetaCtbIdx";
 static const char* jkScanLogicPlanFilesetDelimited = "FilesetDelimited";
 static const char* jkScanLogicPlanParaTablesSort = "ParaTablesSort";
+static const char* jkScanLogicPlanSmallDataTsSort = "SmallDataTsSort";
 
 static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
   const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
@@ -749,6 +750,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddBoolToObject(pJson, jkScanLogicPlanParaTablesSort, pNode->paraTablesSort);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkScanLogicPlanSmallDataTsSort, pNode->paraTablesSort);
+  }
   return code;
 }
 
@@ -800,7 +804,10 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
     code = tjsonGetBoolValue(pJson, jkScanLogicPlanFilesetDelimited, &pNode->filesetDelimited);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonGetBoolValue(pJson, jkScanLogicPlanParaTablesSort, &pNode->paraTablesSort);
+    code = tjsonGetBoolValue(pJson, jkScanLogicPlanParaTablesSort, &pNode->smallDataTsSort);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkScanLogicPlanSmallDataTsSort, &pNode->smallDataTsSort);
   }
   return code;
 }
@@ -1896,6 +1903,7 @@ static const char* jkTableScanPhysiPlanIgnoreUpdate = "IgnoreUpdate";
 static const char* jkTableScanPhysiPlanFilesetDelimited = "FilesetDelimited";
 static const char* jkTableScanPhysiPlanNeedCountEmptyTable = "NeedCountEmptyTable";
 static const char* jkTableScanPhysiPlanParaTablesSort = "ParaTablesSort";
+static const char* jkTableScanPhysiPlanSmallDataTsSort = "SmallDataTsSort";
 
 static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
   const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj;
@@ -1973,6 +1981,9 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanParaTablesSort, pNode->paraTablesSort);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanSmallDataTsSort, pNode->smallDataTsSort);
+  }
   return code;
 }
 
@@ -2052,6 +2063,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanParaTablesSort, &pNode->paraTablesSort);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanSmallDataTsSort, &pNode->smallDataTsSort);
+  }
   return code;
 }
 
diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c
index 357abc2858..95a5c2f51e 100644
--- a/source/libs/nodes/src/nodesMsgFuncs.c
+++ b/source/libs/nodes/src/nodesMsgFuncs.c
@@ -2188,6 +2188,9 @@ static int32_t physiTableScanNodeInlineToMsg(const void* pObj, STlvEncoder* pEnc
   if (TSDB_CODE_SUCCESS == code) {
     code = tlvEncodeValueBool(pEncoder, pNode->paraTablesSort);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tlvEncodeValueBool(pEncoder, pNode->smallDataTsSort);
+  }
   return code;
 }
 
@@ -2275,6 +2278,9 @@ static int32_t msgToPhysiTableScanNodeInline(STlvDecoder* pDecoder, void* pObj)
   if (TSDB_CODE_SUCCESS == code) {
     code = tlvDecodeValueBool(pDecoder, &pNode->paraTablesSort);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tlvDecodeValueBool(pDecoder, &pNode->smallDataTsSort);
+  }
   return code;
 }
 
diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c
index 1d6c5e800e..74a7934985 100644
--- a/source/libs/parser/src/parAstCreater.c
+++ b/source/libs/parser/src/parAstCreater.c
@@ -404,6 +404,9 @@ bool addHintNodeToList(SAstCreateContext* pCxt, SNodeList** ppHintList, EHintOpt
     case HINT_PARA_TABLES_SORT:
       if (paramNum > 0 || hasHint(*ppHintList, HINT_PARA_TABLES_SORT)) return true;
       break;
+    case HINT_SMALLDATA_TS_SORT:
+      if (paramNum > 0 || hasHint(*ppHintList, HINT_SMALLDATA_TS_SORT)) return true;
+      break;
     default:
       return true;
   }
@@ -490,6 +493,14 @@ SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral) {
         }
         opt = HINT_PARA_TABLES_SORT;
         break;
+      case TK_SMALLDATA_TS_SORT:
+        lastComma = false;
+        if (0 != opt || inParamList) {
+          quit = true;
+          break;
+        }
+        opt = HINT_SMALLDATA_TS_SORT;
+        break;
       case TK_NK_LP:
         lastComma = false;
         if (0 == opt || inParamList) {
diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c
index f1013d6157..0d943dd9db 100644
--- a/source/libs/parser/src/parTokenizer.c
+++ b/source/libs/parser/src/parTokenizer.c
@@ -213,6 +213,7 @@ static SKeyword keywordTable[] = {
     {"SLIDING",              TK_SLIDING},
     {"SLIMIT",               TK_SLIMIT},
     {"SMA",                  TK_SMA},
+    {"SMALLDATA_TS_SORT",    TK_SMALLDATA_TS_SORT},
     {"SMALLINT",             TK_SMALLINT},
     {"SNODE",                TK_SNODE},
     {"SNODES",               TK_SNODES},
diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h
index fcccdcf23e..3f1cb0fbd3 100644
--- a/source/libs/planner/inc/planInt.h
+++ b/source/libs/planner/inc/planInt.h
@@ -47,7 +47,8 @@ int32_t validateQueryPlan(SPlanContext* pCxt, SQueryPlan* pPlan);
 
 bool        getBatchScanOptionFromHint(SNodeList* pList);
 bool        getSortForGroupOptHint(SNodeList* pList);
-bool        getparaTablesSortOptHint(SNodeList* pList);
+bool        getParaTablesSortOptHint(SNodeList* pList);
+bool        getSmallDataTsSortOptHint(SNodeList* pList);
 bool        getOptHint(SNodeList* pList, EHintOption hint);
 SLogicNode* getLogicNodeRootNode(SLogicNode* pCurr);
 int32_t     collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes);
diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c
index c5e84898ed..a796c9cdb2 100644
--- a/source/libs/planner/src/planLogicCreater.c
+++ b/source/libs/planner/src/planLogicCreater.c
@@ -501,7 +501,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
   } else {
     nodesDestroyNode((SNode*)pScan);
   }
-  pScan->paraTablesSort = getparaTablesSortOptHint(pSelect->pHint);
+  pScan->paraTablesSort = getParaTablesSortOptHint(pSelect->pHint);
+  pScan->smallDataTsSort = getSmallDataTsSortOptHint(pSelect->pHint);
   pCxt->hasScan = true;
 
   return code;
diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c
index baef39144c..5895f57acd 100644
--- a/source/libs/planner/src/planPhysiCreater.c
+++ b/source/libs/planner/src/planPhysiCreater.c
@@ -652,6 +652,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
   pTableScan->filesetDelimited = pScanLogicNode->filesetDelimited;
   pTableScan->needCountEmptyTable = pScanLogicNode->isCountByTag;
   pTableScan->paraTablesSort = pScanLogicNode->paraTablesSort;
+  pTableScan->smallDataTsSort = pScanLogicNode->smallDataTsSort;
 
   int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pTableScan, pPhyNode);
   if (TSDB_CODE_SUCCESS == code) {
diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c
index f31bf23bc9..a6109cdacb 100644
--- a/source/libs/planner/src/planUtil.c
+++ b/source/libs/planner/src/planUtil.c
@@ -466,7 +466,7 @@ bool getOptHint(SNodeList* pList, EHintOption hint) {
   return false;
 }
 
-bool getparaTablesSortOptHint(SNodeList* pList) {
+bool getParaTablesSortOptHint(SNodeList* pList) {
   if (!pList) return false;
   SNode* pNode;
   FOREACH(pNode, pList) {
@@ -478,6 +478,18 @@ bool getparaTablesSortOptHint(SNodeList* pList) {
   return false;
 }
 
+bool getSmallDataTsSortOptHint(SNodeList* pList) {
+  if (!pList) return false;
+  SNode* pNode;
+  FOREACH(pNode, pList) {
+    SHintNode* pHint = (SHintNode*)pNode;
+    if (pHint->option == HINT_SMALLDATA_TS_SORT) {
+      return true;
+    }
+  }
+  return false;
+}
+
 int32_t collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes) {
   int32_t code = TSDB_CODE_SUCCESS;
   SLogicNode* pCurr = (SLogicNode*)pNode;

From bb8086639529dd17fa59ca8590916387c752eb7c Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 8 Mar 2024 14:49:48 +0800
Subject: [PATCH 179/201] fix:memory leak

---
 source/client/inc/clientInt.h        |  4 +---
 source/client/src/clientEnv.c        |  1 -
 source/client/src/clientImpl.c       | 14 +++++---------
 source/client/src/clientMain.c       |  2 --
 source/client/src/clientMsgHandler.c |  4 ++--
 source/client/src/clientTmq.c        |  6 +-----
 tests/parallel_test/cases.task       |  2 +-
 7 files changed, 10 insertions(+), 23 deletions(-)

diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index 6650d4c8b3..fb6675fe7c 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -181,7 +181,6 @@ typedef struct SResultColumn {
 
 typedef struct SReqResultInfo {
   SExecResult    execRes;
-  const char*    pRspMsg;
   const char*    pData;
   TAOS_FIELD*    fields;      // todo, column names are not needed.
   TAOS_FIELD*    userFields;  // the fields info that return to user
@@ -298,8 +297,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
 
 void    doSetOneRowPtr(SReqResultInfo* pResultInfo);
 void    setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
-int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4,
-                              bool freeAfterUse);
+int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4);
 int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows,
                          bool convertUcs4);
 void    setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 6c20813118..9f41e4cc11 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -363,7 +363,6 @@ void *createRequest(uint64_t connId, int32_t type, int64_t reqid) {
 }
 
 void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
-  taosMemoryFreeClear(pResInfo->pRspMsg);
   taosMemoryFreeClear(pResInfo->length);
   taosMemoryFreeClear(pResInfo->row);
   taosMemoryFreeClear(pResInfo->pCol);
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index 94da1e1998..e1c22fc070 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -302,7 +302,7 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
   int8_t             biMode = atomic_load_8(&pRequest->pTscObj->biMode);
   int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp, biMode);
   if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
-    code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true);
+    code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false);
   }
 
   return code;
@@ -341,7 +341,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
   int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp,
                               atomic_load_8(&pRequest->pTscObj->biMode));
   if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
-    code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true);
+    code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false);
   }
 
   SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
@@ -1721,7 +1721,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
     }
 
     pRequest->code =
-        setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4, true);
+        setQueryResultFromRsp(&pRequest->body.resInfo, (const SRetrieveTableRsp*)pResInfo->pData, convertUcs4);
     if (pRequest->code != TSDB_CODE_SUCCESS) {
       pResultInfo->numOfRows = 0;
       return NULL;
@@ -2180,16 +2180,12 @@ void resetConnectDB(STscObj* pTscObj) {
   taosThreadMutexUnlock(&pTscObj->mutex);
 }
 
-int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4,
-                              bool freeAfterUse) {
+int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4) {
   if (pResultInfo == NULL || pRsp == NULL) {
     tscError("setQueryResultFromRsp paras is null");
     return TSDB_CODE_TSC_INTERNAL_ERROR;
   }
 
-  if (freeAfterUse) taosMemoryFreeClear(pResultInfo->pRspMsg);
-
-  pResultInfo->pRspMsg = (const char*)pRsp;
   pResultInfo->pData = (void*)pRsp->data;
   pResultInfo->numOfRows = htobe64(pRsp->numOfRows);
   pResultInfo->current = 0;
@@ -2618,7 +2614,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
   }
 
   pRequest->code =
-      setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4, true);
+      setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4);
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     pResultInfo->numOfRows = 0;
     pRequest->code = code;
diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 47adb40eaa..a49d2091ac 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -350,7 +350,6 @@ void taos_free_result(TAOS_RES *res) {
     taosArrayDestroy(pRsp->rsp.createTableLen);
     taosArrayDestroyP(pRsp->rsp.createTableReq, taosMemoryFree);
 
-    pRsp->resInfo.pRspMsg = NULL;
     doFreeReqResultInfo(&pRsp->resInfo);
     taosMemoryFree(pRsp);
   } else if (TD_RES_TMQ(res)) {
@@ -359,7 +358,6 @@ void taos_free_result(TAOS_RES *res) {
     taosArrayDestroy(pRsp->rsp.blockDataLen);
     taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree);
     taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSchemaWrapper);
-    pRsp->resInfo.pRspMsg = NULL;
     doFreeReqResultInfo(&pRsp->resInfo);
     taosMemoryFree(pRsp);
   } else if (TD_RES_TMQ_META(res)) {
diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c
index 324b99022b..360a346fb7 100644
--- a/source/client/src/clientMsgHandler.c
+++ b/source/client/src/clientMsgHandler.c
@@ -538,7 +538,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
       code = buildShowVariablesRsp(rsp.variables, &pRes);
     }
     if (TSDB_CODE_SUCCESS == code) {
-      code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
+      code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false);
     }
 
     if (code != 0) {
@@ -651,7 +651,7 @@ int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
       code = buildRetriveTableRspForCompactDb(&rsp, &pRes);
     }
     if (TSDB_CODE_SUCCESS == code) {
-      code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
+      code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false);
     }
 
     if (code != 0) {
diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c
index 0270ae9657..fb1882e472 100644
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@@ -2628,13 +2628,9 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
     SRetrieveTableRspForTmq* pRetrieveTmq =
         (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter);
     if (pRspObj->rsp.withSchema) {
+      doFreeReqResultInfo(&pRspObj->resInfo);
       SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter);
       setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols);
-      taosMemoryFreeClear(pRspObj->resInfo.row);
-      taosMemoryFreeClear(pRspObj->resInfo.pCol);
-      taosMemoryFreeClear(pRspObj->resInfo.length);
-      taosMemoryFreeClear(pRspObj->resInfo.convertBuf);
-      taosMemoryFreeClear(pRspObj->resInfo.convertJson);
     }
 
     pRspObj->resInfo.pData = (void*)pRetrieveTmq->data;
diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task
index 481e111715..a2348bdedd 100644
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@@ -228,7 +228,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py
-,,n,system-test,python3 ./test.py -f 7-tmq/tmq_taosx.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_replay.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSeekAndCommit.py
 ,,n,system-test,python3 ./test.py -f 7-tmq/tmq_offset.py

From 93f71923d7977306b594131836f1e15d6984571f Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 8 Mar 2024 15:47:53 +0800
Subject: [PATCH 180/201] fix:memory leak

---
 source/client/inc/clientInt.h  | 1 +
 source/client/src/clientEnv.c  | 1 +
 source/client/src/clientImpl.c | 2 ++
 3 files changed, 4 insertions(+)

diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h
index fb6675fe7c..6c3603b4e0 100644
--- a/source/client/inc/clientInt.h
+++ b/source/client/inc/clientInt.h
@@ -181,6 +181,7 @@ typedef struct SResultColumn {
 
 typedef struct SReqResultInfo {
   SExecResult    execRes;
+  const char*    pRspMsg;
   const char*    pData;
   TAOS_FIELD*    fields;      // todo, column names are not needed.
   TAOS_FIELD*    userFields;  // the fields info that return to user
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 9f41e4cc11..6c20813118 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -363,6 +363,7 @@ void *createRequest(uint64_t connId, int32_t type, int64_t reqid) {
 }
 
 void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
+  taosMemoryFreeClear(pResInfo->pRspMsg);
   taosMemoryFreeClear(pResInfo->length);
   taosMemoryFreeClear(pResInfo->row);
   taosMemoryFreeClear(pResInfo->pCol);
diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index e1c22fc070..e87e676f08 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -2186,6 +2186,8 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR
     return TSDB_CODE_TSC_INTERNAL_ERROR;
   }
 
+  taosMemoryFreeClear(pResultInfo->pRspMsg);
+  pResultInfo->pRspMsg = (const char*)pRsp;
   pResultInfo->pData = (void*)pRsp->data;
   pResultInfo->numOfRows = htobe64(pRsp->numOfRows);
   pResultInfo->current = 0;

From ecfc5ec439d675b1e51c9e2e60a3cefb3634480d Mon Sep 17 00:00:00 2001
From: Ping Xiao <pxiao@taosdata.com>
Date: Fri, 8 Mar 2024 16:12:27 +0800
Subject: [PATCH 181/201] replace kill signal

---
 tests/script/sh/stop_dnodes.sh | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh
index c63d6daf8a..b447a7325e 100755
--- a/tests/script/sh/stop_dnodes.sh
+++ b/tests/script/sh/stop_dnodes.sh
@@ -15,42 +15,42 @@ fi
 
 PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
 while [ -n "$PID" ]; do
-  echo kill -9 $PID
-  #pkill -9 taosd
-  kill -9 $PID
+  echo kill -15 $PID
+  #pkill -15 taosd
+  kill -15 $PID
   echo "Killing taosd processes"
   if [ "$OS_TYPE" != "Darwin" ]; then
     fuser -k -n tcp 6030
   else
-    lsof -nti:6030 | xargs kill -9
+    lsof -nti:6030 | xargs kill -15
   fi
   PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
 done
 
 PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'`
 while [ -n "$PID" ]; do
-  echo kill -9 $PID
+  echo kill -15 $PID
   #pkill -9 taos
-  kill -9 $PID
+  kill -15 $PID
   echo "Killing taos processes"
   if [ "$OS_TYPE" != "Darwin" ]; then
     fuser -k -n tcp 6030
   else
-    lsof -nti:6030 | xargs kill -9
+    lsof -nti:6030 | xargs kill -15
   fi
   PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'`
 done
 
 PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'`
 while [ -n "$PID" ]; do
-  echo kill -9 $PID
-  #pkill -9 tmq_sim
-  kill -9 $PID
+  echo kill -15 $PID
+  #pkill -15 tmq_sim
+  kill -15 $PID
   echo "Killing tmq_sim processes"
   if [ "$OS_TYPE" != "Darwin" ]; then
     fuser -k -n tcp 6030
   else
-    lsof -nti:6030 | xargs kill -9
+    lsof -nti:6030 | xargs kill -15
   fi
   PID=`ps -ef|grep -w tmq_sim | grep -v grep | awk '{print $2}'`
-done
+done
\ No newline at end of file

From 067fb82adbf2a61f53242bf0e34e4479b9e9bfec Mon Sep 17 00:00:00 2001
From: yihaoDeng <yhdeng@taosdata.com>
Date: Fri, 8 Mar 2024 16:39:55 +0800
Subject: [PATCH 182/201] fix:test case failed in some time

---
 tests/system-test/7-tmq/tmqParamsTest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/system-test/7-tmq/tmqParamsTest.py b/tests/system-test/7-tmq/tmqParamsTest.py
index 82a5d42b47..b25f23ef11 100644
--- a/tests/system-test/7-tmq/tmqParamsTest.py
+++ b/tests/system-test/7-tmq/tmqParamsTest.py
@@ -164,7 +164,7 @@ class TDTestCase:
                                 offset_value_list = list(map(lambda x: (x[-2].replace("wal:", "").replace("earliest", "0").replace("latest", "0").replace(offset_value, "0")), subscription_info))
                                 offset_value_list1 = list(map(lambda x: int(x.split("/")[0]), offset_value_list))
                                 offset_value_list2 = list(map(lambda x: int(x.split("/")[1]), offset_value_list))
-                                tdSql.checkEqual(offset_value_list1 == offset_value_list2, True)
+                                tdSql.checkEqual(offset_value_list1 <= offset_value_list2, True)
                                 tdSql.checkEqual(sum(offset_value_list1) >= 0, True)
                                 rows_value_list  = list(map(lambda x: int(x[-1]), subscription_info))
                                 tdSql.checkEqual(sum(rows_value_list), expected_res)
@@ -187,4 +187,4 @@ class TDTestCase:
 event = threading.Event()
 
 tdCases.addLinux(__file__, TDTestCase())
-tdCases.addWindows(__file__, TDTestCase())
\ No newline at end of file
+tdCases.addWindows(__file__, TDTestCase())

From 9e5ac49538679ef03b34f3897f44e9572e9e3f54 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Mon, 11 Mar 2024 09:53:47 +0800
Subject: [PATCH 183/201] fix:[TS-4551] vgroplist is null in subscribe

---
 source/libs/planner/src/planPhysiCreater.c | 33 ++++++++++++++--------
 tests/system-test/7-tmq/tmq_taosx.py       | 27 ++++++++++++++++++
 2 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c
index baef39144c..64249207f5 100644
--- a/source/libs/planner/src/planPhysiCreater.c
+++ b/source/libs/planner/src/planPhysiCreater.c
@@ -527,7 +527,10 @@ static int32_t createSimpleScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSub
   if (NULL == pScan) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
+
+  if (pScanLogicNode->pVgroupList) {
+    vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
+  }
   return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, pScan, pPhyNode);
 }
 
@@ -538,8 +541,9 @@ static int32_t createTagScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubpla
   if (NULL == pScan) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
-
+  if (pScanLogicNode->pVgroupList) {
+    vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
+  }
   pScan->onlyMetaCtbIdx = pScanLogicNode->onlyMetaCtbIdx;
 
   return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
@@ -563,8 +567,9 @@ static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSu
   pScan->groupSort = pScanLogicNode->groupSort;
   pScan->ignoreNull = pScanLogicNode->igLastNull;
 
-  vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
-
+  if (pScanLogicNode->pVgroupList) {
+    vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
+  }
   int32_t code = createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
 
   if (TSDB_CODE_SUCCESS == code && pScanLogicNode->pFuncTypes != NULL) {
@@ -609,8 +614,9 @@ static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
   }
 
   pScan->groupSort = pScanLogicNode->groupSort;
-  vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
-
+  if (pScanLogicNode->pVgroupList) {
+    vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
+  }
   return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode);
 }
 
@@ -680,7 +686,9 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan*
   if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLES) ||
       0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS) ||
       0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_COLS)) {
-    vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
+    if (pScanLogicNode->pVgroupList) {
+      vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode);
+    }
   } else {
     pSubplan->execNode.nodeId = MNODE_HANDLE;
     pSubplan->execNode.epSet = pCxt->pPlanCxt->mgmtEpSet;
@@ -2216,11 +2224,12 @@ static int32_t createQueryInserter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNod
   pInserter->stableId = pModify->stableId;
   pInserter->tableType = pModify->tableType;
   strcpy(pInserter->tableName, pModify->tableName);
-  pInserter->vgId = pModify->pVgroupList->vgroups[0].vgId;
-  pInserter->epSet = pModify->pVgroupList->vgroups[0].epSet;
   pInserter->explain = (QUERY_NODE_EXPLAIN_STMT == nodeType(pCxt->pPlanCxt->pAstRoot) ? true : false);
-  vgroupInfoToNodeAddr(pModify->pVgroupList->vgroups, &pSubplan->execNode);
-
+  if (pModify->pVgroupList) {
+    pInserter->vgId = pModify->pVgroupList->vgroups[0].vgId;
+    pInserter->epSet = pModify->pVgroupList->vgroups[0].epSet;
+    vgroupInfoToNodeAddr(pModify->pVgroupList->vgroups, &pSubplan->execNode);
+  }
   int32_t code = setListSlotId(pCxt, pSubplan->pNode->pOutputDataBlockDesc->dataBlockId, -1, pModify->pInsertCols,
                                &pInserter->pCols);
   if (TSDB_CODE_SUCCESS == code) {
diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py
index 2712caed3c..45c5284ab3 100644
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@@ -423,9 +423,36 @@ class TDTestCase:
 
         consumer.close()
 
+    def consume_ts_4551(self):
+        tdSql.execute(f'use d1')
+
+        tdSql.execute(f'create topic topic_stable as stable stt where tbname like "t%"')
+        consumer_dict = {
+            "group.id": "g1",
+            "td.connect.user": "root",
+            "td.connect.pass": "taosdata",
+            "auto.offset.reset": "earliest",
+        }
+        consumer = Consumer(consumer_dict)
+
+        try:
+            consumer.subscribe(["topic_stable"])
+        except TmqError:
+            tdLog.exit(f"subscribe error")
+
+        try:
+            while True:
+                res = consumer.poll(1)
+                if not res:
+                    break
+        finally:
+            consumer.close()
+        print("consume_ts_4551 ok")
+
     def run(self):
         self.consumeTest()
         self.consume_ts_4544()
+        self.consume_ts_4551()
         self.consume_TS_4540_Test()
 
         tdSql.prepare()

From 4b9d74cf45e9a5080ca2e179ff0ea9fdbdbb5b57 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Mon, 11 Mar 2024 19:12:29 +0800
Subject: [PATCH 184/201] add some log

---
 source/libs/executor/src/scanoperator.c             | 1 +
 source/libs/executor/src/streamtimewindowoperator.c | 7 ++++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 9024f7a341..d3dd90795c 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -2269,6 +2269,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
 
   if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) {
     if (isTaskKilled(pTaskInfo)) {
+      qInfo("===stream===stream scan is killed. task id:%s, code %s", id, tstrerror(pTaskInfo->code));
       return NULL;
     }
 
diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c
index 7a1bb2729c..a05e6fbfb8 100644
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@@ -1281,8 +1281,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
         tSimpleHashCleanup(pInfo->pUpdatedMap);
         pInfo->pUpdatedMap = NULL;
       }
-      qInfo("%s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
-      T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
+      qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
+      return NULL;
     }
 
     SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
@@ -4332,7 +4332,8 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {
         pInfo->pUpdatedMap = NULL;
       }
 
-      T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
+      qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
+      return NULL;
     }
 
     SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);

From 08f99748172f30ab5536ad502e62c20eb38caf3a Mon Sep 17 00:00:00 2001
From: facetosea <25808407@qq.com>
Date: Tue, 12 Mar 2024 10:06:57 +0800
Subject: [PATCH 185/201] Prevent misconfiguration

---
 source/util/src/tconfig.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c
index e98c264df5..88fa24ee1d 100644
--- a/source/util/src/tconfig.c
+++ b/source/util/src/tconfig.c
@@ -454,7 +454,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) {
     SConfigItem *existItem = taosArrayGet(pCfg->array, i);
     if (existItem != NULL && strcmp(existItem->name, pItem->name) == 0) {
       taosMemoryFree(pItem->name);
-      return 0;
+      return TSDB_CODE_INVALID_CFG;
     }
   }
 

From 31b6d1524098a16f9dcf0c8cee9f96a82851ddea Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 12 Mar 2024 03:08:20 +0000
Subject: [PATCH 186/201] fix invalid free

---
 source/libs/scheduler/src/schRemote.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c
index 43ea418851..1fabc3a6e5 100644
--- a/source/libs/scheduler/src/schRemote.c
+++ b/source/libs/scheduler/src/schRemote.c
@@ -892,11 +892,14 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe
 int32_t schCloneSMsgSendInfo(void *src, void **dst) {
   SMsgSendInfo *pSrc = src;
   int32_t       code = 0;
-  SMsgSendInfo *pDst = taosMemoryMalloc(sizeof(*pSrc));
+  SMsgSendInfo *pDst = taosMemoryCalloc(1, sizeof(*pSrc));
   if (NULL == pDst) {
     qError("malloc SMsgSendInfo for rpcCtx failed, len:%d", (int32_t)sizeof(*pSrc));
     SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
   }
+   if (pDst->target.dbFName != NULL) {
+    pDst->target.dbFName = taosStrdup(pSrc->target.dbFName);
+  } 
 
   memcpy(pDst, pSrc, sizeof(*pSrc));
   pDst->param = NULL;

From efefc205e6b838e3f2700b7173921d1aa2615871 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 12 Mar 2024 03:21:49 +0000
Subject: [PATCH 187/201] fix invalid free

---
 source/libs/scheduler/src/schRemote.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c
index 1fabc3a6e5..8ea2185789 100644
--- a/source/libs/scheduler/src/schRemote.c
+++ b/source/libs/scheduler/src/schRemote.c
@@ -897,7 +897,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) {
     qError("malloc SMsgSendInfo for rpcCtx failed, len:%d", (int32_t)sizeof(*pSrc));
     SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
   }
-   if (pDst->target.dbFName != NULL) {
+   if (pSrc->target.dbFName != NULL) {
     pDst->target.dbFName = taosStrdup(pSrc->target.dbFName);
   } 
 

From 14c6dbd2af6f7a5b440abc25c0c99ea0b7e1c496 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 12 Mar 2024 11:45:54 +0800
Subject: [PATCH 188/201] use non sort merge

---
 source/libs/executor/src/mergeoperator.c   |  14 ++-
 source/libs/executor/src/projectoperator.c |  10 ++
 source/libs/planner/src/planSpliter.c      |   6 +-
 tests/script/tsim/query/interval.sim       | 131 +++++++++++++++++++++
 4 files changed, 158 insertions(+), 3 deletions(-)

diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c
index 093b6ab11e..caef9e1de6 100755
--- a/source/libs/executor/src/mergeoperator.c
+++ b/source/libs/executor/src/mergeoperator.c
@@ -269,6 +269,7 @@ SSDataBlock* doNonSortMerge(SOperatorInfo* pOperator) {
   SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
   SNonSortMergeInfo*          pNonSortMerge = &pInfo->nsortMergeInfo;
   SSDataBlock*                pBlock = NULL;
+  SSDataBlock*                pRes = pInfo->binfo.pRes;
 
   qDebug("start to merge no sorted rows, %s", GET_TASKID(pTaskInfo));
 
@@ -278,13 +279,19 @@ SSDataBlock* doNonSortMerge(SOperatorInfo* pOperator) {
     if (NULL == pBlock) {
       TSWAP(pNonSortMerge->pSourceStatus[pNonSortMerge->sourceWorkIdx], pNonSortMerge->pSourceStatus[idx]);
       pNonSortMerge->sourceWorkIdx++;
-      idx = NON_SORT_NEXT_SRC(pNonSortMerge, idx);
+      idx = NON_SORT_NEXT_SRC(pNonSortMerge, pNonSortMerge->lastSourceIdx);
       continue;
     }
     break;
   }
 
-  return pBlock;
+  if (!pBlock) {
+    return NULL;
+  }
+  blockDataCleanup(pRes);
+  copyDataBlock(pRes, pBlock);
+
+  return pRes;
 }
 
 void destroyNonSortMergeOperatorInfo(void* param) {
@@ -491,6 +498,9 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
     }
     case MERGE_TYPE_NON_SORT: {
       SNonSortMergeInfo* pNonSortMerge = &pInfo->nsortMergeInfo;
+      pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode);
+      initResultSizeInfo(&pOperator->resultInfo, 1024);
+      blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
       break;
     }
     case MERGE_TYPE_COLUMNS: {
diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c
index 691210e5ba..c6ebb04446 100644
--- a/source/libs/executor/src/projectoperator.c
+++ b/source/libs/executor/src/projectoperator.c
@@ -28,6 +28,7 @@ typedef struct SProjectOperatorInfo {
   bool           mergeDataBlocks;
   SSDataBlock*   pFinalRes;
   bool           inputIgnoreGroup;
+  bool           outputIgnoreGroup;
 } SProjectOperatorInfo;
 
 typedef struct SIndefOperatorInfo {
@@ -111,6 +112,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
   pInfo->binfo.inputTsOrder = pProjPhyNode->node.inputTsOrder;
   pInfo->binfo.outputTsOrder = pProjPhyNode->node.outputTsOrder;
   pInfo->inputIgnoreGroup = pProjPhyNode->inputIgnoreGroup;
+  pInfo->outputIgnoreGroup = pProjPhyNode->ignoreGroupId;
   
   if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM || pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) {
     pInfo->mergeDataBlocks = false;
@@ -276,6 +278,10 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       T_LONG_JMP(pTaskInfo->env, code);
     }
 
+    if (pProjectInfo->outputIgnoreGroup) {
+      pRes->info.id.groupId = 0;
+    }
+
     return (pRes->info.rows > 0) ? pRes : NULL;
   }
 
@@ -385,6 +391,10 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
     printDataBlock(p, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
   }
 
+  if (pProjectInfo->outputIgnoreGroup) {
+    p->info.id.groupId = 0;
+  }
+
   return (p->info.rows > 0) ? p : NULL;
 }
 
diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c
index 9664be1da2..49bde5ae4c 100644
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@@ -1320,7 +1320,11 @@ static int32_t stbSplSplitScanNodeWithPartTags(SSplitContext* pCxt, SStableSplit
   SLogicNode* pSplitNode = NULL;
   int32_t     code = stbSplGetSplitNodeForScan(pInfo, &pSplitNode);
   if (TSDB_CODE_SUCCESS == code) {
-    code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pSplitNode, NULL, pSplitNode, true, true);
+    bool needSort = true;
+    if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pSplitNode)) {
+      needSort = !((SProjectLogicNode*)pSplitNode)->ignoreGroupId;
+    }
+    code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pSplitNode, NULL, pSplitNode, needSort, needSort);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim
index 135fcc8591..7f950ea69c 100644
--- a/tests/script/tsim/query/interval.sim
+++ b/tests/script/tsim/query/interval.sim
@@ -226,4 +226,135 @@ print =============== clear
 #  return -1
 #endi
 
+print ================= step12
+
+sql create database test2 vgroups 4;
+sql use test2;
+sql create stable stb (ts timestamp, c1 int) tags (t1 int);
+sql create table t1 using stb tags (1);
+sql create table t2 using stb tags (2);
+sql create table t3 using stb tags (3);
+sql create table t4 using stb tags (4);
+sql create table t5 using stb tags (4);
+sql create table t6 using stb tags (4);
+sql insert into t1 values ("2024-03-01 14:29:07.051", 11);
+sql insert into t2 values ("2024-03-01 14:29:07.051", 21);
+sql insert into t3 values ("2024-03-01 14:29:07.051", 31);
+sql insert into t4 values ("2024-03-01 14:29:07.051", 41);
+sql insert into t5 values ("2024-03-01 14:29:07.051", 51);
+sql insert into t6 values ("2024-03-01 14:29:07.051", 61);
+sql insert into t1 values ("2024-03-01 14:30:07.051", 12);
+sql insert into t2 values ("2024-03-01 14:30:07.051", 22);
+sql insert into t3 values ("2024-03-01 14:30:07.051", 32);
+sql insert into t4 values ("2024-03-01 14:30:07.051", 42);
+sql insert into t5 values ("2024-03-01 14:30:07.051", 52);
+sql insert into t6 values ("2024-03-01 14:30:07.051", 62);
+sql insert into t1 values ("2024-03-01 14:31:07.051", 13);
+sql insert into t2 values ("2024-03-01 14:31:07.051", 23);
+sql insert into t3 values ("2024-03-01 14:31:07.051", 33);
+sql insert into t4 values ("2024-03-01 14:31:07.051", 43);
+sql insert into t5 values ("2024-03-01 14:31:07.051", 53);
+sql insert into t6 values ("2024-03-01 14:31:07.051", 63);
+sql insert into t1 values ("2024-03-01 14:32:07.051", 14);
+sql insert into t2 values ("2024-03-01 14:32:07.051", 24);
+sql insert into t3 values ("2024-03-01 14:32:07.051", 34);
+sql insert into t4 values ("2024-03-01 14:32:07.051", 44);
+sql insert into t5 values ("2024-03-01 14:32:07.051", 54);
+sql insert into t6 values ("2024-03-01 14:32:07.051", 64);
+sql insert into t1 values ("2024-03-01 14:33:07.051", 15);
+sql insert into t2 values ("2024-03-01 14:33:07.051", 25);
+sql insert into t3 values ("2024-03-01 14:33:07.051", 35);
+sql insert into t4 values ("2024-03-01 14:33:07.051", 45);
+sql insert into t5 values ("2024-03-01 14:33:07.051", 55);
+sql insert into t6 values ("2024-03-01 14:33:07.051", 65);
+sql insert into t1 values ("2024-03-01 14:34:07.051", 16);
+sql insert into t2 values ("2024-03-01 14:34:07.051", 26);
+sql insert into t3 values ("2024-03-01 14:34:07.051", 36);
+sql insert into t4 values ("2024-03-01 14:34:07.051", 46);
+sql insert into t5 values ("2024-03-01 14:34:07.051", 56);
+sql insert into t6 values ("2024-03-01 14:34:07.051", 66);
+
+sleep 300
+
+sql select _wstart, count(*) from (select * from stb partition by tbname) interval(2s);
+
+print $data00,$data01
+print $data10,$data11
+print $data20,$data21
+print $data30,$data31
+print $data40,$data41
+print $data50,$data51
+print $data60,$data61
+print $data70,$data71
+
+if $rows != 6 then
+  print $rows
+  return -1
+endi
+
+if $data01 != 6 then
+  print $data01
+endi
+
+if $data11 != 6 then
+  print $data11
+endi
+
+if $data21 != 6 then
+  print $data21
+endi
+
+if $data31 != 6 then
+  print $data31
+endi
+
+if $data41 != 6 then
+  print $data41
+endi
+
+if $data51 != 6 then
+  print $data51
+endi
+
+
+sql select _wstart, count(*) from (select * from stb partition by tbname slimit 2) interval(2s);
+
+print $data00,$data01
+print $data10,$data11
+print $data20,$data21
+print $data30,$data31
+print $data40,$data41
+print $data50,$data51
+print $data60,$data61
+print $data70,$data71
+
+if $rows != 6 then
+  print $rows then
+  return -1
+endi
+
+if $data01 != 2 then
+  print $data01
+endi
+
+if $data11 != 2 then
+  print $data11
+endi
+
+if $data21 != 2 then
+  print $data21
+endi
+
+if $data31 != 2 then
+  print $data31
+endi
+
+if $data41 != 2 then
+  print $data41
+endi
+
+if $data51 != 2 then
+  print $data51
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

From 7429096cb67163f85273cd99492085c3de07c5fb Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 12 Mar 2024 13:39:01 +0800
Subject: [PATCH 189/201] use non sort merge

---
 source/libs/planner/src/planSpliter.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c
index 49bde5ae4c..34be489333 100644
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@@ -1321,7 +1321,7 @@ static int32_t stbSplSplitScanNodeWithPartTags(SSplitContext* pCxt, SStableSplit
   int32_t     code = stbSplGetSplitNodeForScan(pInfo, &pSplitNode);
   if (TSDB_CODE_SUCCESS == code) {
     bool needSort = true;
-    if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pSplitNode)) {
+    if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pSplitNode) && !pSplitNode->pLimit && !pSplitNode->pSlimit) {
       needSort = !((SProjectLogicNode*)pSplitNode)->ignoreGroupId;
     }
     code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pSplitNode, NULL, pSplitNode, needSort, needSort);

From 71c2e66be8a9604b891f8ffdef5e1b52a3d0e787 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Tue, 12 Mar 2024 14:44:47 +0800
Subject: [PATCH 190/201] fix:key words error

---
 tests/system-test/7-tmq/tmq_taosx.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py
index 2712caed3c..6e04b214ba 100644
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@@ -363,9 +363,9 @@ class TDTestCase:
         tdSql.execute(f'create database if not exists test')
         tdSql.execute(f'use test')
         tdSql.execute(f'CREATE STABLE `test`.`b` ( `time` TIMESTAMP , `task_id` NCHAR(1000) ) TAGS( `key` NCHAR(1000))')
-        tdSql.execute(f"insert into `test`.b1 using `test`.`b`(key) tags('1') (time, task_id) values ('2024-03-04 12:50:01.000', '32') `test`.b2 using `test`.`b`(key) tags('2') (time, task_id) values ('2024-03-04 12:50:01.000', '43') `test`.b3 using `test`.`b`(key) tags('3') (time, task_id) values ('2024-03-04 12:50:01.000', '123456')")
+        tdSql.execute(f"insert into `test`.b1 using `test`.`b`(`key`) tags('1') (time, task_id) values ('2024-03-04 12:50:01.000', '32') `test`.b2 using `test`.`b`(`key`) tags('2') (time, task_id) values ('2024-03-04 12:50:01.000', '43') `test`.b3 using `test`.`b`(`key`) tags('3') (time, task_id) values ('2024-03-04 12:50:01.000', '123456')")
 
-        tdSql.execute(f'create topic tt as select tbname,task_id,key from b')
+        tdSql.execute(f'create topic tt as select tbname,task_id,`key` from b')
 
         consumer_dict = {
             "group.id": "g1",

From 9ba7221910847f7ef4e86c43c05fe0ce21862a49 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 12 Mar 2024 15:05:54 +0800
Subject: [PATCH 191/201] use non sort merge

---
 source/libs/executor/src/mergeoperator.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c
index caef9e1de6..c1a51898bc 100755
--- a/source/libs/executor/src/mergeoperator.c
+++ b/source/libs/executor/src/mergeoperator.c
@@ -288,7 +288,6 @@ SSDataBlock* doNonSortMerge(SOperatorInfo* pOperator) {
   if (!pBlock) {
     return NULL;
   }
-  blockDataCleanup(pRes);
   copyDataBlock(pRes, pBlock);
 
   return pRes;

From e25024cca3fd4c2cee8f5392bce1eef3d68fed51 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 12 Mar 2024 07:09:50 +0000
Subject: [PATCH 192/201] fix invalid free

---
 include/libs/transport/trpc.h          |  1 +
 source/dnode/mnode/impl/src/mndTrans.c | 24 +++++++++++++-----------
 source/libs/scheduler/src/schRemote.c  |  3 ---
 source/libs/transport/src/transCli.c   |  2 +-
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h
index e7b0b25653..460b8962ea 100644
--- a/include/libs/transport/trpc.h
+++ b/include/libs/transport/trpc.h
@@ -62,6 +62,7 @@ typedef struct SRpcHandleInfo {
 
   SRpcConnInfo conn;
   int8_t       forbiddenIp;
+  int8_t       notFreeAhandle;
 
 } SRpcHandleInfo;
 
diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c
index 04d74112d4..f7db0d7b15 100644
--- a/source/dnode/mnode/impl/src/mndTrans.c
+++ b/source/dnode/mnode/impl/src/mndTrans.c
@@ -15,11 +15,11 @@
 
 #define _DEFAULT_SOURCE
 #include "mndTrans.h"
-#include "mndSubscribe.h"
 #include "mndDb.h"
 #include "mndPrivilege.h"
 #include "mndShow.h"
 #include "mndStb.h"
+#include "mndSubscribe.h"
 #include "mndSync.h"
 #include "mndUser.h"
 
@@ -801,16 +801,17 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) {
     if (pNew->conflict == TRN_CONFLICT_TOPIC) {
       if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
       if (pTrans->conflict == TRN_CONFLICT_TOPIC || pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
-        if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 ) conflict = true;
+        if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
       }
     }
     if (pNew->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
       if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
-      if (pTrans->conflict == TRN_CONFLICT_TOPIC ) {
-        if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 ) conflict = true;
+      if (pTrans->conflict == TRN_CONFLICT_TOPIC) {
+        if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
       }
       if (pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
-        if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0) conflict = true;
+        if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0)
+          conflict = true;
       }
     }
 
@@ -847,7 +848,7 @@ int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans) {
 }
 
 int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
-  if(pTrans == NULL) return -1;
+  if (pTrans == NULL) return -1;
 
   if (mndTransCheckConflict(pMnode, pTrans) != 0) {
     return -1;
@@ -1142,6 +1143,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio
     return -1;
   }
   rpcMsg.info.traceId.rootId = pTrans->mTraceId;
+  rpcMsg.info.notFreeAhandle = 1;
 
   memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen);
 
@@ -1156,7 +1158,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio
   int32_t code = tmsgSendReq(&pAction->epSet, &rpcMsg);
   if (code == 0) {
     pAction->msgSent = 1;
-    //pAction->msgReceived = 0;
+    // pAction->msgReceived = 0;
     pAction->errCode = TSDB_CODE_ACTION_IN_PROGRESS;
     mInfo("trans:%d, %s:%d is sent, %s", pTrans->id, mndTransStr(pAction->stage), pAction->id, detail);
 
@@ -1253,16 +1255,16 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
 
     for (int32_t action = 0; action < numOfActions; ++action) {
       STransAction *pAction = taosArrayGet(pArray, action);
-      mDebug("trans:%d, %s:%d Sent:%d, Received:%d, errCode:0x%x, acceptableCode:0x%x, retryCode:0x%x", 
-              pTrans->id, mndTransStr(pAction->stage), pAction->id, pAction->msgSent, pAction->msgReceived,
-              pAction->errCode, pAction->acceptableCode, pAction->retryCode);
+      mDebug("trans:%d, %s:%d Sent:%d, Received:%d, errCode:0x%x, acceptableCode:0x%x, retryCode:0x%x", pTrans->id,
+             mndTransStr(pAction->stage), pAction->id, pAction->msgSent, pAction->msgReceived, pAction->errCode,
+             pAction->acceptableCode, pAction->retryCode);
       if (pAction->msgSent) {
         if (pAction->msgReceived) {
           if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) {
             mndTransResetAction(pMnode, pTrans, pAction);
             mInfo("trans:%d, %s:%d reset", pTrans->id, mndTransStr(pAction->stage), pAction->id);
           }
-        } 
+        }
       }
     }
     return TSDB_CODE_ACTION_IN_PROGRESS;
diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c
index 8ea2185789..aa33d6f2fc 100644
--- a/source/libs/scheduler/src/schRemote.c
+++ b/source/libs/scheduler/src/schRemote.c
@@ -897,9 +897,6 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) {
     qError("malloc SMsgSendInfo for rpcCtx failed, len:%d", (int32_t)sizeof(*pSrc));
     SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
   }
-   if (pSrc->target.dbFName != NULL) {
-    pDst->target.dbFName = taosStrdup(pSrc->target.dbFName);
-  } 
 
   memcpy(pDst, pSrc, sizeof(*pSrc));
   pDst->param = NULL;
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index 2e35aecd08..ac45f1eef6 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -322,7 +322,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
     if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) {
       if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) {
         conn->ctx.freeFunc(msg->ctx->ahandle);
-      } else if (msg->ctx->ahandle != NULL && pThrd->destroyAhandleFp != NULL) {
+      } else if (msg->msg.info.notFreeAhandle == 0 && msg->ctx->ahandle != NULL && pThrd->destroyAhandleFp != NULL) {
         tDebug("%s conn %p destroy unfinished ahandle %p", CONN_GET_INST_LABEL(conn), conn, msg->ctx->ahandle);
         pThrd->destroyAhandleFp(msg->ctx->ahandle);
       }

From 3835e0c1562a9214557dc26630c9973acd99ca0c Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Tue, 12 Mar 2024 16:35:55 +0800
Subject: [PATCH 193/201] open operator after resume stream

---
 source/dnode/vnode/src/tqCommon/tqCommon.c      |  1 +
 .../executor/src/streamtimewindowoperator.c     | 17 -----------------
 2 files changed, 1 insertion(+), 17 deletions(-)

diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c
index 9bfdd70477..1b67dce9b0 100644
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@@ -929,6 +929,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t
     }
 
     if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) {
+      pTask->hTaskInfo.operatorOpen = false;
       streamStartScanHistoryAsync(pTask, igUntreated);
     } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) {
       tqScanWalAsync((STQ*)handle, false);
diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c
index a05e6fbfb8..60f577ce12 100644
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@@ -1273,14 +1273,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
 
   while (1) {
     if (isTaskKilled(pTaskInfo)) {
-      if (pInfo->pUpdated != NULL) {
-        pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated);
-      }
-
-      if (pInfo->pUpdatedMap != NULL) {
-        tSimpleHashCleanup(pInfo->pUpdatedMap);
-        pInfo->pUpdatedMap = NULL;
-      }
       qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
       return NULL;
     }
@@ -4323,15 +4315,6 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {
 
   while (1) {
     if (isTaskKilled(pTaskInfo)) {
-      if (pInfo->pUpdated != NULL) {
-        pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated);
-      }
-
-      if (pInfo->pUpdatedMap != NULL) {
-        tSimpleHashCleanup(pInfo->pUpdatedMap);
-        pInfo->pUpdatedMap = NULL;
-      }
-
       qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
       return NULL;
     }

From 5c691685ba83e2b3ff572a17d37b64ef94da90cc Mon Sep 17 00:00:00 2001
From: slzhou <shenglian_zhou@163.com>
Date: Tue, 12 Mar 2024 16:53:56 +0800
Subject: [PATCH 194/201] fix: review comments

---
 docs/en/12-taos-sql/06-select.md        |  4 +-
 docs/zh/12-taos-sql/06-select.md        |  5 ++-
 include/os/osFile.h                     |  7 ++++
 source/libs/executor/src/scanoperator.c |  7 +---
 source/libs/executor/src/tsort.c        | 49 +++++--------------------
 source/os/src/osFile.c                  | 32 ++++++++++++++++
 6 files changed, 58 insertions(+), 46 deletions(-)

diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md
index a2e6bca46c..074fbfbc8d 100755
--- a/docs/en/12-taos-sql/06-select.md
+++ b/docs/en/12-taos-sql/06-select.md
@@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list
 hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */
 
 hint:
-    BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT
+    BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT | PARTITION_FIRST | SMALLDATA_TS_SORT
 
 select_list:
     select_expr [, select_expr] ...
@@ -94,6 +94,7 @@ The list of currently supported Hints is as follows:
 | SORT_FOR_GROUP| None           | Use sort for partition, conflict with PARTITION_FIRST     | With normal column in partition by list |
 | PARTITION_FIRST| None          | Use Partition before aggregate, conflict with SORT_FOR_GROUP | With normal column in partition by list |
 | PARA_TABLES_SORT| None         | When sorting the supertable rows by timestamp, No temporary disk space is used. When there are numerous tables, each with long rows, the corresponding algorithm associated with this prompt may consume a substantial amount of memory, potentially leading to an Out Of Memory (OOM) situation. | Sorting the supertable rows by timestamp  |
+| SMALLDATA_TS_SORT| None             | When sorting the supertable rows by timestamp, if the length of query columns >= 256, and there are relatively few rows, this hint can improve performance. | Sorting the supertable rows by timestamp  |
 
 For example:
 
@@ -102,6 +103,7 @@ SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0
 SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1;
 SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1;
 SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts;
+SELECT /*+ SMALLDATA_TS_SORT() */ * from stable1 order by ts;
 ```
 
 ## Lists
diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md
index eec947ea23..573e854864 100755
--- a/docs/zh/12-taos-sql/06-select.md
+++ b/docs/zh/12-taos-sql/06-select.md
@@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] [TAGS] select_list
 hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */
 
 hint:
-    BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARA_TABLES_SORT
+    BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP | PARTITION_FIRST | PARA_TABLES_SORT | SMALLDATA_TS_SORT
 
 select_list:
     select_expr [, select_expr] ...
@@ -94,6 +94,8 @@ Hints 是用户控制单个语句查询优化的一种手段，当 Hint 不适
 | SORT_FOR_GROUP| 无             | 采用sort方式进行分组, 与PARTITION_FIRST冲突  | partition by 列表有普通列时  |
 | PARTITION_FIRST| 无             | 在聚合之前使用PARTITION计算分组, 与SORT_FOR_GROUP冲突 | partition by 列表有普通列时  |
 | PARA_TABLES_SORT| 无             | 超级表的数据按时间戳排序时, 不使用临时磁盘空间, 只使用内存。当子表数量多, 行长比较大时候, 会使用大量内存, 可能发生OOM | 超级表的数据按时间戳排序时  |
+| SMALLDATA_TS_SORT| 无             | 超级表的数据按时间戳排序时, 查询列长度大于等于256, 但是行数不多, 使用这个提示, 可以提高性能 | 超级表的数据按时间戳排序时  |
+
 举例： 
 
 ```sql
@@ -101,6 +103,7 @@ SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0
 SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1;
 SELECT /*+ PARTITION_FIRST() */ count(*), c1 FROM stable1 PARTITION BY c1;
 SELECT /*+ PARA_TABLES_SORT() */ * from stable1 order by ts;
+SELECT /*+ SMALLDATA_TS_SORT() */ * from stable1 order by ts;
 ```
 
 ## 列表
diff --git a/include/os/osFile.h b/include/os/osFile.h
index eb0862a719..9c9027e931 100644
--- a/include/os/osFile.h
+++ b/include/os/osFile.h
@@ -119,6 +119,13 @@ int32_t taosSetFileHandlesLimit();
 
 int32_t taosLinkFile(char *src, char *dst);
 
+FILE*  taosOpenCFile(const char* filename, const char* mode);
+int    taosSeekCFile(FILE* file, int64_t offset, int whence);
+size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream );
+size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream);
+int	 taosCloseCFile(FILE *);
+int taosSetAutoDelFile(char* path);
+
 bool lastErrorIsFileNotExist();
 
 #ifdef __cplusplus
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index f44f8756a4..4a910c9f79 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -4011,17 +4011,14 @@ int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
   pInfo->sortBufSize = 2048 * pInfo->bufPageSize;
   int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
 
-  if (pInfo->bSortRowId && numOfTable != 1) {
-    pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
+  pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
                                                pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
+  if (pInfo->bSortRowId && numOfTable != 1) {
     int32_t memSize = 512 * 1024 * 1024;
     code = tsortSetSortByRowId(pInfo->pSortHandle, memSize);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
-  } else {
-    pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
-                                               pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
   }
   tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit);
   tsortSetMergeLimitReachedFp(pInfo->pSortHandle, tableMergeScanDoSkipTable, pInfo);
diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c
index 2c8dc7fb04..15c42334e3 100644
--- a/source/libs/executor/src/tsort.c
+++ b/source/libs/executor/src/tsort.c
@@ -13,8 +13,6 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define ALLOW_FORBID_FUNC
-
 #include "query.h"
 #include "tcommon.h"
 
@@ -34,17 +32,6 @@ struct STupleHandle {
   int32_t      rowIndex;
 };
 
-typedef struct SSortMemPageEntry {
-  int32_t pageId;
-  bool  active;
-
-  void* data;
-
-  struct SSortMemPageEntry* next;
-  struct SSortMemPageEntry* prev;
-
-} SSortMemPageEntry;
-
 typedef struct SSortMemFileRegion {
   int64_t fileOffset;
   int32_t regionSize;
@@ -68,7 +55,6 @@ typedef struct SSortMemFile {
   int32_t blockSize;
 
   FILE* pTdFile;
-  // TdFilePtr pTdFile;
   char memFilePath[PATH_MAX];
 } SSortMemFile;
 
@@ -240,21 +226,6 @@ void destroyTuple(void* t) {
   }
 }
 
-int tsortSeekFile(FILE* file, int64_t offset, int whence) {
-#ifdef WINDOWS
-  return _fseeki64(file, offset, whence);
-#else
-  return fseeko(file, offset, whence);
-#endif
-}
-
-int tsortSetAutoDelFile(char* path) {
-#ifdef WINDOWS
-  return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY);
-#else
-  return unlink(path);
-#endif
-}
 
 /**
  *
@@ -1038,9 +1009,9 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
     if (pRegion->buf == NULL) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
-    tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET);
+    taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset, SEEK_SET);
     int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize);
-    int ret = fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
+    int ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
     if (ret != 1) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       return terrno;
@@ -1058,9 +1029,9 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
     }
     int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset);
     memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset, szThisBlock);
-    tsortSeekFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
+    taosSeekCFile(pMemFile->pTdFile, pRegion->fileOffset + pRegion->bufRegOffset + pRegion->bufLen, SEEK_SET);
     int32_t readBytes = TMIN(pMemFile->blockSize, pRegion->regionSize - (pRegion->bufRegOffset + pRegion->bufLen));
-    int     ret = fread(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
+    int     ret = taosReadFromCFile(pRegion->buf, readBytes, 1, pMemFile->pTdFile);
     if (ret != 1) {
       taosMemoryFreeClear(*ppRow);
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1085,13 +1056,13 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
   }
   if (code == TSDB_CODE_SUCCESS) {
     taosGetTmpfilePath(tsTempDir, "sort-ext-mem", pMemFile->memFilePath);
-    pMemFile->pTdFile = fopen(pMemFile->memFilePath, "w+");
+    pMemFile->pTdFile = taosOpenCFile(pMemFile->memFilePath, "w+");
     if (pMemFile->pTdFile == NULL) {
       code = terrno = TAOS_SYSTEM_ERROR(errno);
     }
   }
   if (code == TSDB_CODE_SUCCESS) {
-    tsortSetAutoDelFile(pMemFile->memFilePath);
+    taosSetAutoDelFile(pMemFile->memFilePath);
 
     pMemFile->currRegionId = -1;
     pMemFile->currRegionOffset = -1;
@@ -1119,7 +1090,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
       if (pMemFile->aFileRegions) taosMemoryFreeClear(pMemFile->aFileRegions);
       if (pMemFile->writeBuf) taosMemoryFreeClear(pMemFile->writeBuf);
       if (pMemFile->pTdFile) {
-        fclose(pMemFile->pTdFile);
+        taosCloseCFile(pMemFile->pTdFile);
         pMemFile->pTdFile = NULL;
       }
       taosMemoryFreeClear(pMemFile);
@@ -1142,7 +1113,7 @@ static int32_t destroySortMemFile(SSortHandle* pHandle) {
   taosMemoryFree(pMemFile->writeBuf);
   pMemFile->writeBuf = NULL;
 
-  fclose(pMemFile->pTdFile);
+  taosCloseCFile(pMemFile->pTdFile);
   pMemFile->pTdFile = NULL;
   taosRemoveFile(pMemFile->memFilePath);
   taosMemoryFree(pMemFile);
@@ -1272,8 +1243,8 @@ static void initRowIdSort(SSortHandle* pHandle) {
 
   int32_t  rowSize = blockDataGetRowSize(pHandle->pDataBlock);
   size_t nCols = taosArrayGetSize(pHandle->pDataBlock->pDataBlock);
-  pHandle->pageSize = getProperSortPageSize(rowSize, nCols);
-  pHandle->numOfPages = 2048;
+  pHandle->pageSize = 256 * 1024; // 256k
+  pHandle->numOfPages = 256;
 
   SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);
   SBlockOrderInfo bi = {0};
diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c
index e6491639dc..bdd43fe9fa 100644
--- a/source/os/src/osFile.c
+++ b/source/os/src/osFile.c
@@ -1404,3 +1404,35 @@ int32_t taosLinkFile(char *src, char *dst) {
 #endif
   return 0;
 }
+
+FILE*  taosOpenCFile(const char* filename, const char* mode) {
+  return fopen(filename, mode);
+}
+
+int taosSeekCFile(FILE* file, int64_t offset, int whence) {
+#ifdef WINDOWS
+  return _fseeki64(file, offset, whence);
+#else
+  return fseeko(file, offset, whence);
+#endif  
+}
+
+size_t taosReadFromCFile(void *buffer, size_t size, size_t count, FILE *stream ) {
+  return fread(buffer, size, count, stream);
+}
+
+size_t taosWriteToCFile(const void* ptr, size_t size, size_t nitems, FILE* stream) {
+  return fwrite(ptr, size, nitems, stream);
+}
+
+int	 taosCloseCFile(FILE *f) {
+  return fclose(f);
+}
+
+int taosSetAutoDelFile(char* path) {
+#ifdef WINDOWS
+  return SetFileAttributes(path, FILE_ATTRIBUTE_TEMPORARY);
+#else
+  return unlink(path);
+#endif  
+}
\ No newline at end of file

From e72ddf942942ff4ce7fec3a33e0c5b929258d3ee Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 13 Mar 2024 05:56:13 +0000
Subject: [PATCH 195/201] fix double free

---
 source/dnode/mnode/impl/src/mndIndex.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c
index 622ed0080f..43d13ae602 100644
--- a/source/dnode/mnode/impl/src/mndIndex.c
+++ b/source/dnode/mnode/impl/src/mndIndex.c
@@ -596,6 +596,9 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
   memcpy(pNew, pOld, sizeof(SStbObj));
   taosRUnLockLatch(&pOld->lock);
 
+  pNew->pTags = NULL;
+  pNew->pColumns = NULL;
+
   pNew->pTags = NULL;
   pNew->updateTime = taosGetTimestampMs();
   pNew->lock = 0;

From e9b0c877b073f40c388f9ff2cfeabec614ff0192 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 13 Mar 2024 06:01:16 +0000
Subject: [PATCH 196/201] fix double free

---
 source/dnode/mnode/impl/src/mndIndex.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c
index 43d13ae602..f2e2c556cf 100644
--- a/source/dnode/mnode/impl/src/mndIndex.c
+++ b/source/dnode/mnode/impl/src/mndIndex.c
@@ -598,8 +598,6 @@ static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
 
   pNew->pTags = NULL;
   pNew->pColumns = NULL;
-
-  pNew->pTags = NULL;
   pNew->updateTime = taosGetTimestampMs();
   pNew->lock = 0;
 

From c13338d96568490df978fe01939a75ec098a2f25 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Wed, 13 Mar 2024 16:16:44 +0800
Subject: [PATCH 197/201] stream event window mem leak

---
 source/libs/executor/src/streameventwindowoperator.c | 7 +++++++
 source/libs/executor/src/streamtimewindowoperator.c  | 9 ++-------
 source/libs/stream/src/streamSessionState.c          | 3 +--
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c
index 11d8d1487a..ef5c2572d9 100644
--- a/source/libs/executor/src/streameventwindowoperator.c
+++ b/source/libs/executor/src/streameventwindowoperator.c
@@ -99,6 +99,11 @@ int32_t getEndCondIndex(bool* pEnd, int32_t start, int32_t rows) {
   return -1;
 }
 
+int32_t reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {
+  pAPI->streamStateReleaseBuf(pState, pPos, true);
+  return TSDB_CODE_SUCCESS;
+}
+
 void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd, int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey) {
   int32_t code = TSDB_CODE_SUCCESS;
   int32_t size = pAggSup->resultRowSize;
@@ -143,6 +148,7 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
   pCurWin->winInfo.isOutput = false;
 
 _end:
+  reuseOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->stateStore);
   pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);
   pNextWinKey->groupId = groupId;
   code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pNextWinKey, NULL, 0);
@@ -341,6 +347,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
     }
 
     if (isWindowIncomplete(&curWin)) {
+      releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAggSup->stateStore);
       continue;
     }
 
diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c
index 7a1bb2729c..9085694aed 100644
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@@ -1845,11 +1845,6 @@ int32_t releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {
-  pAPI->streamStateReleaseBuf(pState, pPos, true);
-  return TSDB_CODE_SUCCESS;
-}
-
 void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) {
   SSessionKey key = {0};
   getSessionHashKey(pKey, &key);
@@ -2495,7 +2490,7 @@ void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) {
     return;
   }
   SResultWindowInfo* pWinInfo = taosArrayGet(pAllWins, size - 1);
-  SSessionKey*       pSeKey = pWinInfo->pStatePos->pKey;
+  SSessionKey*       pSeKey = &pWinInfo->sessionWin;
   taosArrayPush(pMaxWins, pSeKey);
   if (pSeKey->groupId == 0) {
     return;
@@ -2503,7 +2498,7 @@ void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) {
   uint64_t preGpId = pSeKey->groupId;
   for (int32_t i = size - 2; i >= 0; i--) {
     pWinInfo = taosArrayGet(pAllWins, i);
-    pSeKey = pWinInfo->pStatePos->pKey;
+    pSeKey = &pWinInfo->sessionWin;
     if (preGpId != pSeKey->groupId) {
       taosArrayPush(pMaxWins, pSeKey);
       preGpId = pSeKey->groupId;
diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c
index 723f04c499..06f7b6a268 100644
--- a/source/libs/stream/src/streamSessionState.c
+++ b/source/libs/stream/src/streamSessionState.c
@@ -310,7 +310,7 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream
   int32_t size = taosArrayGetSize(pWinStates);
   if (pCur->buffIndex >= 0) {
     if (pCur->buffIndex >= size) {
-      pNewPos = insertNewSessionWindow(pFileState, pWinStates, pWinKey, size);
+      pNewPos = addNewSessionWindow(pFileState, pWinStates, pWinKey);
       goto _end;
     }
     pNewPos = insertNewSessionWindow(pFileState, pWinStates, pWinKey, pCur->buffIndex);
@@ -332,7 +332,6 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream
   }
 
 _end:
-  memcpy(pNewPos->pKey, pWinKey, sizeof(SSessionKey));
   (*ppVal) = pNewPos;
   return TSDB_CODE_SUCCESS;
 }

From ef32863f0754da1de15ad1ed81666f3768f00863 Mon Sep 17 00:00:00 2001
From: 54liuyao <54liuyao>
Date: Wed, 13 Mar 2024 19:44:59 +0800
Subject: [PATCH 198/201] stream event window mem leak

---
 source/libs/stream/src/streamSessionState.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c
index 06f7b6a268..295132a4f5 100644
--- a/source/libs/stream/src/streamSessionState.c
+++ b/source/libs/stream/src/streamSessionState.c
@@ -327,6 +327,7 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream
       }
     }
     pNewPos = getNewRowPosForWrite(pFileState);
+    memcpy(pNewPos->pKey, pWinKey, sizeof(SSessionKey));
     pNewPos->needFree = true;
     pNewPos->beFlushed = true;
   }

From 3a0df8be8d6fb255990bd8586d039b4a6fe78aa6 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Wed, 13 Mar 2024 23:13:24 +0800
Subject: [PATCH 199/201] fix(query): handle the calculation of logic
 expression with different result rows, when constant expression exists.

---
 source/libs/scalar/src/scalar.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c
index b1cd10eac9..5f43ae9f3c 100644
--- a/source/libs/scalar/src/scalar.c
+++ b/source/libs/scalar/src/scalar.c
@@ -808,7 +808,11 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
         complete = false;
         continue;
       }
-      char *p = colDataGetData(params[m].columnData, i);
+
+      // 1=1 and tag_column = 1
+      int32_t ind = (i >= params[m].numOfRows)? (params[m].numOfRows - 1):i;
+      char* p = colDataGetData(params[m].columnData, ind);
+
       GET_TYPED_DATA(value, bool, params[m].columnData->info.type, p);
 
       if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) {

From 2b7fd0d15c50075cb89c51e1410317791b86c2d3 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 14 Mar 2024 10:04:14 +0800
Subject: [PATCH 200/201] fix(tsdb): set strict varchar length check.

---
 source/dnode/vnode/src/tsdb/tsdbRead2.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c
index 90a26d17dc..d740f9491c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead2.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c
@@ -658,11 +658,12 @@ static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int
       colDataSetNULL(pColInfoData, rowIndex);
     } else {
       varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData);
-      if (pColVal->value.nData > pColInfoData->info.bytes) {
+      if ((pColVal->value.nData + VARSTR_HEADER_SIZE) > pColInfoData->info.bytes) {
         tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData,
                  pColInfoData->info.bytes);
         return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
       }
+
       if (pColVal->value.nData > 0) {  // pData may be null, if nData is 0
         memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData);
       }

From 190b02dd1aa7973cb7eb99ca2d6ccb99c2a86d09 Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Thu, 14 Mar 2024 10:04:43 +0800
Subject: [PATCH 201/201] fix(stream):set correct version for sink/agg tasks.

---
 source/dnode/vnode/src/tq/tq.c      |  2 +
 source/libs/stream/src/streamTask.c | 73 +++++++++++++++--------------
 2 files changed, 40 insertions(+), 35 deletions(-)

diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 47900d540c..f98a69097b 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -860,6 +860,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
         vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
         pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory,
         (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam, nextProcessVer);
+
+    ASSERT(pChkInfo->checkpointVer <= pChkInfo->nextProcessVer);
   }
 
   return 0;
diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c
index cf606f2fc9..959d1382a3 100644
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@@ -458,12 +458,49 @@ void tFreeStreamTask(SStreamTask* pTask) {
   stDebug("s-task:0x%x free task completed", taskId);
 }
 
+static void setInitialVersionInfo(SStreamTask* pTask, int64_t ver) {
+  SCheckpointInfo* pChkInfo = &pTask->chkInfo;
+  SDataRange*      pRange = &pTask->dataRange;
+
+  // only set the version info for stream tasks without fill-history task
+  if ((pTask->info.fillHistory == 0) && (!HAS_RELATED_FILLHISTORY_TASK(pTask))) {
+    pChkInfo->checkpointVer = ver - 1;  // only update when generating checkpoint
+    pChkInfo->processedVer = ver - 1;   // already processed version
+    pChkInfo->nextProcessVer = ver;     // next processed version
+
+    pRange->range.maxVer = ver;
+    pRange->range.minVer = ver;
+  } else {
+    // the initial value of processedVer/nextProcessVer/checkpointVer for stream task with related fill-history task
+    // is set at the mnode.
+    if (pTask->info.fillHistory == 1) {
+      pChkInfo->checkpointVer = pRange->range.maxVer;
+      pChkInfo->processedVer = pRange->range.maxVer;
+      pChkInfo->nextProcessVer = pRange->range.maxVer + 1;
+    } else {
+      pChkInfo->checkpointVer = pRange->range.minVer - 1;
+      pChkInfo->processedVer = pRange->range.minVer - 1;
+      pChkInfo->nextProcessVer = pRange->range.minVer;
+
+      {  // for compatible purpose, remove it later
+        if (pRange->range.minVer == 0) {
+          pChkInfo->checkpointVer = 0;
+          pChkInfo->processedVer = 0;
+          pChkInfo->nextProcessVer = 1;
+          stDebug("s-task:%s update the processedVer to 0 from -1 due to compatible purpose", pTask->id.idStr);
+        }
+      }
+    }
+  }
+}
+
 int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver) {
   pTask->id.idStr = createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId);
   pTask->refCnt = 1;
 
   pTask->inputq.status = TASK_INPUT_STATUS__NORMAL;
   pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL;
+
   pTask->inputq.queue = streamQueueOpen(512 << 10);
   pTask->outputq.queue = streamQueueOpen(512 << 10);
   if (pTask->inputq.queue == NULL || pTask->outputq.queue == NULL) {
@@ -481,41 +518,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i
   }
 
   pTask->execInfo.created = taosGetTimestampMs();
-  SCheckpointInfo* pChkInfo = &pTask->chkInfo;
-  SDataRange*      pRange = &pTask->dataRange;
-
-  // only set the version info for stream tasks without fill-history task
-  if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-    if ((pTask->info.fillHistory == 0) && (!HAS_RELATED_FILLHISTORY_TASK(pTask))) {
-      pChkInfo->checkpointVer = ver - 1;  // only update when generating checkpoint
-      pChkInfo->processedVer = ver - 1;   // already processed version
-      pChkInfo->nextProcessVer = ver;     // next processed version
-
-      pRange->range.maxVer = ver;
-      pRange->range.minVer = ver;
-    } else {
-      // the initial value of processedVer/nextProcessVer/checkpointVer for stream task with related fill-history task
-      // is set at the mnode.
-      if (pTask->info.fillHistory == 1) {
-        pChkInfo->checkpointVer = pRange->range.maxVer;
-        pChkInfo->processedVer = pRange->range.maxVer;
-        pChkInfo->nextProcessVer = pRange->range.maxVer + 1;
-      } else {
-        pChkInfo->checkpointVer = pRange->range.minVer - 1;
-        pChkInfo->processedVer = pRange->range.minVer - 1;
-        pChkInfo->nextProcessVer = pRange->range.minVer;
-
-        { // for compatible purpose, remove it later
-          if (pRange->range.minVer == 0) {
-            pChkInfo->checkpointVer = 0;
-            pChkInfo->processedVer = 0;
-            pChkInfo->nextProcessVer = 1;
-            stDebug("s-task:%s update the processedVer to 0 from -1 due to compatible purpose", pTask->id.idStr);
-          }
-        }
-      }
-    }
-  }
+  setInitialVersionInfo(pTask, ver);
 
   pTask->pMeta = pMeta;
   pTask->pMsgCb = pMsgCb;