Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/TS-4243-3.0

2024-02-19 09:08:58 +08:00 · 2024-02-19 09:08:58 +08:00 · 45684e49c7
parent 24eff51382 8a4b79bf2c
commit 45684e49c7
65 changed files with 2168 additions and 1296 deletions
--- a/docs/zh/07-develop/07-tmq.mdx
+++ b/docs/zh/07-develop/07-tmq.mdx
--- a/docs/zh/07-develop/img_5.png
+++ b/docs/zh/07-develop/img_5.png
--- a/docs/zh/07-develop/img_6.png
+++ b/docs/zh/07-develop/img_6.png
--- a/docs/zh/07-develop/img_7.png
+++ b/docs/zh/07-develop/img_7.png
--- a/docs/zh/12-taos-sql/13-tmq.md
+++ b/docs/zh/12-taos-sql/13-tmq.md
@ -6,32 +6,68 @@ description: TDengine 消息队列提供的数据订阅功能

 TDengine 3.0.0.0 开始对消息队列做了大幅的优化和增强以简化用户的解决方案。

-## 创建订阅主题
+## 创建 topic
+
+TDengine 创建 topic 的个数上限通过参数 tmqMaxTopicNum 控制，默认 20 个。
+
+TDengine 使用 SQL 创建一个 topic，共有三种类型的 topic：
+
+### 查询 topic
+
+语法：

 ```sql
-CREATE TOPIC [IF NOT EXISTS] topic_name AS subquery;
+CREATE TOPIC [IF NOT EXISTS] topic_name as subquery
 ```

+通过 `SELECT` 语句订阅（包括 `SELECT *`，或 `SELECT ts, c1` 等指定查询订阅，可以带条件过滤、标量函数计算，但不支持聚合函数、不支持时间窗口聚合）。需要注意的是：

-TOPIC 支持过滤和标量函数和 UDF 标量函数，不支持 JOIN、GROUP BY、窗口切分子句、聚合函数和 UDF 聚合函数。列订阅规则如下：
+- 该类型 TOPIC 一旦创建则订阅数据的结构确定。
+- 被订阅或用于计算的列或标签不可被删除（`ALTER table DROP`）、修改（`ALTER table MODIFY`）。
+- 若发生表结构变更，新增的列不出现在结果中。
+- 对于 select \*，则订阅展开为创建时所有的列（子表、普通表为数据列，超级表为数据列加标签列）
+### 超级表 topic

-1. TOPIC 一旦创建则返回结果的字段确定
-2. 被订阅或用于计算的列不可被删除、修改
-3. 列可以新增，但新增的列不出现在订阅结果字段中
-4. 对于 select \*，则订阅展开为创建时所有的列（子表、普通表为数据列，超级表为数据列加标签列）
-
-
-## 删除订阅主题
+语法：

 ```sql
+CREATE TOPIC [IF NOT EXISTS] topic_name [with meta] AS STABLE stb_name [where_condition]
+```
+
+与 `SELECT * from stbName` 订阅的区别是：
+
+- 不会限制用户的表结构变更。
+- 返回的是非结构化的数据：返回数据的结构会随之超级表的表结构变化而变化。
+- with meta 参数可选，选择时将返回创建超级表，子表等语句，主要用于taosx做超级表迁移
+- where_condition 参数可选，选择时将用来过滤符合条件的子表，订阅这些子表。where 条件里不能有普通列，只能是tag或tbname，where条件里可以用函数，用来过滤tag，但是不能是聚合函数，因为子表tag值无法做聚合。也可以是常量表达式，比如 2 > 1（订阅全部子表），或者 false（订阅0个子表）
+- 返回数据不包含标签。
+
+### 数据库 topic
+
+语法：
+
+```sql
+CREATE TOPIC [IF NOT EXISTS] topic_name [with meta] AS DATABASE db_name;
+```
+
+通过该语句可创建一个包含数据库所有表数据的订阅
+
+- with meta 参数可选，选择时将返回创建数据库里所有超级表，子表的语句，主要用于taosx做数据库迁移
+
+说明： 超级表订阅和库订阅属于高级订阅模式，容易出错，如确实要使用，请咨询专业人员。
+
+## 删除 topic
+
+如果不再需要订阅数据，可以删除 topic，需要注意：只有当前未在订阅中的 TOPIC 才能被删除。
+
+```sql
+/* 删除 topic */
 DROP TOPIC [IF EXISTS] topic_name;
 ```

 此时如果该订阅主题上存在 consumer，则此 consumer 会收到一个错误。

-## 查看订阅主题
-
-## SHOW TOPICS
+## 查看 topic

 ```sql
 SHOW TOPICS;
@ -58,3 +94,11 @@ SHOW CONSUMERS;
 ```

 显示当前数据库下所有活跃的消费者的信息。
+
+## 查看订阅信息
+
+```sql
+SHOW SUBSCRIPTIONS;
+```
+
+显示 consumer 与 vgroup 之间的分配关系和消费信息
--- a/include/common/tcommon.h
+++ b/include/common/tcommon.h
@ -171,6 +171,7 @@ typedef enum EStreamType {
  STREAM_CHECKPOINT,
  STREAM_CREATE_CHILD_TABLE,
  STREAM_TRANS_STATE,
+  STREAM_MID_RETRIEVE,
 } EStreamType;

 #pragma pack(push, 1)
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@ -211,6 +211,7 @@ extern int32_t tsUptimeInterval;

 extern bool    tsDisableStream;
 extern int64_t tsStreamBufferSize;
+extern int     tsStreamAggCnt;
 extern bool    tsFilterScalarMode;
 extern int32_t tsMaxStreamBackendCache;
 extern int32_t tsPQSortMemThreshold;
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@ -431,7 +431,8 @@ typedef enum ENodeType {
  QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT,
  QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN,
  QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE,
-  QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL
+  QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL,
+  QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL
 } ENodeType;

 typedef struct {
@ -3639,6 +3640,7 @@ typedef struct {
  int64_t      timeout;
  STqOffsetVal reqOffset;
  int8_t       enableReplay;
+  int8_t       sourceExcluded;
 } SMqPollReq;

 int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq);
@ -3768,6 +3770,7 @@ typedef struct {
  int32_t      vgId;
  STqOffsetVal offset;
  int64_t      rows;
+  int64_t      ever;
 } OffsetRows;

 typedef struct {
@ -3925,6 +3928,9 @@ int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2
 #define SUBMIT_REQ_FROM_FILE          0x4

+#define  SOURCE_NULL  0
+#define  SOURCE_TAOSX 1
+
 typedef struct {
  int32_t        flags;
  SVCreateTbReq* pCreateTbReq;
@ -3935,7 +3941,8 @@ typedef struct {
    SArray* aRowP;
    SArray* aCol;
  };
-  int64_t ctimeMs;
+  int64_t       ctimeMs;
+  int8_t        source;
 } SSubmitTbData;

 typedef struct {
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@ -197,6 +197,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT

 void qStreamSetOpen(qTaskInfo_t tinfo);

+void qStreamSetSourceExcluded(qTaskInfo_t tinfo, int8_t sourceExcluded);
+
 void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset);

 SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
--- a/include/libs/function/functionMgt.h
+++ b/include/libs/function/functionMgt.h
@ -244,7 +244,7 @@ bool fmIsSkipScanCheckFunc(int32_t funcId);
 void getLastCacheDataType(SDataType* pType);
 SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList);

-int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc);
+int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc, SFunctionNode** pMergeFunc);

 typedef enum EFuncDataRequired {
  FUNC_DATA_REQUIRED_DATA_LOAD = 1,
--- a/include/libs/nodes/plannodes.h
+++ b/include/libs/nodes/plannodes.h
@ -257,6 +257,7 @@ typedef enum EWindowAlgorithm {
  SESSION_ALGO_STREAM_FINAL,
  SESSION_ALGO_STREAM_SINGLE,
  SESSION_ALGO_MERGE,
+  INTERVAL_ALGO_STREAM_MID,
 } EWindowAlgorithm;

 typedef struct SWindowLogicNode {
@ -587,6 +588,7 @@ typedef SIntervalPhysiNode SMergeAlignedIntervalPhysiNode;
 typedef SIntervalPhysiNode SStreamIntervalPhysiNode;
 typedef SIntervalPhysiNode SStreamFinalIntervalPhysiNode;
 typedef SIntervalPhysiNode SStreamSemiIntervalPhysiNode;
+typedef SIntervalPhysiNode SStreamMidIntervalPhysiNode;

 typedef struct SFillPhysiNode {
  SPhysiNode  node;
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@ -747,7 +747,6 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p
 int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq);

 int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq);
-void    tDeleteStreamRetrieveReq(SStreamRetrieveReq* pReq);
 void    tDeleteStreamDispatchReq(SStreamDispatchReq* pReq);

 typedef struct SStreamTaskCheckpointReq {
@ -764,7 +763,7 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask);
 int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg);
 int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);

-int32_t             streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg);
+int32_t             streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq);
 SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId);

 void    streamTaskInputFail(SStreamTask* pTask);
@ -890,6 +889,9 @@ int32_t buildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInf

 SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask);
 void*          streamDestroyStateMachine(SStreamTaskSM* pSM);
+
+int32_t broadcastRetrieveMsg(SStreamTask* pTask, SStreamRetrieveReq *req);
+void    sendRetrieveRsp(SStreamRetrieveReq *pReq, SRpcMsg* pRsp);
 #ifdef __cplusplus
 }
 #endif
--- a/include/os/osThread.h
+++ b/include/os/osThread.h
@ -69,6 +69,19 @@ typedef pthread_key_t        TdThreadKey;

 #define taosThreadCleanupPush pthread_cleanup_push
 #define taosThreadCleanupPop  pthread_cleanup_pop
+#if !defined(WINDOWS)
+#if defined(_TD_DARWIN_64)  // MACOS
+#define taosThreadRwlockAttrSetKindNP(A, B) ((void)0)
+#else  // LINUX
+#if _XOPEN_SOURCE >= 500 || _POSIX_C_SOURCE >= 200809L
+#define taosThreadRwlockAttrSetKindNP(A, B) pthread_rwlockattr_setkind_np(A, B)
+#else
+#define taosThreadRwlockAttrSetKindNP(A, B) ((void)0)
+#endif
+#endif
+#else  // WINDOWS
+#define taosThreadRwlockAttrSetKindNP(A, B) ((void)0)
+#endif

 #if defined(WINDOWS) && !defined(__USE_PTHREAD)
 #define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER_FORBID
--- a/source/client/src/clientTmq.c
+++ b/source/client/src/clientTmq.c
@ -63,6 +63,7 @@ struct tmq_conf_t {
  int8_t         withTbName;
  int8_t         snapEnable;
  int8_t         replayEnable;
+  int8_t         sourceExcluded;   // do not consume, bit
  uint16_t       port;
  int32_t        autoCommitInterval;
  char*          ip;
@ -82,6 +83,7 @@ struct tmq_t {
  int32_t        autoCommitInterval;
  int8_t         resetOffsetCfg;
  int8_t         replayEnable;
+  int8_t         sourceExcluded;   // do not consume, bit
  uint64_t       consumerId;
  tmq_commit_cb* commitCb;
  void*          commitCbUserParam;
@ -385,6 +387,10 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
      return TMQ_CONF_INVALID;
    }
  }
+  if (strcasecmp(key, "msg.consume.excluded") == 0) {
+    conf->sourceExcluded = taosStr2int64(value);
+    return TMQ_CONF_OK;
+  }

  if (strcasecmp(key, "td.connect.db") == 0) {
    return TMQ_CONF_OK;
@ -783,27 +789,26 @@ void tmqSendHbReq(void* param, void* tmrId) {
  req.consumerId = tmq->consumerId;
  req.epoch = tmq->epoch;
  taosRLockLatch(&tmq->lock);
-//  if(tmq->needReportOffsetRows){
-    req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
-    for(int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++){
-      SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
-      int32_t         numOfVgroups = taosArrayGetSize(pTopic->vgs);
-      TopicOffsetRows* data = taosArrayReserve(req.topics, 1);
-      strcpy(data->topicName, pTopic->topicName);
-      data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows));
-      for(int j = 0; j < numOfVgroups; j++){
-        SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
-        OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1);
-        offRows->vgId = pVg->vgId;
-        offRows->rows = pVg->numOfRows;
-        offRows->offset = pVg->offsetInfo.beginOffset;
-        char buf[TSDB_OFFSET_LEN] = {0};
-        tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset);
-        tscInfo("consumer:0x%" PRIx64 ",report offset: vgId:%d, offset:%s, rows:%"PRId64, tmq->consumerId, offRows->vgId, buf, offRows->rows);
-      }
+  req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
+  for(int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++){
+    SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
+    int32_t         numOfVgroups = taosArrayGetSize(pTopic->vgs);
+    TopicOffsetRows* data = taosArrayReserve(req.topics, 1);
+    strcpy(data->topicName, pTopic->topicName);
+    data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows));
+    for(int j = 0; j < numOfVgroups; j++){
+      SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
+      OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1);
+      offRows->vgId = pVg->vgId;
+      offRows->rows = pVg->numOfRows;
+      offRows->offset = pVg->offsetInfo.endOffset;
+      offRows->ever   = pVg->offsetInfo.walVerEnd;
+      char buf[TSDB_OFFSET_LEN] = {0};
+      tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset);
+      tscInfo("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%"PRId64", rows:%"PRId64,
+              tmq->consumerId, tmq->groupId, offRows->vgId, buf, offRows->ever, offRows->rows);
    }
-//    tmq->needReportOffsetRows = false;
-//  }
+  }
  taosRUnLockLatch(&tmq->lock);

  int32_t tlen = tSerializeSMqHbReq(NULL, 0, &req);
@ -1108,6 +1113,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
  pTmq->commitCbUserParam = conf->commitCbUserParam;
  pTmq->resetOffsetCfg = conf->resetOffset;
  pTmq->replayEnable = conf->replayEnable;
+  pTmq->sourceExcluded = conf->sourceExcluded;
  if(conf->replayEnable){
    pTmq->autoCommit = false;
  }
@ -1576,6 +1582,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl
  pReq->useSnapshot = tmq->useSnapshot;
  pReq->reqId = generateRequestId();
  pReq->enableReplay = tmq->replayEnable;
+  pReq->sourceExcluded = tmq->sourceExcluded;
 }

 SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@ -265,6 +265,7 @@ bool    tsDisableStream = false;
 int64_t tsStreamBufferSize = 128 * 1024 * 1024;
 bool    tsFilterScalarMode = false;
 int     tsResolveFQDNRetryTime = 100;  // seconds
+int     tsStreamAggCnt = 1000;

 char   tsS3Endpoint[TSDB_FQDN_LEN] = "<endpoint>";
 char   tsS3AccessKey[TSDB_FQDN_LEN] = "<accesskey>";
@ -750,6 +751,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
  if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
  if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
    return -1;
+  if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
+    return -1;

  if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER,
                  CFG_DYN_ENT_SERVER) != 0)
@ -1213,6 +1216,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {

  tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval;
  tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
+  tsStreamAggCnt = cfgGetItem(pCfg, "streamAggCnt")->i32;
+  tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
  tsStreamCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i32;
  tsSinkDataRate = cfgGetItem(pCfg, "streamSinkDataRate")->fval;

--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@ -6252,6 +6252,7 @@ int32_t tSerializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) {
      if (tEncodeI32(&encoder, offRows->vgId) < 0) return -1;
      if (tEncodeI64(&encoder, offRows->rows) < 0) return -1;
      if (tEncodeSTqOffsetVal(&encoder, &offRows->offset) < 0) return -1;
+      if (tEncodeI64(&encoder, offRows->ever) < 0) return -1;
    }
  }

@ -6289,6 +6290,7 @@ int32_t tDeserializeSMqHbReq(void *buf, int32_t bufLen, SMqHbReq *pReq) {
          if (tDecodeI32(&decoder, &offRows->vgId) < 0) return -1;
          if (tDecodeI64(&decoder, &offRows->rows) < 0) return -1;
          if (tDecodeSTqOffsetVal(&decoder, &offRows->offset) < 0) return -1;
+          if (tDecodeI64(&decoder, &offRows->ever) < 0) return -1;
        }
      }
    }
@ -6600,6 +6602,7 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
  if (tEncodeI64(&encoder, pReq->timeout) < 0) return -1;
  if (tSerializeSTqOffsetVal(&encoder, &pReq->reqOffset) < 0) return -1;
  if (tEncodeI8(&encoder, pReq->enableReplay) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->sourceExcluded) < 0) return -1;

  tEndEncode(&encoder);

@ -6640,6 +6643,10 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
    if (tDecodeI8(&decoder, &pReq->enableReplay) < 0) return -1;
  }

+  if (!tDecodeIsEnd(&decoder)) {
+    if (tDecodeI8(&decoder, &pReq->sourceExcluded) < 0) return -1;
+  }
+
  tEndDecode(&decoder);

  tDecoderClear(&decoder);
@ -8663,6 +8670,7 @@ static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubm
    }
  }
  if (tEncodeI64(pCoder, pSubmitTbData->ctimeMs) < 0) return -1;
+  if (tEncodeI8(pCoder, pSubmitTbData->source) < 0) return -1;

  tEndEncode(pCoder);
  return 0;
@ -8750,6 +8758,12 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa
      goto _exit;
    }
  }
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI8(pCoder, &pSubmitTbData->source) < 0) {
+      code = TSDB_CODE_INVALID_MSG;
+      goto _exit;
+    }
+  }

  tEndDecode(pCoder);

--- a/source/dnode/mnode/impl/inc/mndDef.h
+++ b/source/dnode/mnode/impl/inc/mndDef.h
@ -697,6 +697,8 @@ typedef struct {

  // 3.0.5.
  int64_t checkpointId;
+
+  int32_t indexForMultiAggBalance;
  char    reserve[256];

 } SStreamObj;
--- a/source/dnode/mnode/impl/src/mndDef.c
+++ b/source/dnode/mnode/impl/src/mndDef.c
@ -422,27 +422,12 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer, int8_t s
  return (void *)buf;
 }

-// SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) {
-//   SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp));
-//   if (pConsumerEpNew == NULL) return NULL;
-//   pConsumerEpNew->consumerId = pConsumerEpOld->consumerId;
-//   pConsumerEpNew->vgs = taosArrayDup(pConsumerEpOld->vgs, NULL);
-//   return pConsumerEpNew;
-// }
-//
-// void tDeleteSMqConsumerEp(void *data) {
-//   SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)data;
-//   taosArrayDestroy(pConsumerEp->vgs);
-// }
-
-int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) {
+int32_t tEncodeOffRows(void **buf, SArray *offsetRows){
  int32_t tlen = 0;
-  tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
-  tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp);
-  int32_t szVgs = taosArrayGetSize(pConsumerEp->offsetRows);
+  int32_t szVgs = taosArrayGetSize(offsetRows);
  tlen += taosEncodeFixedI32(buf, szVgs);
  for (int32_t j = 0; j < szVgs; ++j) {
-    OffsetRows *offRows = taosArrayGet(pConsumerEp->offsetRows, j);
+    OffsetRows *offRows = taosArrayGet(offsetRows, j);
    tlen += taosEncodeFixedI32(buf, offRows->vgId);
    tlen += taosEncodeFixedI64(buf, offRows->rows);
    tlen += taosEncodeFixedI8(buf, offRows->offset.type);
@ -454,53 +439,54 @@ int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) {
    } else {
      // do nothing
    }
+    tlen += taosEncodeFixedI64(buf, offRows->ever);
  }
-  // #if 0
-  //   int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
-  //   tlen += taosEncodeFixedI32(buf, sz);
-  //   for (int32_t i = 0; i < sz; i++) {
-  //     SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
-  //     tlen += tEncodeSMqVgEp(buf, pVgEp);
-  //   }
-  // #endif
+
  return tlen;
 }

+int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
+  tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp);
+
+
+  return tlen + tEncodeOffRows(buf, pConsumerEp->offsetRows);
+}
+
+void *tDecodeOffRows(const void *buf, SArray **offsetRows, int8_t sver){
+  int32_t szVgs = 0;
+  buf = taosDecodeFixedI32(buf, &szVgs);
+  if (szVgs > 0) {
+    *offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows));
+    if (NULL == *offsetRows) return NULL;
+    for (int32_t j = 0; j < szVgs; ++j) {
+      OffsetRows *offRows = taosArrayReserve(*offsetRows, 1);
+      buf = taosDecodeFixedI32(buf, &offRows->vgId);
+      buf = taosDecodeFixedI64(buf, &offRows->rows);
+      buf = taosDecodeFixedI8(buf, &offRows->offset.type);
+      if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) {
+        buf = taosDecodeFixedI64(buf, &offRows->offset.uid);
+        buf = taosDecodeFixedI64(buf, &offRows->offset.ts);
+      } else if (offRows->offset.type == TMQ_OFFSET__LOG) {
+        buf = taosDecodeFixedI64(buf, &offRows->offset.version);
+      } else {
+        // do nothing
+      }
+      if(sver > 2){
+        buf = taosDecodeFixedI64(buf, &offRows->ever);
+      }
+    }
+  }
+  return (void *)buf;
+}
+
 void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp, int8_t sver) {
  buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
  buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp), sver);
  if (sver > 1) {
-    int32_t szVgs = 0;
-    buf = taosDecodeFixedI32(buf, &szVgs);
-    if (szVgs > 0) {
-      pConsumerEp->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows));
-      if (NULL == pConsumerEp->offsetRows) return NULL;
-      for (int32_t j = 0; j < szVgs; ++j) {
-        OffsetRows *offRows = taosArrayReserve(pConsumerEp->offsetRows, 1);
-        buf = taosDecodeFixedI32(buf, &offRows->vgId);
-        buf = taosDecodeFixedI64(buf, &offRows->rows);
-        buf = taosDecodeFixedI8(buf, &offRows->offset.type);
-        if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) {
-          buf = taosDecodeFixedI64(buf, &offRows->offset.uid);
-          buf = taosDecodeFixedI64(buf, &offRows->offset.ts);
-        } else if (offRows->offset.type == TMQ_OFFSET__LOG) {
-          buf = taosDecodeFixedI64(buf, &offRows->offset.version);
-        } else {
-          // do nothing
-        }
-      }
-    }
+    buf = tDecodeOffRows(buf, &pConsumerEp->offsetRows, sver);
  }
-  // #if 0
-  //   int32_t sz;
-  //   buf = taosDecodeFixedI32(buf, &sz);
-  //   pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *));
-  //   for (int32_t i = 0; i < sz; i++) {
-  //     SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
-  //     buf = tDecodeSMqVgEp(buf, pVgEp);
-  //     taosArrayPush(pConsumerEp->vgs, &pVgEp);
-  //   }
-  // #endif

  return (void *)buf;
 }
@ -596,22 +582,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
  tlen += taosEncodeArray(buf, pSub->unassignedVgs, (FEncode)tEncodeSMqVgEp);
  tlen += taosEncodeString(buf, pSub->dbName);

-  int32_t szVgs = taosArrayGetSize(pSub->offsetRows);
-  tlen += taosEncodeFixedI32(buf, szVgs);
-  for (int32_t j = 0; j < szVgs; ++j) {
-    OffsetRows *offRows = taosArrayGet(pSub->offsetRows, j);
-    tlen += taosEncodeFixedI32(buf, offRows->vgId);
-    tlen += taosEncodeFixedI64(buf, offRows->rows);
-    tlen += taosEncodeFixedI8(buf, offRows->offset.type);
-    if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) {
-      tlen += taosEncodeFixedI64(buf, offRows->offset.uid);
-      tlen += taosEncodeFixedI64(buf, offRows->offset.ts);
-    } else if (offRows->offset.type == TMQ_OFFSET__LOG) {
-      tlen += taosEncodeFixedI64(buf, offRows->offset.version);
-    } else {
-      // do nothing
-    }
-  }
+  tlen += tEncodeOffRows(buf, pSub->offsetRows);
  tlen += taosEncodeString(buf, pSub->qmsg);
  return tlen;
 }
@ -639,26 +610,7 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub, int8_t sver) {
  buf = taosDecodeStringTo(buf, pSub->dbName);

  if (sver > 1) {
-    int32_t szVgs = 0;
-    buf = taosDecodeFixedI32(buf, &szVgs);
-    if (szVgs > 0) {
-      pSub->offsetRows = taosArrayInit(szVgs, sizeof(OffsetRows));
-      if (NULL == pSub->offsetRows) return NULL;
-      for (int32_t j = 0; j < szVgs; ++j) {
-        OffsetRows *offRows = taosArrayReserve(pSub->offsetRows, 1);
-        buf = taosDecodeFixedI32(buf, &offRows->vgId);
-        buf = taosDecodeFixedI64(buf, &offRows->rows);
-        buf = taosDecodeFixedI8(buf, &offRows->offset.type);
-        if (offRows->offset.type == TMQ_OFFSET__SNAPSHOT_DATA || offRows->offset.type == TMQ_OFFSET__SNAPSHOT_META) {
-          buf = taosDecodeFixedI64(buf, &offRows->offset.uid);
-          buf = taosDecodeFixedI64(buf, &offRows->offset.ts);
-        } else if (offRows->offset.type == TMQ_OFFSET__LOG) {
-          buf = taosDecodeFixedI64(buf, &offRows->offset.version);
-        } else {
-          // do nothing
-        }
-      }
-    }
+    buf = tDecodeOffRows(buf, &pSub->offsetRows, sver);
    buf = taosDecodeString(buf, &pSub->qmsg);
  } else {
    pSub->qmsg = taosStrdup("");
--- a/source/dnode/mnode/impl/src/mndScheduler.c
+++ b/source/dnode/mnode/impl/src/mndScheduler.c
@ -27,9 +27,6 @@
 #define SINK_NODE_LEVEL (0)
 extern bool tsDeployOnSnode;

-static int32_t doAddSinkTask(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup,
-                             SEpSet* pEpset, bool isFillhistory);
-
 int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
                           int64_t watermark, int64_t deleteMark) {
  SNode*      pAst = NULL;
@ -89,6 +86,8 @@ END:
 int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
  STaskOutputInfo* pInfo = &pTask->outputInfo;

+  mDebug("mndSetSinkTaskInfo to sma or table, taskId:%s", pTask->id.idStr);
+
  if (pStream->smaId != 0) {
    pInfo->type = TASK_OUTPUT__SMA;
    pInfo->smaSink.smaId = pStream->smaId;
@ -157,12 +156,7 @@ int32_t mndAssignStreamTaskToVgroup(SMnode* pMnode, SStreamTask* pTask, SSubplan

  plan->execNode.nodeId = pTask->info.nodeId;
  plan->execNode.epSet = pTask->info.epSet;
-  if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) {
-    terrno = TSDB_CODE_QRY_INVALID_INPUT;
-    return -1;
-  }
-
-  return 0;
+  return qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen);
 }

 SSnodeObj* mndSchedFetchOneSnode(SMnode* pMnode) {
@ -184,32 +178,79 @@ int32_t mndAssignStreamTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan*
  plan->execNode.epSet = pTask->info.epSet;
  mDebug("s-task:0x%x set the agg task to snode:%d", pTask->id.taskId, SNODE_HANDLE);

-  if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) {
-    terrno = TSDB_CODE_QRY_INVALID_INPUT;
-    return -1;
-  }
-  return 0;
+  return qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen);
 }

-// todo random choose a node to do compute
-SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) {
+// random choose a node to do compute
+SVgObj* mndSchedFetchOneVg(SMnode* pMnode, SStreamObj* pStream) {
+  SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->sourceDb);
+  if (pDbObj == NULL) {
+    terrno = TSDB_CODE_QRY_INVALID_INPUT;
+    return NULL;
+  }
+
+  if(pStream->indexForMultiAggBalance == -1){
+    taosSeedRand(taosSafeRand());
+    pStream->indexForMultiAggBalance = taosRand() % pDbObj->cfg.numOfVgroups;
+  }
+
+  int32_t index = 0;
  void*   pIter = NULL;
  SVgObj* pVgroup = NULL;
  while (1) {
    pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
    if (pIter == NULL) break;
-    if (pVgroup->dbUid != dbUid) {
+    if (pVgroup->dbUid != pStream->sourceDbUid) {
      sdbRelease(pMnode->pSdb, pVgroup);
      continue;
    }
-    sdbCancelFetch(pMnode->pSdb, pIter);
-    return pVgroup;
+    if (index++ == pStream->indexForMultiAggBalance){
+      pStream->indexForMultiAggBalance++;
+      pStream->indexForMultiAggBalance %= pDbObj->cfg.numOfVgroups;
+      sdbCancelFetch(pMnode->pSdb, pIter);
+      break;
+    }
+    sdbRelease(pMnode->pSdb, pVgroup);
  }
+  sdbRelease(pMnode->pSdb, pDbObj);
+
  return pVgroup;
 }

+static int32_t doAddSinkTask(SStreamObj* pStream, SMnode* pMnode, SVgObj* pVgroup,
+                      SEpSet* pEpset, bool isFillhistory) {
+  int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
+  SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
+
+  SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, *pTaskList, pStream->conf.fillHistory);
+  if (pTask == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return terrno;
+  }
+
+  mDebug("doAddSinkTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);
+
+  pTask->info.nodeId = pVgroup->vgId;
+  pTask->info.epSet = mndGetVgroupEpset(pMnode, pVgroup);
+  return mndSetSinkTaskInfo(pStream, pTask);
+}
+
+static int32_t doAddSinkTaskToVg(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset, SVgObj* vgObj){
+  int32_t code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, false);
+  if (code != 0) {
+    return code;
+  }
+  if(pStream->conf.fillHistory){
+    code = doAddSinkTask(pStream, pMnode, vgObj, pEpset, true);
+    if (code != 0) {
+      return code;
+    }
+  }
+  return TDB_CODE_SUCCESS;
+}
+
 // create sink node for each vgroup.
-int32_t doAddShuffleSinkTask(SMnode* pMnode, SArray* pTaskList, SStreamObj* pStream, SEpSet* pEpset, bool fillHistory) {
+static int32_t doAddShuffleSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset) {
  SSdb* pSdb = pMnode->pSdb;
  void* pIter = NULL;

@ -225,27 +266,16 @@ int32_t doAddShuffleSinkTask(SMnode* pMnode, SArray* pTaskList, SStreamObj* pStr
      continue;
    }

-    doAddSinkTask(pStream, pTaskList, pMnode, pVgroup->vgId, pVgroup, pEpset, fillHistory);
+    int32_t code = doAddSinkTaskToVg(pMnode, pStream, pEpset, pVgroup);
+    if(code != 0){
+      sdbRelease(pSdb, pVgroup);
+      return code;
+    }
+
    sdbRelease(pSdb, pVgroup);
  }

-  return 0;
-}
-
-int32_t doAddSinkTask(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup,
-                      SEpSet* pEpset, bool isFillhistory) {
-  int64_t      uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
-  SStreamTask* pTask =
-      tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, pTaskList, pStream->conf.fillHistory);
-  if (pTask == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  pTask->info.nodeId = vgId;
-  pTask->info.epSet = mndGetVgroupEpset(pMnode, pVgroup);
-  mndSetSinkTaskInfo(pStream, pTask);
-  return 0;
+  return TDB_CODE_SUCCESS;
 }

 static int64_t getVgroupLastVer(const SArray* pList, int32_t vgId) {
@ -290,44 +320,35 @@ static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVe
  }
 }

-static int32_t addSourceTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTaskList, SArray* pSinkTaskList,
-                             SStreamObj* pStream, SSubplan* plan, uint64_t uid, SEpSet* pEpset, int64_t skey,
-                             SArray* pVerList, bool fillHistory, bool hasExtraSink, bool hasFillHistory) {
-  int64_t t = pStream->conf.triggerParam;
-  SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, fillHistory, t, pTaskList, hasFillHistory);
+static SStreamTask* buildSourceTask(SStreamObj* pStream, SEpSet* pEpset,
+                                    bool isFillhistory, bool useTriggerParam) {
+  uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
+  SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
+
+  SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset,
+                                      isFillhistory, useTriggerParam ? pStream->conf.triggerParam : 0,
+                                      *pTaskList, pStream->conf.fillHistory);
  if (pTask == NULL) {
-    return terrno;
+    return NULL;
  }

-  streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
-
-  // sink or dispatch
-  if (hasExtraSink) {
-    mndAddDispatcherForInternalTask(pMnode, pStream, pSinkTaskList, pTask);
-  } else {
-    mndSetSinkTaskInfo(pStream, pTask);
-  }
-
-  if (mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup) < 0) {
-    return terrno;
-  }
-
-  for(int32_t i = 0; i < taosArrayGetSize(pSinkTaskList); ++i) {
-    SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, i);
-    streamTaskSetUpstreamInfo(pSinkTask, pTask);
-  }
-
-  return TSDB_CODE_SUCCESS;
+  return pTask;
 }

-static SArray* addNewTaskList(SArray* pTasksList) {
+static void addNewTaskList(SStreamObj* pStream){
  SArray* pTaskList = taosArrayInit(0, POINTER_BYTES);
-  taosArrayPush(pTasksList, &pTaskList);
-  return pTaskList;
+  taosArrayPush(pStream->tasks, &pTaskList);
+  if (pStream->conf.fillHistory) {
+    pTaskList = taosArrayInit(0, POINTER_BYTES);
+    taosArrayPush(pStream->pHTasksList, &pTaskList);
+  }
 }

 // set the history task id
-static void setHTasksId(SArray* pTaskList, const SArray* pHTaskList) {
+static void setHTasksId(SStreamObj* pStream) {
+  SArray* pTaskList = *(SArray**)taosArrayGetLast(pStream->tasks);
+  SArray* pHTaskList = *(SArray**)taosArrayGetLast(pStream->pHTasksList);
+
  for (int32_t i = 0; i < taosArrayGetSize(pTaskList); ++i) {
    SStreamTask** pStreamTask = taosArrayGet(pTaskList, i);
    SStreamTask** pHTask = taosArrayGet(pHTaskList, i);
@ -343,30 +364,63 @@ static void setHTasksId(SArray* pTaskList, const SArray* pHTaskList) {
  }
 }

-static int32_t addSourceTasksForOneLevelStream(SMnode* pMnode, const SQueryPlan* pPlan, SStreamObj* pStream,
-                                               SEpSet* pEpset, bool hasExtraSink, int64_t skey, SArray* pVerList) {
-  // create exec stream task, since only one level, the exec task is also the source task
-  SArray* pTaskList = addNewTaskList(pStream->tasks);
-  SSdb*   pSdb = pMnode->pSdb;
-
-  SArray* pHTaskList = NULL;
-  if (pStream->conf.fillHistory) {
-    pHTaskList = addNewTaskList(pStream->pHTasksList);
+static int32_t doAddSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream, SEpSet* pEpset,
+                               int64_t skey, SArray* pVerList, SVgObj* pVgroup, bool isFillhistory, bool useTriggerParam ){
+  // new stream task
+  SStreamTask* pTask = buildSourceTask(pStream, pEpset, isFillhistory, useTriggerParam);
+  if(pTask == NULL){
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return terrno;
  }
+  mDebug("doAddSourceTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);

-  SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
+  streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
+
+  int32_t code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup);
+  if(code != 0){
+    terrno = code;
+    return terrno;
+  }
+  return TDB_CODE_SUCCESS;
+}
+
+static SSubplan* getScanSubPlan(const SQueryPlan* pPlan){
+  int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans);
+  SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, numOfPlanLevel - 1);
  if (LIST_LENGTH(inner->pNodeList) != 1) {
    terrno = TSDB_CODE_QRY_INVALID_INPUT;
-    return -1;
+    return NULL;
  }

  SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
  if (plan->subplanType != SUBPLAN_TYPE_SCAN) {
    terrno = TSDB_CODE_QRY_INVALID_INPUT;
-    return -1;
+    return NULL;
+  }
+  return plan;
+}
+
+static SSubplan* getAggSubPlan(const SQueryPlan* pPlan, int index){
+  SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, index);
+  if (LIST_LENGTH(inner->pNodeList) != 1) {
+    terrno = TSDB_CODE_QRY_INVALID_INPUT;
+    return NULL;
  }

+  SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
+  if (plan->subplanType != SUBPLAN_TYPE_MERGE) {
+    terrno = TSDB_CODE_QRY_INVALID_INPUT;
+    return NULL;
+  }
+  return plan;
+}
+
+static int32_t addSourceTask(SMnode* pMnode, SSubplan* plan, SStreamObj* pStream,
+                             SEpSet* pEpset, int64_t nextWindowSkey, SArray* pVerList, bool useTriggerParam) {
+  addNewTaskList(pStream);
+
  void* pIter = NULL;
+  SSdb* pSdb = pMnode->pSdb;
  while (1) {
    SVgObj* pVgroup;
    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
@ -379,187 +433,15 @@ static int32_t addSourceTasksForOneLevelStream(SMnode* pMnode, const SQueryPlan*
      continue;
    }

-    // new stream task
-    SArray** pSinkTaskList = taosArrayGet(pStream->tasks, SINK_NODE_LEVEL);
-    int32_t  code = addSourceTask(pMnode, pVgroup, pTaskList, *pSinkTaskList, pStream, plan, pStream->uid, pEpset, skey,
-                                  pVerList, false, hasExtraSink, pStream->conf.fillHistory);
-    if (code != TSDB_CODE_SUCCESS) {
+    int code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, false, useTriggerParam);
+    if(code != 0){
      sdbRelease(pSdb, pVgroup);
-      return -1;
-    }
-
-    if (pStream->conf.fillHistory) {
-      SArray** pHSinkTaskList = taosArrayGet(pStream->pHTasksList, SINK_NODE_LEVEL);
-      code = addSourceTask(pMnode, pVgroup, pHTaskList, *pHSinkTaskList, pStream, plan, pStream->hTaskUid, pEpset, skey,
-                           pVerList, true, hasExtraSink, true);
-    }
-
-    sdbRelease(pSdb, pVgroup);
-    if (code != TSDB_CODE_SUCCESS) {
-      return -1;
-    }
-  }
-
-  if (pStream->conf.fillHistory) {
-    setHTasksId(pTaskList, pHTaskList);
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t addSourceTaskForMultiLevelStream(SArray* pTaskList, bool isFillhistory, int64_t uid,
-                                                SStreamTask* pDownstreamTask, SMnode* pMnode, SSubplan* pPlan,
-                                                SVgObj* pVgroup, SEpSet* pEpset, int64_t skey, SArray* pVerList,
-                                                bool hasFillHistory) {
-  SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, 0, pTaskList, hasFillHistory);
-  if (pTask == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
-
-  // all the source tasks dispatch result to a single agg node.
-  streamTaskSetFixedDownstreamInfo(pTask, pDownstreamTask);
-  if (mndAssignStreamTaskToVgroup(pMnode, pTask, pPlan, pVgroup) < 0) {
-    return -1;
-  }
-
-  return streamTaskSetUpstreamInfo(pDownstreamTask, pTask);
-}
-
-static int32_t doAddAggTask(uint64_t uid, SArray* pTaskList, SArray* pSinkNodeList, SMnode* pMnode, SStreamObj* pStream,
-                            SEpSet* pEpset, bool fillHistory, SStreamTask** pAggTask, bool hasFillhistory) {
-  *pAggTask = tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, fillHistory, pStream->conf.triggerParam, pTaskList, hasFillhistory);
-  if (*pAggTask == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  // dispatch
-  if (mndAddDispatcherForInternalTask(pMnode, pStream, pSinkNodeList, *pAggTask) < 0) {
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, SEpSet* pEpset,
-                          SStreamTask** pAggTask, SStreamTask** pHAggTask) {
-  SArray* pAggTaskList = addNewTaskList(pStream->tasks);
-  SSdb*   pSdb = pMnode->pSdb;
-
-  SNodeListNode* pInnerNode = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
-  SSubplan*      plan = (SSubplan*)nodesListGetNode(pInnerNode->pNodeList, 0);
-  if (plan->subplanType != SUBPLAN_TYPE_MERGE) {
-    terrno = TSDB_CODE_QRY_INVALID_INPUT;
-    return -1;
-  }
-
-  *pAggTask = NULL;
-  SArray* pSinkNodeList = taosArrayGetP(pStream->tasks, SINK_NODE_LEVEL);
-
-  int32_t code = doAddAggTask(pStream->uid, pAggTaskList, pSinkNodeList, pMnode, pStream, pEpset, false, pAggTask,
-                              pStream->conf.fillHistory);
-  if (code != TSDB_CODE_SUCCESS) {
-    return -1;
-  }
-
-  SVgObj*    pVgroup = NULL;
-  SSnodeObj* pSnode = NULL;
-
-  if (tsDeployOnSnode) {
-    pSnode = mndSchedFetchOneSnode(pMnode);
-    if (pSnode == NULL) {
-      pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid);
-    }
-  } else {
-    pVgroup = mndSchedFetchOneVg(pMnode, pStream->sourceDbUid);
-  }
-
-  if (pSnode != NULL) {
-    code = mndAssignStreamTaskToSnode(pMnode, *pAggTask, plan, pSnode);
-  } else {
-    code = mndAssignStreamTaskToVgroup(pMnode, *pAggTask, plan, pVgroup);
-  }
-
-  if (pStream->conf.fillHistory) {
-    SArray* pHAggTaskList = addNewTaskList(pStream->pHTasksList);
-    SArray* pHSinkNodeList = taosArrayGetP(pStream->pHTasksList, SINK_NODE_LEVEL);
-
-    *pHAggTask = NULL;
-    code = doAddAggTask(pStream->hTaskUid, pHAggTaskList, pHSinkNodeList, pMnode, pStream, pEpset, pStream->conf.fillHistory,
-                        pHAggTask, pStream->conf.fillHistory);
-    if (code != TSDB_CODE_SUCCESS) {
-      if (pSnode != NULL) {
-        sdbRelease(pSdb, pSnode);
-      } else {
-        sdbRelease(pSdb, pVgroup);
-      }
      return code;
    }

-    if (pSnode != NULL) {
-      code = mndAssignStreamTaskToSnode(pMnode, *pHAggTask, plan, pSnode);
-    } else {
-      code = mndAssignStreamTaskToVgroup(pMnode, *pHAggTask, plan, pVgroup);
-    }
-
-    setHTasksId(pAggTaskList, pHAggTaskList);
-  }
-
-  if (pSnode != NULL) {
-    sdbRelease(pSdb, pSnode);
-  } else {
-    sdbRelease(pSdb, pVgroup);
-  }
-
-  return code;
-}
-
-static int32_t addSourceTasksForMultiLevelStream(SMnode* pMnode, SQueryPlan* pPlan, SStreamObj* pStream,
-                                                 SStreamTask* pDownstreamTask, SStreamTask* pHDownstreamTask,
-                                                 SEpSet* pEpset, int64_t skey, SArray* pVerList) {
-  SArray* pSourceTaskList = addNewTaskList(pStream->tasks);
-
-  SArray* pHSourceTaskList = NULL;
-  if (pStream->conf.fillHistory) {
-    pHSourceTaskList = addNewTaskList(pStream->pHTasksList);
-  }
-
-  SSdb*          pSdb = pMnode->pSdb;
-  SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 1);
-  SSubplan*      plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
-  if (plan->subplanType != SUBPLAN_TYPE_SCAN) {
-    terrno = TSDB_CODE_QRY_INVALID_INPUT;
-    return -1;
-  }
-
-  void* pIter = NULL;
-  while (1) {
-    SVgObj* pVgroup;
-    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
-    if (pIter == NULL) {
-      break;
-    }
-
-    if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) {
-      sdbRelease(pSdb, pVgroup);
-      continue;
-    }
-
-    int32_t code = addSourceTaskForMultiLevelStream(pSourceTaskList, false, pStream->uid, pDownstreamTask, pMnode, plan, pVgroup, pEpset,
-                                   skey, pVerList, pStream->conf.fillHistory);
-    if (code != TSDB_CODE_SUCCESS) {
-      sdbRelease(pSdb, pVgroup);
-      terrno = code;
-      return -1;
-    }
-
    if (pStream->conf.fillHistory) {
-      code = addSourceTaskForMultiLevelStream(pHSourceTaskList, true, pStream->hTaskUid, pHDownstreamTask, pMnode, plan, pVgroup, pEpset,
-                             skey, pVerList, pStream->conf.fillHistory);
-      if (code != TSDB_CODE_SUCCESS) {
+      code = doAddSourceTask(pMnode, plan, pStream, pEpset, nextWindowSkey, pVerList, pVgroup, true, useTriggerParam);
+      if(code != 0){
        sdbRelease(pSdb, pVgroup);
        return code;
      }
@ -569,49 +451,160 @@ static int32_t addSourceTasksForMultiLevelStream(SMnode* pMnode, SQueryPlan* pPl
  }

  if (pStream->conf.fillHistory) {
-    setHTasksId(pSourceTaskList, pHSourceTaskList);
+    setHTasksId(pStream);
  }

  return TSDB_CODE_SUCCESS;
 }

-static int32_t addSinkTasks(SArray* pTasksList, SMnode* pMnode, SStreamObj* pStream, SArray** pCreatedTaskList,
-                            SEpSet* pEpset, bool fillHistory) {
-  SArray* pSinkTaskList = addNewTaskList(pTasksList);
-  if (pStream->fixedSinkVgId == 0) {
-    if (doAddShuffleSinkTask(pMnode, pSinkTaskList, pStream, pEpset, fillHistory) < 0) {
-      // TODO free
-      return -1;
+static SStreamTask* buildAggTask(SStreamObj* pStream, SEpSet* pEpset, bool isFillhistory, bool useTriggerParam) {
+  uint64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
+  SArray** pTaskList = (isFillhistory) ? taosArrayGetLast(pStream->pHTasksList) : taosArrayGetLast(pStream->tasks);
+
+  SStreamTask* pAggTask = tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, isFillhistory,
+                                         useTriggerParam ? pStream->conf.triggerParam : 0,
+                                         *pTaskList, pStream->conf.fillHistory);
+  if (pAggTask == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  return pAggTask;
+}
+
+static int32_t doAddAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset,
+                            SVgObj* pVgroup, SSnodeObj* pSnode, bool isFillhistory, bool useTriggerParam){
+  int32_t      code = 0;
+  SStreamTask* pTask = buildAggTask(pStream, pEpset, isFillhistory, useTriggerParam);
+  if (pTask == NULL) {
+    return terrno;
+  }
+  if (pSnode != NULL) {
+    code = mndAssignStreamTaskToSnode(pMnode, pTask, plan, pSnode);
+    mDebug("doAddAggTask taskId:%s, snode id:%d, isFillHistory:%d", pTask->id.idStr, pSnode->id, isFillhistory);
+
+  } else {
+    code = mndAssignStreamTaskToVgroup(pMnode, pTask, plan, pVgroup);
+    mDebug("doAddAggTask taskId:%s, vgId:%d, isFillHistory:%d", pTask->id.idStr, pVgroup->vgId, isFillhistory);
+  }
+  return code;
+}
+
+static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SSubplan* plan, SEpSet* pEpset, bool useTriggerParam){
+  SVgObj*    pVgroup = NULL;
+  SSnodeObj* pSnode = NULL;
+  int32_t    code = 0;
+  if (tsDeployOnSnode) {
+    pSnode = mndSchedFetchOneSnode(pMnode);
+    if (pSnode == NULL) {
+      pVgroup = mndSchedFetchOneVg(pMnode, pStream);
    }
  } else {
-    if (doAddSinkTask(pStream, pSinkTaskList, pMnode, pStream->fixedSinkVgId, &pStream->fixedSinkVg, pEpset,
-                           fillHistory) < 0) {
-      // TODO free
-      return -1;
+    pVgroup = mndSchedFetchOneVg(pMnode, pStream);
+  }
+
+  code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, false, useTriggerParam);
+  if(code != 0){
+    goto END;
+  }
+
+  if (pStream->conf.fillHistory) {
+    code = doAddAggTask(pStream, pMnode, plan, pEpset, pVgroup, pSnode, true, useTriggerParam);
+    if(code != 0){
+      goto END;
+    }
+
+    setHTasksId(pStream);
+  }
+
+  END:
+  if (pSnode != NULL) {
+    sdbRelease(pMnode->pSdb, pSnode);
+  } else {
+    sdbRelease(pMnode->pSdb, pVgroup);
+  }
+  return code;
+}
+
+static int32_t addSinkTask(SMnode* pMnode, SStreamObj* pStream, SEpSet* pEpset){
+  int32_t code = 0;
+  addNewTaskList(pStream);
+
+  if (pStream->fixedSinkVgId == 0) {
+    code = doAddShuffleSinkTask(pMnode, pStream, pEpset);
+    if (code != 0) {
+      return code;
+    }
+  } else {
+    code = doAddSinkTaskToVg(pMnode, pStream, pEpset, &pStream->fixedSinkVg);
+    if (code != 0) {
+      return code;
    }
  }

-  *pCreatedTaskList = pSinkTaskList;
-  return TSDB_CODE_SUCCESS;
+  if (pStream->conf.fillHistory) {
+    setHTasksId(pStream);
+  }
+  return TDB_CODE_SUCCESS;
 }

-static void setSinkTaskUpstreamInfo(SArray* pTasksList, const SStreamTask* pUpstreamTask) {
-  if (taosArrayGetSize(pTasksList) < SINK_NODE_LEVEL || pUpstreamTask == NULL) {
-    return;
+static void bindTaskToSinkTask(SStreamObj* pStream, SMnode* pMnode, SArray* pSinkTaskList, SStreamTask* task){
+  mndAddDispatcherForInternalTask(pMnode, pStream, pSinkTaskList, task);
+  for(int32_t k = 0; k < taosArrayGetSize(pSinkTaskList); k++) {
+    SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, k);
+    streamTaskSetUpstreamInfo(pSinkTask, task);
  }
+  mDebug("bindTaskToSinkTask taskId:%s to sink task list", task->id.idStr);
+}

-  SArray* pSinkTaskList = taosArrayGetP(pTasksList, SINK_NODE_LEVEL);
-  for(int32_t i = 0; i < taosArrayGetSize(pSinkTaskList); ++i) {
-    SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, i);
-    streamTaskSetUpstreamInfo(pSinkTask, pUpstreamTask);
+static void bindAggSink(SStreamObj* pStream, SMnode* pMnode, SArray* tasks) {
+  SArray* pSinkTaskList = taosArrayGetP(tasks, SINK_NODE_LEVEL);
+  SArray** pAggTaskList = taosArrayGetLast(tasks);
+
+  for(int i = 0; i < taosArrayGetSize(*pAggTaskList); i++){
+    SStreamTask* pAggTask = taosArrayGetP(*pAggTaskList, i);
+    bindTaskToSinkTask(pStream, pMnode, pSinkTaskList, pAggTask);
+    mDebug("bindAggSink taskId:%s to sink task list", pAggTask->id.idStr);
  }
 }

+static void bindSourceSink(SStreamObj* pStream, SMnode* pMnode, SArray* tasks, bool hasExtraSink) {
+  SArray* pSinkTaskList = taosArrayGetP(tasks, SINK_NODE_LEVEL);
+  SArray* pSourceTaskList = taosArrayGetP(tasks, hasExtraSink ? SINK_NODE_LEVEL + 1 : SINK_NODE_LEVEL);
+
+  for(int i = 0; i < taosArrayGetSize(pSourceTaskList); i++){
+    SStreamTask* pSourceTask = taosArrayGetP(pSourceTaskList, i);
+    mDebug("bindSourceSink taskId:%s to sink task list", pSourceTask->id.idStr);
+
+    if (hasExtraSink) {
+      bindTaskToSinkTask(pStream, pMnode, pSinkTaskList, pSourceTask);
+    } else {
+      mndSetSinkTaskInfo(pStream, pSourceTask);
+    }
+  }
+}
+
+static void bindTwoLevel(SArray* tasks, int32_t begin, int32_t end) {
+  size_t size = taosArrayGetSize(tasks);
+  ASSERT(size >= 2);
+  SArray* pDownTaskList = taosArrayGetP(tasks, size - 1);
+  SArray* pUpTaskList = taosArrayGetP(tasks, size - 2);
+
+  SStreamTask** pDownTask = taosArrayGetLast(pDownTaskList);
+  end = end > taosArrayGetSize(pUpTaskList) ? taosArrayGetSize(pUpTaskList): end;
+  for(int i = begin; i < end; i++){
+    SStreamTask* pUpTask = taosArrayGetP(pUpTaskList, i);
+    pUpTask->info.selfChildId = i - begin;
+    streamTaskSetFixedDownstreamInfo(pUpTask, *pDownTask);
+    streamTaskSetUpstreamInfo(*pDownTask, pUpTask);
+  }
+  mDebug("bindTwoLevel task list(%d-%d) to taskId:%s", begin, end - 1, (*(pDownTask))->id.idStr);
+}
+
 static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, SEpSet* pEpset, int64_t skey,
                                SArray* pVerList) {
  SSdb*   pSdb = pMnode->pSdb;
  int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans);
-
  bool    hasExtraSink = false;
  bool    externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0;
  SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb);
@ -623,54 +616,90 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
  bool multiTarget = (pDbObj->cfg.numOfVgroups > 1);
  sdbRelease(pSdb, pDbObj);

+  mDebug("doScheduleStream numOfPlanLevel:%d, exDb:%d, multiTarget:%d, fix vgId:%d, physicalPlan:%s",
+         numOfPlanLevel, externalTargetDB, multiTarget, pStream->fixedSinkVgId, pStream->physicalPlan);
  pStream->tasks = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);
  pStream->pHTasksList = taosArrayInit(numOfPlanLevel + 1, POINTER_BYTES);

-  if (numOfPlanLevel == 2 || externalTargetDB || multiTarget || pStream->fixedSinkVgId) {
+  if (numOfPlanLevel > 1 || externalTargetDB || multiTarget || pStream->fixedSinkVgId) {
    // add extra sink
    hasExtraSink = true;
-
-    SArray* pSinkTaskList = NULL;
-    int32_t code = addSinkTasks(pStream->tasks, pMnode, pStream, &pSinkTaskList, pEpset, 0);
+    int32_t code = addSinkTask(pMnode, pStream, pEpset);
    if (code != TSDB_CODE_SUCCESS) {
      return code;
    }
-
-    // check for fill history
-    if (pStream->conf.fillHistory) {
-      SArray* pHSinkTaskList = NULL;
-      code = addSinkTasks(pStream->pHTasksList, pMnode, pStream, &pHSinkTaskList, pEpset, 1);
-      if (code != TSDB_CODE_SUCCESS) {
-        return code;
-      }
-
-      setHTasksId(pSinkTaskList, pHSinkTaskList);
-    }
  }

  pStream->totalLevel = numOfPlanLevel + hasExtraSink;

-  if (numOfPlanLevel > 1) {
-    SStreamTask* pAggTask = NULL;
-    SStreamTask* pHAggTask = NULL;
-
-    int32_t code = addAggTask(pStream, pMnode, pPlan, pEpset, &pAggTask, &pHAggTask);
-    if (code != TSDB_CODE_SUCCESS) {
-      return code;
-    }
-
-    setSinkTaskUpstreamInfo(pStream->tasks, pAggTask);
-    if (pHAggTask != NULL) {
-      setSinkTaskUpstreamInfo(pStream->pHTasksList, pHAggTask);
-    }
-
-    // source level
-    return addSourceTasksForMultiLevelStream(pMnode, pPlan, pStream, pAggTask, pHAggTask, pEpset, skey, pVerList);
-  } else if (numOfPlanLevel == 1) {
-    return addSourceTasksForOneLevelStream(pMnode, pPlan, pStream, pEpset, hasExtraSink, skey, pVerList);
+  SSubplan* plan = getScanSubPlan(pPlan);   // source plan
+  if (plan == NULL) {
+    return terrno;
+  }
+  int32_t code = addSourceTask(pMnode, plan, pStream, pEpset, skey, pVerList, numOfPlanLevel == 1);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
  }

-  return 0;
+  if (numOfPlanLevel == 1) {
+    bindSourceSink(pStream, pMnode, pStream->tasks, hasExtraSink);
+    if (pStream->conf.fillHistory) {
+      bindSourceSink(pStream, pMnode, pStream->pHTasksList, hasExtraSink);
+    }
+    return TDB_CODE_SUCCESS;
+  }
+
+  if(numOfPlanLevel == 3){
+    plan = getAggSubPlan(pPlan, 1);  // middle agg plan
+    if (plan == NULL) {
+      return terrno;
+    }
+    do{
+      SArray** list = taosArrayGetLast(pStream->tasks);
+      float size = (float)taosArrayGetSize(*list);
+      size_t cnt = (size_t)ceil(size/tsStreamAggCnt);
+      if(cnt <= 1) break;
+
+      mDebug("doScheduleStream add middle agg, size:%d, cnt:%d", (int)size, (int)cnt);
+      addNewTaskList(pStream);
+
+      for(int j = 0; j < cnt; j++){
+        code = addAggTask(pStream, pMnode, plan, pEpset, false);
+        if (code != TSDB_CODE_SUCCESS) {
+          return code;
+        }
+
+        bindTwoLevel(pStream->tasks, j*tsStreamAggCnt, (j+1)*tsStreamAggCnt);
+        if (pStream->conf.fillHistory) {
+          bindTwoLevel(pStream->pHTasksList, j*tsStreamAggCnt, (j+1)*tsStreamAggCnt);
+        }
+      }
+    }while(1);
+  }
+
+  plan = getAggSubPlan(pPlan, 0);
+  if (plan == NULL) {
+    return terrno;
+  }
+
+  mDebug("doScheduleStream add final agg");
+  SArray** list = taosArrayGetLast(pStream->tasks);
+  size_t size = taosArrayGetSize(*list);
+  addNewTaskList(pStream);
+  code = addAggTask(pStream, pMnode, plan, pEpset, true);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+  bindTwoLevel(pStream->tasks, 0, size);
+  if (pStream->conf.fillHistory) {
+    bindTwoLevel(pStream->pHTasksList, 0, size);
+  }
+
+  bindAggSink(pStream, pMnode, pStream->tasks);
+  if (pStream->conf.fillHistory) {
+    bindAggSink(pStream, pMnode, pStream->pHTasksList);
+  }
+  return TDB_CODE_SUCCESS;
 }

 int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t skey, SArray* pVgVerList) {
--- a/source/dnode/mnode/impl/src/mndSma.c
+++ b/source/dnode/mnode/impl/src/mndSma.c
@ -566,6 +566,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
  streamObj.conf.trigger = STREAM_TRIGGER_WINDOW_CLOSE;
  streamObj.conf.triggerParam = pCreate->maxDelay;
  streamObj.ast = taosStrdup(smaObj.ast);
+  streamObj.indexForMultiAggBalance = -1;

  // check the maxDelay
  if (streamObj.conf.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) {
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@ -297,6 +297,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
  pObj->updateTime = pObj->createTime;
  pObj->version = 1;
  pObj->smaId = 0;
+  pObj->indexForMultiAggBalance = -1;

  pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name));

--- a/source/dnode/mnode/impl/src/mndSubscribe.c
+++ b/source/dnode/mnode/impl/src/mndSubscribe.c
@ -24,7 +24,7 @@
 #include "tcompare.h"
 #include "tname.h"

-#define MND_SUBSCRIBE_VER_NUMBER   2
+#define MND_SUBSCRIBE_VER_NUMBER   3
 #define MND_SUBSCRIBE_RESERVE_SIZE 64

 #define MND_CONSUMER_LOST_HB_CNT          6
@ -530,51 +530,50 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
    }
  }

-//  if(taosHashGetSize(pOutput->pSub->consumerHash) == 0) {                            // if all consumer is removed
-    SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key);  // put all offset rows
-    if (pSub) {
-      taosRLockLatch(&pSub->lock);
-      if (pOutput->pSub->offsetRows == NULL) {
-        pOutput->pSub->offsetRows = taosArrayInit(4, sizeof(OffsetRows));
-      }
-      pIter = NULL;
-      while (1) {
-        pIter = taosHashIterate(pSub->consumerHash, pIter);
-        if (pIter == NULL) break;
-        SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
-        SMqConsumerEp *pConsumerEpNew = taosHashGet(pOutput->pSub->consumerHash, &pConsumerEp->consumerId, sizeof(int64_t));
+  SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key);  // put all offset rows
+  if (pSub) {
+    taosRLockLatch(&pSub->lock);
+    if (pOutput->pSub->offsetRows == NULL) {
+      pOutput->pSub->offsetRows = taosArrayInit(4, sizeof(OffsetRows));
+    }
+    pIter = NULL;
+    while (1) {
+      pIter = taosHashIterate(pSub->consumerHash, pIter);
+      if (pIter == NULL) break;
+      SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
+      SMqConsumerEp *pConsumerEpNew = taosHashGet(pOutput->pSub->consumerHash, &pConsumerEp->consumerId, sizeof(int64_t));

-        for (int j = 0; j < taosArrayGetSize(pConsumerEp->offsetRows); j++) {
-          OffsetRows *d1 = taosArrayGet(pConsumerEp->offsetRows, j);
-          bool jump = false;
-          for (int i = 0; pConsumerEpNew && i < taosArrayGetSize(pConsumerEpNew->vgs); i++){
-            SMqVgEp *pVgEp = taosArrayGetP(pConsumerEpNew->vgs, i);
-            if(pVgEp->vgId == d1->vgId){
-              jump = true;
-              mInfo("pSub->offsetRows jump, because consumer id:0x%"PRIx64 " and vgId:%d not change", pConsumerEp->consumerId, pVgEp->vgId);
-              break;
-            }
-          }
-          if(jump) continue;
-          bool find = false;
-          for (int i = 0; i < taosArrayGetSize(pOutput->pSub->offsetRows); i++) {
-            OffsetRows *d2 = taosArrayGet(pOutput->pSub->offsetRows, i);
-            if (d1->vgId == d2->vgId) {
-              d2->rows += d1->rows;
-              d2->offset = d1->offset;
-              find = true;
-              mInfo("pSub->offsetRows add vgId:%d, after:%"PRId64", before:%"PRId64, d2->vgId, d2->rows, d1->rows);
-              break;
-            }
-          }
-          if(!find){
-            taosArrayPush(pOutput->pSub->offsetRows, d1);
+      for (int j = 0; j < taosArrayGetSize(pConsumerEp->offsetRows); j++) {
+        OffsetRows *d1 = taosArrayGet(pConsumerEp->offsetRows, j);
+        bool jump = false;
+        for (int i = 0; pConsumerEpNew && i < taosArrayGetSize(pConsumerEpNew->vgs); i++){
+          SMqVgEp *pVgEp = taosArrayGetP(pConsumerEpNew->vgs, i);
+          if(pVgEp->vgId == d1->vgId){
+            jump = true;
+            mInfo("pSub->offsetRows jump, because consumer id:0x%"PRIx64 " and vgId:%d not change", pConsumerEp->consumerId, pVgEp->vgId);
+            break;
          }
        }
+        if(jump) continue;
+        bool find = false;
+        for (int i = 0; i < taosArrayGetSize(pOutput->pSub->offsetRows); i++) {
+          OffsetRows *d2 = taosArrayGet(pOutput->pSub->offsetRows, i);
+          if (d1->vgId == d2->vgId) {
+            d2->rows += d1->rows;
+            d2->offset = d1->offset;
+            d2->ever = d1->ever;
+            find = true;
+            mInfo("pSub->offsetRows add vgId:%d, after:%"PRId64", before:%"PRId64, d2->vgId, d2->rows, d1->rows);
+            break;
+          }
+        }
+        if(!find){
+          taosArrayPush(pOutput->pSub->offsetRows, d1);
+        }
      }
-      taosRUnLockLatch(&pSub->lock);
-      mndReleaseSubscribe(pMnode, pSub);
-//    }
+    }
+    taosRUnLockLatch(&pSub->lock);
+    mndReleaseSubscribe(pMnode, pSub);
  }

  // 8. generate logs
@ -1405,8 +1404,9 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t* numOfRows, int64_t cons
    }
    if(data){
      // vg id
-      char buf[TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0};
+      char buf[TSDB_OFFSET_LEN*2 + VARSTR_HEADER_SIZE] = {0};
      tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset);
+      sprintf(varDataVal(buf) + strlen(varDataVal(buf)), "/%"PRId64, data->ever);
      varDataSetLen(buf, strlen(varDataVal(buf)));
      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
      colDataSetVal(pColInfo, *numOfRows, (const char *)buf, false);
--- a/source/dnode/mnode/impl/test/stream/CMakeLists.txt
+++ b/source/dnode/mnode/impl/test/stream/CMakeLists.txt
@ -4,7 +4,7 @@ aux_source_directory(. MNODE_STREAM_TEST_SRC)
 add_executable(streamTest ${MNODE_STREAM_TEST_SRC})
 target_link_libraries(
        streamTest
-        PRIVATE dnode  gtest
+        PRIVATE dnode nodes planner gtest qcom
 )

 add_test(
--- a/source/dnode/mnode/impl/test/stream/stream.cpp
+++ b/source/dnode/mnode/impl/test/stream/stream.cpp
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@ -223,7 +223,7 @@ bool tqReaderIsQueriedTable(STqReader *pReader, uint64_t uid);
 bool tqCurrentBlockConsumed(const STqReader *pReader);

 int32_t      tqReaderSeek(STqReader *pReader, int64_t ver, const char *id);
-bool         tqNextBlockInWal(STqReader *pReader, const char *idstr);
+bool         tqNextBlockInWal(STqReader *pReader, const char *idstr, int sourceExcluded);
 bool         tqNextBlockImpl(STqReader *pReader, const char *idstr);
 SWalReader  *tqGetWalReader(STqReader *pReader);
 SSDataBlock *tqGetResultBlock(STqReader *pReader);
--- a/source/dnode/vnode/src/inc/tq.h
+++ b/source/dnode/vnode/src/inc/tq.h
@ -118,7 +118,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
 int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId);

 // tqExec
-int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows);
+int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded);
 int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
 int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp,
                      int32_t type, int32_t vgId);
--- a/source/dnode/vnode/src/meta/metaOpen.c
+++ b/source/dnode/vnode/src/meta/metaOpen.c
@ -27,7 +27,14 @@ static int taskIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int k
 static int btimeIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
 static int ncolIdxCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);

-static int32_t metaInitLock(SMeta *pMeta) { return taosThreadRwlockInit(&pMeta->lock, NULL); }
+static int32_t metaInitLock(SMeta *pMeta) {
+  TdThreadRwlockAttr attr;
+  taosThreadRwlockAttrInit(&attr);
+  taosThreadRwlockAttrSetKindNP(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
+  taosThreadRwlockInit(&pMeta->lock, &attr);
+  taosThreadRwlockAttrDestroy(&attr);
+  return 0;
+}
 static int32_t metaDestroyLock(SMeta *pMeta) { return taosThreadRwlockDestroy(&pMeta->lock); }

 static void metaCleanup(SMeta **ppMeta);
--- a/source/dnode/vnode/src/sma/smaTimeRange.c
+++ b/source/dnode/vnode/src/sma/smaTimeRange.c
@ -193,7 +193,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
      continue;
    }

-    SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE,};
+    SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE, .source = SOURCE_NULL};

    int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
    tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray, true);
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@ -368,7 +368,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
 }

 // todo ignore the error in wal?
-bool tqNextBlockInWal(STqReader* pReader, const char* id) {
+bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) {
  SWalReader* pWalReader = pReader->pWalReader;
  SSDataBlock* pDataBlock = NULL;

@ -391,7 +391,10 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id) {
          numOfBlocks, pReader->msg.msgLen, pReader->msg.ver);

      SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
-
+      if ((pSubmitTbData->source & sourceExcluded) != 0){
+        pReader->nextBlk += 1;
+        continue;
+      }
      if (pReader->tbIdHash == NULL || taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)) != NULL) {
        tqTrace("tq reader return submit block, uid:%" PRId64, pSubmitTbData->uid);
        SSDataBlock* pRes = NULL;
--- a/source/dnode/vnode/src/tq/tqScan.c
+++ b/source/dnode/vnode/src/tq/tqScan.c
@ -93,6 +93,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
    return -1;
  }

+  qStreamSetSourceExcluded(task, pRequest->sourceExcluded);
  while (1) {
    SSDataBlock* pDataBlock = NULL;
    code = getDataBlock(task, pHandle, vgId, &pDataBlock);
@ -249,7 +250,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
  return 0;
 }

-int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows) {
+int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded) {
  STqExecHandle* pExec = &pHandle->execHandle;
  SArray*        pBlocks = taosArrayInit(0, sizeof(SSDataBlock));
  SArray*        pSchemas = taosArrayInit(0, sizeof(void*));
@ -264,6 +265,10 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
      if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) {
        if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_table;
      }
+
+      if ((pSubmitTbDataRet->source & sourceExcluded) != 0){
+        goto loop_table;
+      }
      if (pRsp->withTbName) {
        int64_t uid = pExec->pTqReader->lastBlkUid;
        if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) {
@ -328,6 +333,10 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
      if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) {
        if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_db;
      }
+
+      if ((pSubmitTbDataRet->source & sourceExcluded) != 0){
+        goto loop_db;
+      }
      if (pRsp->withTbName) {
        int64_t uid = pExec->pTqReader->lastBlkUid;
        if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) {
--- a/source/dnode/vnode/src/tq/tqSink.c
+++ b/source/dnode/vnode/src/tq/tqSink.c
@ -815,7 +815,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
          return;
        }

-        SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version};
+        SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL};
        code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData);
        if (code != TSDB_CODE_SUCCESS) {
          continue;
@ -859,7 +859,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
      pTask->execInfo.sink.numOfBlocks += 1;
      uint64_t groupId = pDataBlock->info.id.groupId;

-      SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version};
+      SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .source = SOURCE_NULL};

      int32_t* index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId));
      if (index == NULL) {  // no data yet, append it
--- a/source/dnode/vnode/src/tq/tqUtil.c
+++ b/source/dnode/vnode/src/tq/tqUtil.c
@ -250,7 +250,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
          .ver = pHead->version,
      };

-      code = tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows);
+      code = tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest->sourceExcluded);
      if (code < 0) {
        tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId,
                pRequest->subKey);
--- a/source/dnode/vnode/src/tqCommon/tqCommon.c
+++ b/source/dnode/vnode/src/tqCommon/tqCommon.c
@ -317,15 +317,25 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
  if (pTask == NULL) {
    tqError("vgId:%d process retrieve req, failed to acquire task:0x%x, it may have been dropped already", pMeta->vgId,
            req.dstTaskId);
+    taosMemoryFree(req.pRetrieve);
    return -1;
  }

+  int32_t code = 0;
+  if(pTask->info.taskLevel == TASK_LEVEL__SOURCE){
+    code = streamProcessRetrieveReq(pTask, &req);
+  }else{
+    req.srcNodeId = pTask->info.nodeId;
+    req.srcTaskId = pTask->id.taskId;
+    code = broadcastRetrieveMsg(pTask, &req);
+  }
+
  SRpcMsg rsp = {.info = pMsg->info, .code = 0};
-  streamProcessRetrieveReq(pTask, &req, &rsp);
+  sendRetrieveRsp(&req, &rsp);

  streamMetaReleaseTask(pMeta, pTask);
-  tDeleteStreamRetrieveReq(&req);
-  return 0;
+  taosMemoryFree(req.pRetrieve);
+  return code;
 }

 int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
--- a/source/libs/executor/inc/executorInt.h
+++ b/source/libs/executor/inc/executorInt.h
@ -556,6 +556,11 @@ typedef struct SStreamIntervalOperatorInfo {
  bool                reCkBlock;
  SSDataBlock*        pCheckpointRes;
  struct SUpdateInfo* pUpdateInfo;
+  bool                recvRetrive;
+  SSDataBlock*        pMidRetriveRes;
+  bool                recvPullover;
+  SSDataBlock*        pMidPulloverRes;
+  bool                clearState;
 } SStreamIntervalOperatorInfo;

 typedef struct SDataGroupInfo {
--- a/source/libs/executor/inc/querytask.h
+++ b/source/libs/executor/inc/querytask.h
@ -59,6 +59,7 @@ typedef struct STaskStopInfo {
 typedef struct {
  STqOffsetVal         currentOffset;  // for tmq
  SMqMetaRsp           metaRsp;        // for tmq fetching meta
+  int8_t               sourceExcluded;
  int64_t              snapshotVer;
  SSchemaWrapper*      schema;
  char                 tbName[TSDB_TABLE_NAME_LEN];  // this is the current scan table: todo refactor
--- a/source/libs/executor/src/executil.c
+++ b/source/libs/executor/src/executil.c
@ -2229,6 +2229,8 @@ char* getStreamOpName(uint16_t opType) {
      return "interval final";
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
      return "interval semi";
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
+      return "interval mid";
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
      return "stream fill";
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION:
--- a/source/libs/executor/src/executor.c
+++ b/source/libs/executor/src/executor.c
@ -947,7 +947,7 @@ int32_t qSetStreamOperatorOptionForScanHistory(qTaskInfo_t tinfo) {
  while (1) {
    int32_t type = pOperator->operatorType;
    if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
-        type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
+        type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
      SStreamIntervalOperatorInfo* pInfo = pOperator->info;
      STimeWindowAggSupp* pSup = &pInfo->twAggSup;

@ -1035,7 +1035,7 @@ int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo) {
  while (1) {
    uint16_t type = pOperator->operatorType;
    if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
-        type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) {
+        type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
      SStreamIntervalOperatorInfo* pInfo = pOperator->info;
      pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved;
      pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved;
@ -1152,6 +1152,11 @@ void qStreamSetOpen(qTaskInfo_t tinfo) {
  pOperator->status = OP_NOT_OPENED;
 }

+void qStreamSetSourceExcluded(qTaskInfo_t tinfo, int8_t sourceExcluded) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  pTaskInfo->streamInfo.sourceExcluded = sourceExcluded;
+}
+
 int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) {
  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
--- a/source/libs/executor/src/operator.c
+++ b/source/libs/executor/src/operator.c
@ -489,6 +489,9 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR
  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL == type) {
    int32_t children = 0;
    pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children, pHandle);
+  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL == type) {
+    int32_t children = pHandle->numOfVgroups;
+    pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children, pHandle);
  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == type) {
    int32_t children = pHandle->numOfVgroups;
    pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children, pHandle);
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@ -1272,6 +1272,7 @@ static bool isStateWindow(SStreamScanInfo* pInfo) {
 static bool isIntervalWindow(SStreamScanInfo* pInfo) {
  return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL ||
         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
+         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL ||
         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
 }

@ -2010,7 +2011,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
  if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) {

    while (1) {
-      bool hasResult = pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id);
+      bool hasResult = pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id, pTaskInfo->streamInfo.sourceExcluded);

      SSDataBlock* pRes = pAPI->tqReaderFn.tqGetResultBlock(pInfo->tqReader);
      struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader);
--- a/source/libs/executor/src/streamtimewindowoperator.c
+++ b/source/libs/executor/src/streamtimewindowoperator.c
@ -28,6 +28,7 @@
 #include "ttime.h"

 #define IS_FINAL_INTERVAL_OP(op)           ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
+#define IS_MID_INTERVAL_OP(op)             ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL)
 #define IS_FINAL_SESSION_OP(op)            ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
 #define DEAULT_DELETE_MARK                 INT64_MAX
 #define STREAM_INTERVAL_OP_STATE_NAME      "StreamIntervalHistoryState"
@ -48,6 +49,8 @@ typedef struct SPullWindowInfo {
  STimeWindow calWin;
 } SPullWindowInfo;

+static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator);
+
 typedef int32_t (*__compare_fn_t)(void* pKey, void* data, int32_t index);

 static int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {
@ -235,7 +238,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa
    dumyInfo.cur.pageId = -1;

    STimeWindow win = {0};
-    if (IS_FINAL_INTERVAL_OP(pOperator)) {
+    if (IS_FINAL_INTERVAL_OP(pOperator) || IS_MID_INTERVAL_OP(pOperator)) {
      win.skey = startTsCols[i];
      win.ekey = endTsCols[i];
    } else {
@ -407,6 +410,8 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) {
  blockDataDestroy(pInfo->pPullDataRes);
  taosArrayDestroy(pInfo->pDelWins);
  blockDataDestroy(pInfo->pDelRes);
+  blockDataDestroy(pInfo->pMidRetriveRes);
+  blockDataDestroy(pInfo->pMidPulloverRes);
  pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState);

  if (pInfo->pState->dump == 1) {
@ -599,7 +604,7 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB
  blockDataUpdateTsWindow(pBlock, 0);
 }

-void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins,
+static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins,
                     int32_t numOfCh, SOperatorInfo* pOperator) {
  SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
  TSKEY*           tsData = (TSKEY*)pStartCol->pData;
@ -608,6 +613,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, S
  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
  uint64_t*        groupIdData = (uint64_t*)pGroupCol->pData;
  int32_t          chId = getChildIndex(pBlock);
+  bool             res = false;
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
    TSKEY winTs = tsData[i];
    while (winTs <= tsEndData[i]) {
@ -623,6 +629,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, S
            // pull data is over
            taosArrayDestroy(chArray);
            taosHashRemove(pMap, &winRes, sizeof(SWinKey));
+            res =true;
            qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts);

            void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey));
@ -646,6 +653,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, S
      winTs = taosTimeAdd(winTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision);
    }
  }
+  return res;
 }

 static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, int32_t childId) {
@ -1182,6 +1190,12 @@ static SSDataBlock* buildIntervalResult(SOperatorInfo* pOperator) {
    printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));
    return pInfo->binfo.pRes;
  }
+
+  if (pInfo->recvPullover) {
+    pInfo->recvPullover = false;
+    printDataBlock(pInfo->pMidPulloverRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+    return pInfo->pMidPulloverRes;
+  }
  return NULL;
 }

@ -1236,11 +1250,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
    return NULL;
  } else {
    if (!IS_FINAL_INTERVAL_OP(pOperator)) {
-      doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
-      if (pInfo->pDelRes->info.rows != 0) {
-        // process the rest of the data
-        printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
-        return pInfo->pDelRes;
+      SSDataBlock* resBlock = buildIntervalResult(pOperator);
+      if (resBlock != NULL) {
+        return resBlock;
+      }
+
+      if (pInfo->recvRetrive) {
+        pInfo->recvRetrive = false;
+        printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+        return pInfo->pMidRetriveRes;
      }
    }
  }
@ -1314,9 +1332,12 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
      pInfo->recvGetAll = true;
      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);
      continue;
-    } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_INTERVAL_OP(pOperator)) {
-      doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap);
-      if (taosArrayGetSize(pInfo->pUpdated) > 0) {
+    } else if (pBlock->info.type == STREAM_RETRIEVE) {
+      if(!IS_FINAL_INTERVAL_OP(pOperator)) {
+        pInfo->recvRetrive = true;
+        copyDataBlock(pInfo->pMidRetriveRes, pBlock);
+        pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;
+        doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap);
        break;
      }
      continue;
@ -1331,6 +1352,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
      doStreamIntervalSaveCheckpoint(pOperator);
      copyDataBlock(pInfo->pCheckpointRes, pBlock);
      continue;
+    } else if (IS_FINAL_INTERVAL_OP(pOperator) && pBlock->info.type == STREAM_MID_RETRIEVE) {
+      continue;
    } else {
      ASSERTS(pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
    }
@ -1359,7 +1382,18 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
  pInfo->pUpdated = NULL;
  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

-  return buildIntervalResult(pOperator);
+  SSDataBlock* resBlock = buildIntervalResult(pOperator);
+  if (resBlock != NULL) {
+    return resBlock;
+  }
+
+  if (pInfo->recvRetrive) {
+    pInfo->recvRetrive = false;
+    printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+    return pInfo->pMidRetriveRes;
+  }
+
+  return NULL;
 }

 int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval) {
@ -1400,7 +1434,8 @@ static int32_t getMaxFunResSize(SExprSupp* pSup, int32_t numOfCols) {
 }

 static void streamIntervalReleaseState(SOperatorInfo* pOperator) {
-  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
+  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&
+      pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
    SStreamIntervalOperatorInfo* pInfo = pOperator->info;
    int32_t                      resSize = sizeof(TSKEY);
    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,
@ -1417,7 +1452,8 @@ static void streamIntervalReleaseState(SOperatorInfo* pOperator) {

 void streamIntervalReloadState(SOperatorInfo* pOperator) {
  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
-  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
+  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&
+      pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
    int32_t                      size = 0;
    void*                        pBuf = NULL;
    int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,
@ -1442,6 +1478,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
  SIntervalPhysiNode*          pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  int32_t                      code = 0;
  if (pInfo == NULL || pOperator == NULL) {
    goto _error;
  }
@ -1471,7 +1508,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
  if (pIntervalPhyNode->window.pExprs != NULL) {
    int32_t    numOfScalar = 0;
    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
-    int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
+    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
    if (code != TSDB_CODE_SUCCESS) {
      goto _error;
    }
@ -1490,7 +1527,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
  qInfo("copy state %p to %p", pTaskInfo->streamInfo.pState, pInfo->pState);

  pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1);
-  int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
+  code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
                            pInfo->pState, &pTaskInfo->storageAPI.functionStore);
  if (code != TSDB_CODE_SUCCESS) {
    goto _error;
@ -1526,6 +1563,10 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
  pInfo->stateStore = pTaskInfo->storageAPI.stateStore;
  pInfo->recvGetAll = false;
  pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
+  pInfo->recvRetrive = false;
+  pInfo->pMidRetriveRes = createSpecialDataBlock(STREAM_MID_RETRIEVE);
+  pInfo->pMidPulloverRes = createSpecialDataBlock(STREAM_MID_RETRIEVE);
+  pInfo->clearState = false;

  pOperator->operatorType = pPhyNode->type;
  if (!IS_FINAL_INTERVAL_OP(pOperator) || numOfChild == 0) {
@ -1536,10 +1577,16 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
  pOperator->status = OP_NOT_OPENED;
  pOperator->info = pInfo;

-  pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo,
-                                         optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
+  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
+    pOperator->fpSet = createOperatorFpSet(NULL, doStreamMidIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo,
+                                           optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
+  } else {
+    pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo,
+                                           optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
+  }
  setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);
-  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
+  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
+      pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
    initIntervalDownStream(downstream, pPhyNode->type, pInfo);
  }
  code = appendDownstream(pOperator, &downstream, 1);
@ -4114,6 +4161,285 @@ _error:
  return NULL;
 }

+static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pUpdatedMap) {
+  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;
+  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
+
+  SResultRowInfo*  pResultRowInfo = &(pInfo->binfo.resultRowInfo);
+  SExecTaskInfo*   pTaskInfo = pOperator->pTaskInfo;
+  SExprSupp*       pSup = &pOperator->exprSupp;
+  int32_t          numOfOutput = pSup->numOfExprs;
+  int32_t          step = 1;
+  SRowBuffPos*     pResPos = NULL;
+  SResultRow*      pResult = NULL;
+  int32_t          forwardRows = 1;
+  uint64_t         groupId = pSDataBlock->info.id.groupId;
+  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
+  TSKEY*           tsCol = (int64_t*)pColDataInfo->pData;
+
+  int32_t     startPos = 0;
+  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCol);
+  STimeWindow nextWin = getFinalTimeWindow(ts, &pInfo->interval);
+
+  while (1) {
+    SWinKey key = {
+        .ts = nextWin.skey,
+        .groupId = groupId,
+    };
+    void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey));
+    int32_t index = -1;
+    SArray* chArray = NULL;
+    int32_t chId = 0;
+    if (chIds) {
+      chArray = *(void**)chIds;
+      chId = getChildIndex(pSDataBlock);
+      index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
+    }
+    if (!(index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA)) {
+      startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCol, startPos);
+      if (startPos < 0) {
+        break;
+      }
+      continue;
+    }
+
+    if (!inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
+      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCol, startPos, nextWin.ekey, &nextWin);
+      if (startPos < 0) {
+        break;
+      }
+      continue;
+    }
+
+    int32_t code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,
+                                        pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore);
+    pResult = (SResultRow*)pResPos->pRowBuff;
+    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
+      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY);
+    }
+
+    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
+      saveWinResult(&key, pResPos, pUpdatedMap);
+    }
+
+    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
+      tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES);
+    }
+
+    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1);
+    applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
+                                    pSDataBlock->info.rows, numOfOutput);
+    key.ts = nextWin.skey;
+
+    if (pInfo->delKey.ts > key.ts) {
+      pInfo->delKey = key;
+    }
+    int32_t prevEndPos = (forwardRows - 1) * step + startPos;
+    if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {
+      qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64
+             ",maxKey %" PRId64,
+             pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
+      blockDataUpdateTsWindow(pSDataBlock, 0);
+
+      // timestamp of the data is incorrect
+      if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {
+        qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,
+               pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
+      }
+    }
+    startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCol, prevEndPos);
+    if (startPos < 0) {
+      break;
+    }
+  }
+}
+
+static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) {
+  int32_t size = taosArrayGetSize(wins);
+  for (int32_t i = 0; i < size; i++) {
+    SWinKey*    winKey = taosArrayGet(wins, i);
+    void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey));
+    if (!chIds) {
+      addPullWindow(pMidPullMap, winKey, numOfChild);
+      qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild);
+    }
+  }
+}
+
+static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {
+  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
+  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
+  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;
+  SOperatorInfo*               downstream = pOperator->pDownstream[0];
+  SExprSupp*                   pSup = &pOperator->exprSupp;
+
+  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);
+
+  if (pOperator->status == OP_EXEC_DONE) {
+    return NULL;
+  } else if (pOperator->status == OP_RES_TO_RETURN) {
+    SSDataBlock* resBlock = buildIntervalResult(pOperator);
+    if (resBlock != NULL) {
+      return resBlock;
+    }
+
+    setOperatorCompleted(pOperator);
+    clearFunctionContext(&pOperator->exprSupp);
+    clearStreamIntervalOperator(pInfo);
+    qDebug("stask:%s  ===stream===%s clear", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType));
+    return NULL;
+  } else {
+    SSDataBlock* resBlock = buildIntervalResult(pOperator);
+    if (resBlock != NULL) {
+      return resBlock;
+    }
+
+    if (pInfo->recvRetrive) {
+      pInfo->recvRetrive = false;
+      printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+      return pInfo->pMidRetriveRes;
+    }
+
+    if (pInfo->clearState) {
+      pInfo->clearState = false;
+      clearFunctionContext(&pOperator->exprSupp);
+      clearStreamIntervalOperator(pInfo);
+    }
+  }
+
+  if (!pInfo->pUpdated) {
+    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);
+  }
+  if (!pInfo->pUpdatedMap) {
+    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
+    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);
+  }
+
+  while (1) {
+    if (isTaskKilled(pTaskInfo)) {
+      if (pInfo->pUpdated != NULL) {
+        pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated);
+      }
+
+      if (pInfo->pUpdatedMap != NULL) {
+        tSimpleHashCleanup(pInfo->pUpdatedMap);
+        pInfo->pUpdatedMap = NULL;
+      }
+
+      T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
+    }
+
+    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
+    if (pBlock == NULL) {
+      pOperator->status = OP_RES_TO_RETURN;
+      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),
+             pInfo->numOfDatapack);
+      pInfo->numOfDatapack = 0;
+      break;
+    }
+    pInfo->numOfDatapack++;
+    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
+
+    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
+      pInfo->binfo.pRes->info.type = pBlock->info.type;
+    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
+               pBlock->info.type == STREAM_CLEAR) {
+      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
+      doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap);
+      removeResults(delWins, pInfo->pUpdatedMap);
+      taosArrayAddAll(pInfo->pDelWins, delWins);
+      taosArrayDestroy(delWins);
+
+      doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
+      if (pInfo->pDelRes->info.rows != 0) {
+        // process the rest of the data
+        printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+        if (pBlock->info.type == STREAM_CLEAR) {
+          pInfo->pDelRes->info.type = STREAM_CLEAR;
+        } else {
+          pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
+        }
+        ASSERT(taosArrayGetSize(pInfo->pUpdated) == 0);
+        return pInfo->pDelRes;
+      }
+      continue;
+    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
+      return pBlock;
+    } else if (pBlock->info.type == STREAM_PULL_OVER) {
+      pInfo->recvPullover = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval,
+                                            pInfo->pPullWins, pInfo->numOfChild, pOperator);
+      if (pInfo->recvPullover) {
+        copyDataBlock(pInfo->pMidPulloverRes, pBlock);
+        pInfo->clearState = true;
+        break;
+      }
+      continue;
+    } else if (pBlock->info.type == STREAM_CHECKPOINT) {
+      pAPI->stateStore.streamStateCommit(pInfo->pState);
+      doStreamIntervalSaveCheckpoint(pOperator);
+      copyDataBlock(pInfo->pCheckpointRes, pBlock);
+      continue;
+    } else if (pBlock->info.type == STREAM_MID_RETRIEVE) {
+      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
+      doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap);
+      addMidRetriveWindow(delWins, pInfo->pPullDataMap, pInfo->numOfChild);
+      taosArrayDestroy(delWins);
+      pInfo->recvRetrive = true;
+      copyDataBlock(pInfo->pMidRetriveRes, pBlock);
+      pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;
+      pInfo->clearState = true;
+      break;
+    } else {
+      ASSERTS(pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type");
+    }
+
+    if (pInfo->scalarSupp.pExprInfo != NULL) {
+      SExprSupp* pExprSup = &pInfo->scalarSupp;
+      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
+    }
+    setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
+    doStreamMidIntervalAggImpl(pOperator, pBlock, pInfo->pUpdatedMap);
+    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
+    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);
+    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);
+  }
+
+  removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);
+  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;
+
+  void*   pIte = NULL;
+  int32_t iter = 0;
+  while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {
+    taosArrayPush(pInfo->pUpdated, pIte);
+  }
+
+  tSimpleHashCleanup(pInfo->pUpdatedMap);
+  pInfo->pUpdatedMap = NULL;
+  taosArraySort(pInfo->pUpdated, winPosCmprImpl);
+
+  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);
+  pInfo->pUpdated = NULL;
+  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+
+  SSDataBlock* resBlock = buildIntervalResult(pOperator);
+  if (resBlock != NULL) {
+    return resBlock;
+  }
+
+  if (pInfo->recvRetrive) {
+    pInfo->recvRetrive = false;
+    printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+    return pInfo->pMidRetriveRes;
+  }
+
+  if (pInfo->clearState) {
+    pInfo->clearState = false;
+    clearFunctionContext(&pOperator->exprSupp);
+    clearStreamIntervalOperator(pInfo);
+  }
+  return NULL;
+}
+
 void setStreamOperatorCompleted(SOperatorInfo* pOperator) {
  setOperatorCompleted(pOperator);
  qDebug("stask:%s  %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);
--- a/source/libs/function/inc/builtins.h
+++ b/source/libs/function/inc/builtins.h
@ -45,6 +45,7 @@ typedef struct SBuiltinFuncDefinition {
 #endif
  FExecCombine               combineFunc;
  const char*                pPartialFunc;
+  const char*                pMiddleFunc;
  const char*                pMergeFunc;
  FCreateMergeFuncParameters createMergeParaFuc;
  FEstimateReturnRows        estimateReturnRowsFunc;
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@ -433,6 +433,20 @@ static int32_t translateAvgPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t
  return TSDB_CODE_SUCCESS;
 }

+static int32_t translateAvgMiddle(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  if (1 != LIST_LENGTH(pFunc->pParameterList)) {
+    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  if (TSDB_DATA_TYPE_BINARY != paraType) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  }
+
+  pFunc->node.resType = (SDataType){.bytes = getAvgInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateAvgMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
  if (1 != LIST_LENGTH(pFunc->pParameterList)) {
    return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@ -2515,6 +2529,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
 #endif
    .combineFunc  = avgCombine,
    .pPartialFunc = "_avg_partial",
+    .pMiddleFunc  = "_avg_middle",
    .pMergeFunc   = "_avg_merge"
  },
  {
@ -3775,6 +3790,21 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
    .sprocessFunc = toCharFunction,
    .finalizeFunc = NULL
  },
+  {
+    .name = "_avg_middle",
+    .type = FUNCTION_TYPE_AVG_PARTIAL,
+    .classification = FUNC_MGT_AGG_FUNC,
+    .translateFunc = translateAvgMiddle,
+    .dataRequiredFunc = statisDataRequired,
+    .getEnvFunc   = getAvgFuncEnv,
+    .initFunc     = avgFunctionSetup,
+    .processFunc  = avgFunctionMerge,
+    .finalizeFunc = avgPartialFinalize,
+#ifdef BUILD_NO_CALL
+    .invertFunc   = avgInvertFunction,
+#endif
+    .combineFunc  = avgCombine,
+  },
  {
    .name = "_vgver",
    .type = FUNCTION_TYPE_VGVER,
@ -3785,7 +3815,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
    .sprocessFunc = qPseudoTagFunction,
    .finalizeFunc = NULL
  }
-  
 };
 // clang-format on

--- a/source/libs/function/src/functionMgt.c
+++ b/source/libs/function/src/functionMgt.c
@ -424,6 +424,35 @@ static int32_t createMergeFuncPara(const SFunctionNode* pSrcFunc, const SFunctio
  }
 }

+static int32_t createMidFunction(const SFunctionNode* pSrcFunc, const SFunctionNode* pPartialFunc,
+                                   SFunctionNode** pMidFunc) {
+  SNodeList*     pParameterList = NULL;
+  SFunctionNode* pFunc = NULL;
+
+  int32_t code = createMergeFuncPara(pSrcFunc, pPartialFunc, &pParameterList);
+  if (TSDB_CODE_SUCCESS == code) {
+    if(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc != NULL){
+      pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMiddleFunc, pParameterList);
+    }else{
+      pFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList);
+    }
+    if (NULL == pFunc) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    strcpy(pFunc->node.aliasName, pPartialFunc->node.aliasName);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    *pMidFunc = pFunc;
+  } else {
+    nodesDestroyList(pParameterList);
+  }
+
+  return code;
+}
+
 static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctionNode* pPartialFunc,
                                   SFunctionNode** pMergeFunc) {
  SNodeList*     pParameterList = NULL;
@ -453,18 +482,22 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio
  return code;
 }

-int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc) {
+int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMidFunc, SFunctionNode** pMergeFunc) {
  if (!fmIsDistExecFunc(pFunc->funcId)) {
    return TSDB_CODE_FAILED;
  }

  int32_t code = createPartialFunction(pFunc, pPartialFunc);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createMidFunction(pFunc, *pPartialFunc, pMidFunc);
+  }
  if (TSDB_CODE_SUCCESS == code) {
    code = createMergeFunction(pFunc, *pPartialFunc, pMergeFunc);
  }

  if (TSDB_CODE_SUCCESS != code) {
    nodesDestroyNode((SNode*)*pPartialFunc);
+    nodesDestroyNode((SNode*)*pMidFunc);
    nodesDestroyNode((SNode*)*pMergeFunc);
  }

--- a/source/libs/nodes/src/nodesCloneFuncs.c
+++ b/source/libs/nodes/src/nodesCloneFuncs.c
@ -959,6 +959,7 @@ SNode* nodesCloneNode(const SNode* pNode) {
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
      code = physiIntervalCopy((const SIntervalPhysiNode*)pNode, (SIntervalPhysiNode*)pDst);
      break;
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION:
--- a/source/libs/nodes/src/nodesCodeFuncs.c
+++ b/source/libs/nodes/src/nodesCodeFuncs.c
@ -369,6 +369,8 @@ const char* nodesNodeName(ENodeType type) {
      return "PhysiStreamFinalInterval";
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
      return "PhysiStreamSemiInterval";
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
+      return "PhysiStreamMidInterval";
    case QUERY_NODE_PHYSICAL_PLAN_FILL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
      return "PhysiFill";
@ -7212,6 +7214,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
      return physiIntervalNodeToJson(pObj, pJson);
    case QUERY_NODE_PHYSICAL_PLAN_FILL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
@ -7551,6 +7554,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
      return jsonToPhysiIntervalNode(pJson, pObj);
    case QUERY_NODE_PHYSICAL_PLAN_FILL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
--- a/source/libs/nodes/src/nodesMsgFuncs.c
+++ b/source/libs/nodes/src/nodesMsgFuncs.c
@ -4179,6 +4179,7 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
      code = physiIntervalNodeToMsg(pObj, pEncoder);
      break;
    case QUERY_NODE_PHYSICAL_PLAN_FILL:
@ -4333,6 +4334,7 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
      code = msgToPhysiIntervalNode(pDecoder, pObj);
      break;
    case QUERY_NODE_PHYSICAL_PLAN_FILL:
--- a/source/libs/nodes/src/nodesUtilFuncs.c
+++ b/source/libs/nodes/src/nodesUtilFuncs.c
@ -564,6 +564,8 @@ SNode* nodesMakeNode(ENodeType type) {
      return makeNode(type, sizeof(SStreamFinalIntervalPhysiNode));
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
      return makeNode(type, sizeof(SStreamSemiIntervalPhysiNode));
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
+      return makeNode(type, sizeof(SStreamMidIntervalPhysiNode));
    case QUERY_NODE_PHYSICAL_PLAN_FILL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
      return makeNode(type, sizeof(SFillPhysiNode));
@ -1391,6 +1393,7 @@ void nodesDestroyNode(SNode* pNode) {
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
      destroyWinodwPhysiNode((SWindowPhysiNode*)pNode);
      break;
    case QUERY_NODE_PHYSICAL_PLAN_FILL:
--- a/source/libs/parser/src/parInsertUtil.c
+++ b/source/libs/parser/src/parInsertUtil.c
@ -211,6 +211,7 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat
                                  bool colMode, bool ignoreColVals) {
  STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt));
  if (NULL == pTableCxt) {
+    *pOutput = NULL;
    return TSDB_CODE_OUT_OF_MEMORY;
  }

@ -268,12 +269,8 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat
    }
  }

-  if (TSDB_CODE_SUCCESS == code) {
-    *pOutput = pTableCxt;
-    qDebug("tableDataCxt created, uid:%" PRId64 ", vgId:%d", pTableMeta->uid, pTableMeta->vgId);
-  } else {
-    taosMemoryFree(pTableCxt);
-  }
+  *pOutput = pTableCxt;
+  qDebug("tableDataCxt created, code:%d, uid:%" PRId64 ", vgId:%d", code, pTableMeta->uid, pTableMeta->vgId);

  return code;
 }
@ -288,6 +285,7 @@ static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) {
    pTmp->suid = pSrc->suid;
    pTmp->uid = pSrc->uid;
    pTmp->sver = pSrc->sver;
+    pTmp->source = pSrc->source;
    pTmp->pCreateTbReq = NULL;
    if (pTmp->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) {
      if (pSrc->pCreateTbReq) {
@ -344,6 +342,10 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta*
    void* pData = *pTableCxt;  // deal scan coverity
    code = taosHashPut(pHash, id, idLen, &pData, POINTER_BYTES);
  }
+
+  if (TSDB_CODE_SUCCESS != code) {
+    insDestroyTableDataCxt(*pTableCxt);
+  }
  return code;
 }

@ -651,6 +653,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
    goto end;
  }

+  pTableCxt->pData->source = SOURCE_TAOSX;
  if(tmp == NULL){
    ret = initTableColSubmitData(pTableCxt);
    if (ret != TSDB_CODE_SUCCESS) {
--- a/source/libs/planner/src/planPhysiCreater.c
+++ b/source/libs/planner/src/planPhysiCreater.c
@ -1617,6 +1617,8 @@ static ENodeType getIntervalOperatorType(EWindowAlgorithm windowAlgo) {
      return QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
    case INTERVAL_ALGO_STREAM_SEMI:
      return QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL;
+    case INTERVAL_ALGO_STREAM_MID:
+      return QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL;
    case INTERVAL_ALGO_STREAM_SINGLE:
      return QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
    case SESSION_ALGO_STREAM_FINAL:
--- a/source/libs/planner/src/planSpliter.c
+++ b/source/libs/planner/src/planSpliter.c
@ -344,11 +344,12 @@ static bool stbSplFindSplitNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SL
  return false;
 }

-static int32_t stbSplRewriteFuns(const SNodeList* pFuncs, SNodeList** pPartialFuncs, SNodeList** pMergeFuncs) {
+static int32_t stbSplRewriteFuns(const SNodeList* pFuncs, SNodeList** pPartialFuncs, SNodeList** pMidFuncs, SNodeList** pMergeFuncs) {
  SNode* pNode = NULL;
  FOREACH(pNode, pFuncs) {
    SFunctionNode* pFunc = (SFunctionNode*)pNode;
    SFunctionNode* pPartFunc = NULL;
+    SFunctionNode* pMidFunc = NULL;
    SFunctionNode* pMergeFunc = NULL;
    int32_t        code = TSDB_CODE_SUCCESS;
    if (fmIsWindowPseudoColumnFunc(pFunc->funcId)) {
@ -359,18 +360,33 @@ static int32_t stbSplRewriteFuns(const SNodeList* pFuncs, SNodeList** pPartialFu
        nodesDestroyNode((SNode*)pMergeFunc);
        code = TSDB_CODE_OUT_OF_MEMORY;
      }
+      if(pMidFuncs != NULL){
+        pMidFunc = (SFunctionNode*)nodesCloneNode(pNode);
+        if (NULL == pMidFunc) {
+          nodesDestroyNode((SNode*)pMidFunc);
+          code = TSDB_CODE_OUT_OF_MEMORY;
+        }
+      }
    } else {
-      code = fmGetDistMethod(pFunc, &pPartFunc, &pMergeFunc);
+      code = fmGetDistMethod(pFunc, &pPartFunc, &pMidFunc, &pMergeFunc);
    }
    if (TSDB_CODE_SUCCESS == code) {
      code = nodesListMakeStrictAppend(pPartialFuncs, (SNode*)pPartFunc);
    }
+    if (TSDB_CODE_SUCCESS == code) {
+      if(pMidFuncs != NULL){
+        code = nodesListMakeStrictAppend(pMidFuncs, (SNode*)pMidFunc);
+      }else{
+        nodesDestroyNode((SNode*)pMidFunc);
+      }
+    }
    if (TSDB_CODE_SUCCESS == code) {
      code = nodesListMakeStrictAppend(pMergeFuncs, (SNode*)pMergeFunc);
    }
    if (TSDB_CODE_SUCCESS != code) {
-      nodesDestroyList(*pPartialFuncs);
-      nodesDestroyList(*pMergeFuncs);
+      nodesDestroyNode((SNode*)pPartFunc);
+      nodesDestroyNode((SNode*)pMidFunc);
+      nodesDestroyNode((SNode*)pMergeFunc);
      return code;
    }
  }
@ -463,7 +479,7 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic
  splSetParent((SLogicNode*)pPartWin);

  int32_t index = 0;
-  int32_t code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs);
+  int32_t code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, NULL, &pMergeWindow->pFuncs);
  if (TSDB_CODE_SUCCESS == code) {
    code = stbSplAppendWStart(pPartWin->pFuncs, &index, ((SColumnNode*)pMergeWindow->pTspk)->node.resType.precision);
  }
@ -488,6 +504,85 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic
  return code;
 }

+static int32_t stbSplCreatePartMidWindowNode(SWindowLogicNode* pMergeWindow, SLogicNode** pPartWindow, SLogicNode** pMidWindow) {
+  SNodeList* pFunc = pMergeWindow->pFuncs;
+  pMergeWindow->pFuncs = NULL;
+  SNodeList* pTargets = pMergeWindow->node.pTargets;
+  pMergeWindow->node.pTargets = NULL;
+  SNodeList* pChildren = pMergeWindow->node.pChildren;
+  pMergeWindow->node.pChildren = NULL;
+  SNode* pConditions = pMergeWindow->node.pConditions;
+  pMergeWindow->node.pConditions = NULL;
+
+  SWindowLogicNode* pPartWin = (SWindowLogicNode*)nodesCloneNode((SNode*)pMergeWindow);
+  if (NULL == pPartWin) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  SWindowLogicNode* pMidWin = (SWindowLogicNode*)nodesCloneNode((SNode*)pMergeWindow);
+  if (NULL == pMidWin) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  pPartWin->node.groupAction = GROUP_ACTION_KEEP;
+  pMidWin->node.groupAction = GROUP_ACTION_KEEP;
+  pMergeWindow->node.pTargets = pTargets;
+  pMergeWindow->node.pConditions = pConditions;
+
+  pPartWin->node.pChildren = pChildren;
+  splSetParent((SLogicNode*)pPartWin);
+
+  SNodeList* pFuncPart = NULL;
+  SNodeList* pFuncMid = NULL;
+  SNodeList* pFuncMerge = NULL;
+  int32_t code = stbSplRewriteFuns(pFunc, &pFuncPart, &pFuncMid, &pFuncMerge);
+  pPartWin->pFuncs = pFuncPart;
+  pMidWin->pFuncs = pFuncMid;
+  pMergeWindow->pFuncs = pFuncMerge;
+
+  int32_t index = 0;
+  if (TSDB_CODE_SUCCESS == code) {
+    code = stbSplAppendWStart(pPartWin->pFuncs, &index, ((SColumnNode*)pMergeWindow->pTspk)->node.resType.precision);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createColumnByRewriteExprs(pPartWin->pFuncs, &pPartWin->node.pTargets);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    nodesDestroyNode(pMidWin->pTspk);
+    pMidWin->pTspk = nodesCloneNode(nodesListGetNode(pPartWin->node.pTargets, index));
+    if (NULL == pMidWin->pTspk) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    code = stbSplAppendWStart(pMidWin->pFuncs, &index, ((SColumnNode*)pMergeWindow->pTspk)->node.resType.precision);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createColumnByRewriteExprs(pMidWin->pFuncs, &pMidWin->node.pTargets);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    nodesDestroyNode(pMergeWindow->pTspk);
+    pMergeWindow->pTspk = nodesCloneNode(nodesListGetNode(pMidWin->node.pTargets, index));
+    if (NULL == pMergeWindow->pTspk) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+
+  nodesDestroyList(pFunc);
+  if (TSDB_CODE_SUCCESS == code) {
+    *pPartWindow = (SLogicNode*)pPartWin;
+    *pMidWindow = (SLogicNode*)pMidWin;
+  } else {
+    nodesDestroyNode((SNode*)pPartWin);
+    nodesDestroyNode((SNode*)pMidWin);
+  }
+
+  return code;
+}
+
 static int32_t stbSplGetNumOfVgroups(SLogicNode* pNode) {
  if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
    return ((SScanLogicNode*)pNode)->pVgroupList->numOfVgroups;
@ -635,18 +730,31 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo
  return code;
 }

-static int32_t stbSplSplitIntervalForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
+static int32_t stbSplSplitIntervalForStreamMultiAgg(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
  SLogicNode* pPartWindow = NULL;
-  int32_t     code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow);
+  SLogicNode* pMidWindow  = NULL;
+  int32_t     code = stbSplCreatePartMidWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow, &pMidWindow);
  if (TSDB_CODE_SUCCESS == code) {
-    ((SWindowLogicNode*)pPartWindow)->windowAlgo = INTERVAL_ALGO_STREAM_SEMI;
+    ((SWindowLogicNode*)pMidWindow)->windowAlgo = INTERVAL_ALGO_STREAM_MID;
    ((SWindowLogicNode*)pInfo->pSplitNode)->windowAlgo = INTERVAL_ALGO_STREAM_FINAL;
-    code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow);
+    ((SWindowLogicNode*)pPartWindow)->windowAlgo = INTERVAL_ALGO_STREAM_SEMI;
+    code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pMidWindow);
+    if (TSDB_CODE_SUCCESS == code) {
+      code = stbSplCreateExchangeNode(pCxt, pMidWindow, pPartWindow);
+    }
  }
+
  if (TSDB_CODE_SUCCESS == code) {
-    code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
+    SNode* subPlan = (SNode*)splCreateSubplan(pCxt, pMidWindow);
+    ((SLogicSubplan*)subPlan)->subplanType = SUBPLAN_TYPE_MERGE;
+
+    code = nodesListMakeStrictAppend(&((SLogicSubplan*)subPlan)->pChildren,
                                     (SNode*)splCreateScanSubplan(pCxt, pPartWindow, SPLIT_FLAG_STABLE_SPLIT));
+    if (TSDB_CODE_SUCCESS == code) {
+      code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, subPlan);
+    }
  }
+
  pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
  ++(pCxt->groupId);
  return code;
@ -654,7 +762,7 @@ static int32_t stbSplSplitIntervalForStream(SSplitContext* pCxt, SStableSplitInf

 static int32_t stbSplSplitInterval(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
  if (pCxt->pPlanCxt->streamQuery) {
-    return stbSplSplitIntervalForStream(pCxt, pInfo);
+    return stbSplSplitIntervalForStreamMultiAgg(pCxt, pInfo);
  } else {
    return stbSplSplitIntervalForBatch(pCxt, pInfo);
  }
@ -860,7 +968,7 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO
    pPartAgg->node.pChildren = pChildren;
    splSetParent((SLogicNode*)pPartAgg);

-    code = stbSplRewriteFuns(pFunc, &pPartAgg->pAggFuncs, &pMergeAgg->pAggFuncs);
+    code = stbSplRewriteFuns(pFunc, &pPartAgg->pAggFuncs, NULL, &pMergeAgg->pAggFuncs);
  }
  if (TSDB_CODE_SUCCESS == code) {
    code = createColumnByRewriteExprs(pPartAgg->pAggFuncs, &pPartAgg->node.pTargets);
--- a/source/libs/planner/src/planValidator.c
+++ b/source/libs/planner/src/planValidator.c
@ -95,6 +95,7 @@ int32_t doValidatePhysiNode(SValidatePlanContext* pCxt, SNode* pNode) {
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL:
    case QUERY_NODE_PHYSICAL_PLAN_FILL:
    case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL:
    case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION:
--- a/source/libs/stream/inc/streamInt.h
+++ b/source/libs/stream/inc/streamInt.h
@ -107,7 +107,7 @@ SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamT
 void              destroyStreamDataBlock(SStreamDataBlock* pBlock);

 int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData);
-int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock);
+int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock);

 int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq);

--- a/source/libs/stream/src/stream.c
+++ b/source/libs/stream/src/stream.c
@ -181,7 +181,7 @@ static int32_t streamTaskAppendInputBlocks(SStreamTask* pTask, const SStreamDisp
  return status;
 }

-int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) {
+int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq) {
  SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SStreamDataBlock));
  int8_t            status = TASK_INPUT_STATUS__NORMAL;

@ -203,17 +203,6 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq,
    /*status = TASK_INPUT_STATUS__FAILED;*/
  }

-  // rsp by input status
-  void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp));
-  ((SMsgHead*)buf)->vgId = htonl(pReq->srcNodeId);
-  SStreamRetrieveRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead));
-  pCont->streamId = pReq->streamId;
-  pCont->rspToTaskId = pReq->srcTaskId;
-  pCont->rspFromTaskId = pReq->dstTaskId;
-  pRsp->pCont = buf;
-  pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp);
-  tmsgSendRsp(pRsp);
-
  return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1;
 }

@ -295,11 +284,12 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S
  return 0;
 }

-int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) {
-  streamTaskEnqueueRetrieve(pTask, pReq, pRsp);
-  ASSERT(pTask->info.taskLevel != TASK_LEVEL__SINK);
-  streamSchedExec(pTask);
-  return 0;
+int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq) {
+  int32_t code = streamTaskEnqueueRetrieve(pTask, pReq);
+  if(code != 0){
+    return code;
+  }
+  return streamSchedExec(pTask);
 }

 void streamTaskInputFail(SStreamTask* pTask) { atomic_store_8(&pTask->inputq.status, TASK_INPUT_STATUS__FAILED); }
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@ -162,16 +162,71 @@ int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) {
  return 0;
 }

-void tDeleteStreamRetrieveReq(SStreamRetrieveReq* pReq) { taosMemoryFree(pReq->pRetrieve); }
+void sendRetrieveRsp(SStreamRetrieveReq *pReq, SRpcMsg* pRsp){
+  void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp));
+  ((SMsgHead*)buf)->vgId = htonl(pReq->srcNodeId);
+  SStreamRetrieveRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead));
+  pCont->streamId = pReq->streamId;
+  pCont->rspToTaskId = pReq->srcTaskId;
+  pCont->rspFromTaskId = pReq->dstTaskId;
+  pRsp->pCont = buf;
+  pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp);
+  tmsgSendRsp(pRsp);
+}

-int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) {
-  int32_t            code = -1;
+int32_t broadcastRetrieveMsg(SStreamTask* pTask, SStreamRetrieveReq *req){
+  int32_t code = 0;
+  void*   buf = NULL;
+  int32_t sz = taosArrayGetSize(pTask->upstreamInfo.pList);
+  ASSERT(sz > 0);
+  for (int32_t i = 0; i < sz; i++) {
+    req->reqId = tGenIdPI64();
+    SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
+    req->dstNodeId = pEpInfo->nodeId;
+    req->dstTaskId = pEpInfo->taskId;
+    int32_t len;
+    tEncodeSize(tEncodeStreamRetrieveReq, req, len, code);
+    if (code != 0) {
+      ASSERT(0);
+      return code;
+    }
+
+    buf = rpcMallocCont(sizeof(SMsgHead) + len);
+    if (buf == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      return code;
+    }
+
+    ((SMsgHead*)buf)->vgId = htonl(pEpInfo->nodeId);
+    void*    abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+    SEncoder encoder;
+    tEncoderInit(&encoder, abuf, len);
+    tEncodeStreamRetrieveReq(&encoder, req);
+    tEncoderClear(&encoder);
+
+    SRpcMsg rpcMsg = {0};
+    initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE, buf, len + sizeof(SMsgHead));
+
+    code = tmsgSendReq(&pEpInfo->epSet, &rpcMsg);
+    if (code != 0) {
+      ASSERT(0);
+      rpcFreeCont(buf);
+      return code;
+    }
+
+    buf = NULL;
+    stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr,
+            pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId);
+  }
+  return code;
+}
+
+static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBlock, SStreamRetrieveReq* req){
  SRetrieveTableRsp* pRetrieve = NULL;
-  void*              buf = NULL;
  int32_t            dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);

  pRetrieve = taosMemoryCalloc(1, dataStrLen);
-  if (pRetrieve == NULL) return -1;
+  if (pRetrieve == NULL) return TSDB_CODE_OUT_OF_MEMORY;

  int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
  pRetrieve->useconds = 0;
@ -187,57 +242,24 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)

  int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols);

-  SStreamRetrieveReq req = {
-      .streamId = pTask->id.streamId,
-      .srcNodeId = pTask->info.nodeId,
-      .srcTaskId = pTask->id.taskId,
-      .pRetrieve = pRetrieve,
-      .retrieveLen = dataStrLen,
-  };
+  req->streamId = pTask->id.streamId;
+  req->srcNodeId = pTask->info.nodeId;
+  req->srcTaskId = pTask->id.taskId;
+  req->pRetrieve = pRetrieve;
+  req->retrieveLen = dataStrLen;
+  return 0;
+}

-  int32_t sz = taosArrayGetSize(pTask->upstreamInfo.pList);
-  ASSERT(sz > 0);
-  for (int32_t i = 0; i < sz; i++) {
-    req.reqId = tGenIdPI64();
-    SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
-    req.dstNodeId = pEpInfo->nodeId;
-    req.dstTaskId = pEpInfo->taskId;
-    int32_t len;
-    tEncodeSize(tEncodeStreamRetrieveReq, &req, len, code);
-    if (code < 0) {
-      ASSERT(0);
-      return -1;
-    }
-
-    buf = rpcMallocCont(sizeof(SMsgHead) + len);
-    if (buf == NULL) {
-      goto CLEAR;
-    }
-
-    ((SMsgHead*)buf)->vgId = htonl(pEpInfo->nodeId);
-    void*    abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-    SEncoder encoder;
-    tEncoderInit(&encoder, abuf, len);
-    tEncodeStreamRetrieveReq(&encoder, &req);
-    tEncoderClear(&encoder);
-
-    SRpcMsg rpcMsg = {0};
-    initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE, buf, len + sizeof(SMsgHead));
-
-    if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) {
-      ASSERT(0);
-      goto CLEAR;
-    }
-
-    buf = NULL;
-    stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr,
-            pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId);
+int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock) {
+  SStreamRetrieveReq req;
+  int32_t code = buildStreamRetrieveReq(pTask, pBlock, &req);
+  if(code != 0){
+    return code;
  }
-  code = 0;

-CLEAR:
-  taosMemoryFree(pRetrieve);
-  rpcFreeCont(buf);
+  code = broadcastRetrieveMsg(pTask, &req);
+  taosMemoryFree(req.pRetrieve);
+
  return code;
 }

--- a/source/libs/stream/src/streamExec.c
+++ b/source/libs/stream/src/streamExec.c
@ -138,7 +138,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i
    }

    if (output->info.type == STREAM_RETRIEVE) {
-      if (streamBroadcastToChildren(pTask, output) < 0) {
+      if (streamBroadcastToUpTasks(pTask, output) < 0) {
        // TODO
      }
      continue;
--- a/source/libs/stream/src/streamTask.c
+++ b/source/libs/stream/src/streamTask.c
@ -104,7 +104,7 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset,
  }

  char buf[128] = {0};
-  sprintf(buf, "0x%" PRIx64 "-%d", pTask->id.streamId, pTask->id.taskId);
+  sprintf(buf, "0x%" PRIx64 "-0x%x", pTask->id.streamId, pTask->id.taskId);

  pTask->id.idStr = taosStrdup(buf);
  pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE;
--- a/tests/parallel_test/cases.task
+++ b/tests/parallel_test/cases.task
@ -4,12 +4,8 @@

 #unit-test

-archOs=$(arch)
-if [[ $archOs =~ "aarch64" ]]; then
 ,,n,unit-test,bash test.sh
-else
-,,y,unit-test,bash test.sh
-fi
+

 #
 # army-test
@ -25,10 +21,10 @@ fi
 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1

-
 #
 # system test
 #
+,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_multi_agg.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/scalar_function.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_interval.py
--- a/tests/script/tsim/stream/distributeInterval0.sim
+++ b/tests/script/tsim/stream/distributeInterval0.sim
@ -475,6 +475,8 @@ sql create table ts1 using st tags(1,1,1);
 sql create table ts2 using st tags(2,2,2);
 sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ;

+sleep 1000
+
 sql insert into ts1 values(1648791211000,1,2,3);
 sleep 50
 sql insert into ts1 values(1648791222001,2,2,3);
@ -488,6 +490,9 @@ $loop_count = 0
 loop3:
 sql select * from streamtST3;

+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+
 sleep 1000
 $loop_count = $loop_count + 1
 if $loop_count == 30 then
--- a/tests/script/tsim/stream/fillHistoryBasic2.sim
+++ b/tests/script/tsim/stream/fillHistoryBasic2.sim
@ -243,6 +243,8 @@ sql create table ts1 using st tags(1,1,1);
 sql create table ts2 using st tags(2,2,2);
 sql create stream stream_t3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0  into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ;

+sleep 1000
+
 sql insert into ts1 values(1648791211000,1,2,3);
 sleep 50
 sql insert into ts1 values(1648791222001,2,2,3);
--- a/tests/system-test/7-tmq/tmqParamsTest.py
+++ b/tests/system-test/7-tmq/tmqParamsTest.py
@ -133,13 +133,17 @@ class TDTestCase:
                        if snapshot_value == "true":
                            if offset_value != "earliest" and offset_value != "":
                                if offset_value == "latest":
-                                    offset_value_list = list(map(lambda x: int(x[-2].replace("wal:", "").replace("earliest", "0").replace("latest", "0").replace(offset_value, "0")), subscription_info))
-                                    tdSql.checkEqual(sum(offset_value_list) >= 0, True)
+                                    offset_value_list = list(map(lambda x: (x[-2].replace("wal:", "").replace("earliest", "0").replace("latest", "0").replace(offset_value, "0")), subscription_info))
+                                    offset_value_list1 = list(map(lambda x: int(x.split("/")[0]), offset_value_list))
+                                    offset_value_list2 = list(map(lambda x: int(x.split("/")[1]), offset_value_list))
+                                    tdSql.checkEqual(offset_value_list1 == offset_value_list2, True)
+                                    tdSql.checkEqual(sum(offset_value_list1) >= 0, True)
                                    rows_value_list  = list(map(lambda x: int(x[-1]), subscription_info))
                                    tdSql.checkEqual(sum(rows_value_list), expected_res)
                                elif offset_value == "none":
                                    offset_value_list = list(map(lambda x: x[-2], subscription_info))
-                                    tdSql.checkEqual(offset_value_list, ['none']*len(subscription_info))
+                                    offset_value_list1 = list(map(lambda x: (x.split("/")[0]), offset_value_list))
+                                    tdSql.checkEqual(offset_value_list1, ['none']*len(subscription_info))
                                    rows_value_list  = list(map(lambda x: x[-1], subscription_info))
                                    tdSql.checkEqual(rows_value_list, [0]*len(subscription_info))
                            else:
@ -151,18 +155,23 @@ class TDTestCase:
                                    # tdSql.checkEqual(sum(rows_value_list), expected_res)
                                else:
                                    offset_value_list = list(map(lambda x: x[-2], subscription_info))
-                                    tdSql.checkEqual(offset_value_list, [None]*len(subscription_info))
+                                    offset_value_list1 = list(map(lambda x: (x.split("/")[0]), offset_value_list))
+                                    tdSql.checkEqual(offset_value_list1, [None]*len(subscription_info))
                                    rows_value_list  = list(map(lambda x: x[-1], subscription_info))
                                    tdSql.checkEqual(rows_value_list, [None]*len(subscription_info))
                        else:
                            if offset_value != "none":
-                                offset_value_list = list(map(lambda x: int(x[-2].replace("wal:", "").replace("earliest", "0").replace("latest", "0").replace(offset_value, "0")), subscription_info))
-                                tdSql.checkEqual(sum(offset_value_list) >= 0, True)
+                                offset_value_list = list(map(lambda x: (x[-2].replace("wal:", "").replace("earliest", "0").replace("latest", "0").replace(offset_value, "0")), subscription_info))
+                                offset_value_list1 = list(map(lambda x: int(x.split("/")[0]), offset_value_list))
+                                offset_value_list2 = list(map(lambda x: int(x.split("/")[1]), offset_value_list))
+                                tdSql.checkEqual(offset_value_list1 == offset_value_list2, True)
+                                tdSql.checkEqual(sum(offset_value_list1) >= 0, True)
                                rows_value_list  = list(map(lambda x: int(x[-1]), subscription_info))
                                tdSql.checkEqual(sum(rows_value_list), expected_res)
                            else:
                                offset_value_list = list(map(lambda x: x[-2], subscription_info))
-                                tdSql.checkEqual(offset_value_list, ['none']*len(subscription_info))
+                                offset_value_list1 = list(map(lambda x: (x.split("/")[0]), offset_value_list))
+                                tdSql.checkEqual(offset_value_list1, ['none']*len(subscription_info))
                                rows_value_list  = list(map(lambda x: x[-1], subscription_info))
                                tdSql.checkEqual(rows_value_list, [0]*len(subscription_info))
                        tdSql.execute(f"drop topic if exists {topic_name}")
--- a/tests/system-test/7-tmq/tmq_taosx.py
+++ b/tests/system-test/7-tmq/tmq_taosx.py
@ -11,6 +11,7 @@ from util.sql import *
 from util.cases import *
 from util.dnodes import *
 from util.common import *
+from taos.tmq import *
 sys.path.append("./7-tmq")
 from tmqCommon import *

@ -310,6 +311,43 @@ class TDTestCase:

        return

+    def consumeExcluded(self):
+        tdSql.execute(f'create topic topic_excluded as database db_taosx')
+        consumer_dict = {
+            "group.id": "g1",
+            "td.connect.user": "root",
+            "td.connect.pass": "taosdata",
+            "auto.offset.reset": "earliest",
+            "msg.consume.excluded": 1
+        }
+        consumer = Consumer(consumer_dict)
+
+        tdLog.debug("test subscribe topic created by other user")
+        exceptOccured = False
+        try:
+            consumer.subscribe(["topic_excluded"])
+        except TmqError:
+            exceptOccured = True
+
+        if exceptOccured:
+            tdLog.exit(f"subscribe error")
+
+        try:
+            while True:
+                res = consumer.poll(1)
+                if not res:
+                    break
+                err = res.error()
+                if err is not None:
+                    raise err
+                val = res.value()
+
+                for block in val:
+                    print(block.fetchall())
+
+        finally:
+            consumer.close()
+
    def run(self):
        tdSql.prepare()
        self.checkWal1VgroupOnlyMeta()
@ -324,6 +362,8 @@ class TDTestCase:
        self.checkSnapshotMultiVgroups()

        self.checkWalMultiVgroupsWithDropTable()
+        # self.consumeExcluded()
+
        self.checkSnapshotMultiVgroupsWithDropTable()

    def stop(self):
--- a/tests/system-test/8-stream/stream_multi_agg.py
+++ b/tests/system-test/8-stream/stream_multi_agg.py
@ -0,0 +1,100 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+from util.sqlset import *
+from util.autogen import *
+
+import random
+import time
+import traceback
+import os
+from   os import path
+
+
+class TDTestCase:
+    updatecfgDict = {'debugFlag': 135, 'asynclog': 0, 'streamAggCnt': 2}
+    # init 
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), True)
+
+    def case1(self):
+        tdLog.debug("========case1 start========")
+
+        os.system("nohup taosBenchmark -y -B 1 -t 40 -S 1000 -n 10 -i 1000 -v 5  > /dev/null 2>&1 &")
+        time.sleep(10)
+        tdSql.query("use test")
+        tdSql.query("create stream if not exists s1 trigger at_once  ignore expired 0 ignore update 0  fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)")
+        tdLog.debug("========create stream and insert data ok========")
+        time.sleep(15)
+
+        tdSql.query("select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s) order by groupid,_wstart")
+        rowCnt = tdSql.getRows()
+        results = []
+        for i in range(rowCnt):
+            results.append(tdSql.getData(i,1))
+
+        tdSql.query("select * from st1 order by groupid,_wstart")
+        tdSql.checkRows(rowCnt)
+        for i in range(rowCnt):
+            data1 = tdSql.getData(i,1)
+            data2 = results[i]
+            if data1 != data2:
+                tdLog.info("num: %d, act data: %d, expect data: %d"%(i, data1, data2))
+                tdLog.exit("check data error!")
+
+        tdLog.debug("case1 end")
+
+    def case2(self):
+        tdLog.debug("========case2 start========")
+
+        os.system("taosBenchmark -d db -t 20 -v 6 -n 1000 -y  > /dev/null 2>&1")
+        # create stream
+        tdSql.execute("use db")
+        tdSql.execute("create stream stream1 fill_history 1 into sta as select count(*) as cnt from meters interval(10a);",show=True)
+        time.sleep(5)
+
+        sql = "select count(*) from sta"
+        # loop wait max 60s to check count is ok
+        tdLog.info("loop wait result ...")
+        tdSql.checkDataLoop(0, 0, 100, sql, loopCount=10, waitTime=0.5)
+
+        # check all data is correct
+        sql = "select * from sta where cnt != 200;"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+
+        # check ts interval is correct
+        sql = "select * from ( select diff(_wstart) as tsdif from sta ) where tsdif != 10;"
+        tdSql.query(sql)
+        tdSql.checkRows(0)
+        tdLog.debug("case2 end")
+
+# run
+    def run(self):
+        self.case1()
+        self.case2()
+
+    # stop
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addLinux(__file__, TDTestCase())
--- a/tests/system-test/test.py
+++ b/tests/system-test/test.py
@ -683,6 +683,6 @@ if __name__ == "__main__":
    if conn is not None:
        conn.close()
    if asan:
-        tdDnodes.StopAllSigint()
+        # tdDnodes.StopAllSigint()
        tdLog.info("Address sanitizer mode finished")
    sys.exit(0)
--- a/tests/unit-test/test.sh
+++ b/tests/unit-test/test.sh
@ -40,7 +40,7 @@ pgrep taosd || taosd >> /dev/null 2>&1 &

 sleep 10

-ctest -E "smlTest|funcTest|profileTest|sdbTest|showTest|geomTest|idxFstUtilUT|idxTest|idxUtilUT|idxFstUT|parserTest|plannerTest|transUT|transUtilUt" -j8
+ctest -j8

 ret=$?
 exit $ret
--- a/utils/test/c/tmq_taosx_ci.c
+++ b/utils/test/c/tmq_taosx_ci.c
@ -909,6 +909,88 @@ void initLogFile() {
  taosCloseFile(&pFile2);
 }

+void testConsumeExcluded(int topic_type){
+  TAOS* pConn = use_db();
+  TAOS_RES *pRes = NULL;
+
+  if(topic_type == 1){
+    char *topic = "create topic topic_excluded with meta as database db_taosx";
+    pRes = taos_query(pConn, topic);
+    if (taos_errno(pRes) != 0) {
+      printf("failed to create topic topic_excluded, reason:%s\n", taos_errstr(pRes));
+      taos_close(pConn);
+      return;
+    }
+    taos_free_result(pRes);
+  }else if(topic_type == 2){
+    char *topic = "create topic topic_excluded as select * from stt";
+    pRes = taos_query(pConn, topic);
+    if (taos_errno(pRes) != 0) {
+      printf("failed to create topic topic_excluded, reason:%s\n", taos_errstr(pRes));
+      taos_close(pConn);
+      return;
+    }
+    taos_free_result(pRes);
+  }
+  taos_close(pConn);
+
+  tmq_conf_t* conf = tmq_conf_new();
+  tmq_conf_set(conf, "group.id", "tg2");
+  tmq_conf_set(conf, "client.id", "my app 1");
+  tmq_conf_set(conf, "td.connect.user", "root");
+  tmq_conf_set(conf, "td.connect.pass", "taosdata");
+  tmq_conf_set(conf, "msg.with.table.name", "true");
+  tmq_conf_set(conf, "enable.auto.commit", "true");
+  tmq_conf_set(conf, "auto.offset.reset", "earliest");
+  tmq_conf_set(conf, "msg.consume.excluded", "1");
+
+
+  tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
+  tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
+  assert(tmq);
+  tmq_conf_destroy(conf);
+
+  tmq_list_t* topic_list = tmq_list_new();
+  tmq_list_append(topic_list, "topic_excluded");
+
+  int32_t code = 0;
+
+  if ((code = tmq_subscribe(tmq, topic_list))) {
+    fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code));
+    printf("subscribe err\n");
+    return;
+  }
+  while (running) {
+    TAOS_RES* msg = tmq_consumer_poll(tmq, 1000);
+    if (msg) {
+      tmq_raw_data raw = {0};
+      tmq_get_raw(msg, &raw);
+      if(topic_type == 1){
+        assert(raw.raw_type != 2 && raw.raw_type != 4);
+      }else if(topic_type == 2){
+        assert(0);
+      }
+//      printf("write raw data type: %d\n", raw.raw_type);
+      tmq_free_raw(raw);
+
+      taos_free_result(msg);
+    } else {
+      break;
+    }
+  }
+
+  tmq_consumer_close(tmq);
+  tmq_list_destroy(topic_list);
+
+  pConn = use_db();
+  pRes = taos_query(pConn, "drop topic if exists topic_excluded");
+  if (taos_errno(pRes) != 0) {
+    printf("error in drop topic, reason:%s\n", taos_errstr(pRes));
+    taos_close(pConn);
+    return;
+  }
+  taos_free_result(pRes);
+}
 int main(int argc, char* argv[]) {
  for (int32_t i = 1; i < argc; i++) {
    if (strcmp(argv[i], "-c") == 0) {
@ -942,5 +1024,8 @@ int main(int argc, char* argv[]) {
  tmq_list_t* topic_list = build_topic_list();
  basic_consume_loop(tmq, topic_list);
  tmq_list_destroy(topic_list);
+
+  testConsumeExcluded(1);
+  testConsumeExcluded(2);
  taosCloseFile(&g_fp);
 }