Merge branch '3.0' into cpwu/3.0

docs/en/14-reference/03-connector/java.mdx

@@ -91,7 +91,7 @@ Add following dependency in the `pom.xml` file of your Maven project:
 You can build Java connector from source code after cloning the TDengine project:
 
 ```
-git clone https://github.com/taosdata/taos-connector-jdbc.git
+git clone https://github.com/taosdata/taos-connector-jdbc.git --branch 2.0
 cd taos-connector-jdbc
 mvn clean install -Dmaven.test.skip=true
 ```

docs/zh/14-reference/03-connector/java.mdx

@@ -93,7 +93,7 @@ Maven 项目中，在 pom.xml 中添加以下依赖：
 可以通过下载 TDengine 的源码，自己编译最新版本的 Java connector
 
 ```shell
-git clone https://github.com/taosdata/taos-connector-jdbc.git
+git clone https://github.com/taosdata/taos-connector-jdbc.git --branch 2.0
 cd taos-connector-jdbc
 mvn clean install -Dmaven.test.skip=true
 ```

include/common/taosdef.h

@@ -34,11 +34,10 @@ typedef enum {
   TSDB_SUPER_TABLE = 1,   // super table
   TSDB_CHILD_TABLE = 2,   // table created from super table
   TSDB_NORMAL_TABLE = 3,  // ordinary table
-  TSDB_STREAM_TABLE = 4,  // table created from stream computing
+  TSDB_TEMP_TABLE = 4,    // temp table created by nest query
-  TSDB_TEMP_TABLE = 5,    // temp table created by nest query
+  TSDB_SYSTEM_TABLE = 5,
-  TSDB_SYSTEM_TABLE = 6,
+  TSDB_TSMA_TABLE = 6,  // time-range-wise sma
-  TSDB_TSMA_TABLE = 7,  // time-range-wise sma
+  TSDB_TABLE_MAX = 7
-  TSDB_TABLE_MAX = 8
 } ETableType;
 
 typedef enum {

include/common/tdatablock.h

@@ -184,8 +184,8 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u
 int32_t getJsonValueLen(const char* data);
 
 int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull);
-int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity,
+int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity,
-                        const SColumnInfoData* pSource, uint32_t numOfRow2);
+                        const SColumnInfoData* pSource, int32_t numOfRow2);
 int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,
                       const SDataBlockInfo* pBlockInfo);
 int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex);

include/common/tmsg.h

@@ -2537,6 +2537,15 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq)
   return (void*)buf;
 }
 
+typedef struct {
+  char    topic[TSDB_TOPIC_FNAME_LEN];
+  int64_t ntbUid;
+  SArray* colIdList;  // SArray<int16_t>
+} SCheckAlterInfo;
+
+int32_t tEncodeSCheckAlterInfo(SEncoder* pEncoder, const SCheckAlterInfo* pInfo);
+int32_t tDecodeSCheckAlterInfo(SDecoder* pDecoder, SCheckAlterInfo* pInfo);
+
 typedef struct {
   int32_t vgId;
   int64_t offset;

include/common/tmsgdef.h

@@ -186,6 +186,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_DELETE, "vnode-mq-vg-delete", SMqVDeleteReq, SMqVDeleteRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_MQ_COMMIT_OFFSET, "vnode-commit-offset", STqOffset, STqOffset)
+  TD_DEF_MSG_TYPE(TDMT_VND_CHECK_ALTER_INFO, "vnode-alter-check-info", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TOPIC, "vnode-create-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TOPIC, "vnode-alter-topic", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL)

include/libs/executor/executor.h

@@ -64,7 +64,8 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers);
  * @param SReadHandle
  * @return
  */
-qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchemaWrapper);
+qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols,
+                                     SSchemaWrapper** pSchema);
 
 /**
  * Set the input data block for the stream scan.
@@ -195,6 +196,8 @@ int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem);
 
 int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer);
 
+STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/plannodes.h

@@ -111,6 +111,7 @@ typedef struct SAggLogicNode {
   SNodeList* pGroupKeys;
   SNodeList* pAggFuncs;
   bool       hasLastRow;
+  bool       hasTimeLineFunc;
 } SAggLogicNode;
 
 typedef struct SProjectLogicNode {

include/util/taoserror.h

@@ -332,6 +332,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_VND_INVALID_TABLE_ACTION      TAOS_DEF_ERROR_CODE(0, 0x0519)
 #define TSDB_CODE_VND_COL_ALREADY_EXISTS        TAOS_DEF_ERROR_CODE(0, 0x051a)
 #define TSDB_CODE_VND_TABLE_COL_NOT_EXISTS      TAOS_DEF_ERROR_CODE(0, 0x051b)
+#define TSDB_CODE_VND_COL_SUBSCRIBED            TAOS_DEF_ERROR_CODE(0, 0x051c)
 
 // tsdb
 #define TSDB_CODE_TDB_INVALID_TABLE_ID          TAOS_DEF_ERROR_CODE(0, 0x0600)

source/client/test/CMakeLists.txt

@@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(clientTest clientTests.cpp)
 TARGET_LINK_LIBRARIES(
         clientTest
-        PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom
+        PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom executor
 )
 
 ADD_EXECUTABLE(tmqTest tmqTest.cpp)

source/client/test/clientTests.cpp

@@ -27,6 +27,7 @@
 #pragma GCC diagnostic ignored "-Wsign-compare"
 
 #include "taos.h"
+#include "executor.h"
 
 namespace {
 void showDB(TAOS* pConn) {
@@ -823,6 +824,17 @@ TEST(testCase, async_api_test) {
 
 
 TEST(testCase, update_test) {
+
+  SInterval interval = {0};
+  interval.offset = 8000;
+  interval.interval = 10000;
+  interval.sliding = 4000;
+  interval.intervalUnit = 's';
+  interval.offsetUnit = 's';
+  interval.slidingUnit = 's';
+// STimeWindow w = getAlignQueryTimeWindow(&interval, 0, 1630000000000);
+ STimeWindow w = getAlignQueryTimeWindow(&interval, 0, 1629999999999);
+
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
 

source/common/src/tdatablock.c

@@ -214,8 +214,8 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c
   }
 }
 
-int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity,
+int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity,
-                        const SColumnInfoData* pSource, uint32_t numOfRow2) {
+                        const SColumnInfoData* pSource, int32_t numOfRow2) {
   ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type);
   if (numOfRow2 == 0) {
     return numOfRow1;
@@ -263,7 +263,8 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, ui
     pColumnInfoData->varmeta.length = len + oldLen;
   } else {
     if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) {
-      ASSERT(finalNumOfRows * pColumnInfoData->info.bytes);
+      // all data may be null, when the pColumnInfoData->info.type == 0, bytes == 0;
+//      ASSERT(finalNumOfRows * pColumnInfoData->info.bytes);
       char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes);
       if (tmp == NULL) {
         return TSDB_CODE_VND_OUT_OF_MEMORY;

source/common/src/tglobal.c

@@ -183,7 +183,7 @@ bool tsStartUdfd = true;
 int32_t tsTransPullupInterval = 2;
 int32_t tsMqRebalanceInterval = 2;
 int32_t tsTtlUnit = 86400;
-int32_t tsTtlPushInterval = 60;
+int32_t tsTtlPushInterval = 86400;
 int32_t tsGrantHBInterval = 60;
 
 void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
@@ -226,9 +226,18 @@ static int32_t taosSetTfsCfg(SConfig *pCfg) {
   }
 
   if (tsDataDir[0] == 0) {
+    if (pItem->str != NULL) {
+      taosAddDataDir(0, pItem->str, 0, 1);
+      tstrncpy(tsDataDir, pItem->str, PATH_MAX);
+      if (taosMulMkDir(tsDataDir) != 0) {
+        uError("failed to create dataDir:%s since %s", tsDataDir, terrstr());
+        return -1;
+      }
+    } else {
       uError("datadir not set");
       return -1;
     }
+  }
 
   return 0;
 }
@@ -466,7 +475,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, 1) != 0) return -1;
-  if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 10000, 1) != 0) return -1;
+  if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 100000, 1) != 0) return -1;
 
   if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1;
   return 0;

source/common/src/tmsg.c

@@ -5623,6 +5623,33 @@ int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) {
   return 0;
 }
 
+int32_t tEncodeSCheckAlterInfo(SEncoder *pEncoder, const SCheckAlterInfo *pInfo) {
+  if (tEncodeCStr(pEncoder, pInfo->topic) < 0) return -1;
+  if (tEncodeI64(pEncoder, pInfo->ntbUid) < 0) return -1;
+  int32_t sz = taosArrayGetSize(pInfo->colIdList);
+  if (tEncodeI32(pEncoder, sz) < 0) return -1;
+  for (int32_t i = 0; i < sz; i++) {
+    int16_t colId = *(int16_t *)taosArrayGet(pInfo->colIdList, i);
+    if (tEncodeI16(pEncoder, colId) < 0) return -1;
+  }
+  return pEncoder->pos;
+}
+
+int32_t tDecodeSCheckAlterInfo(SDecoder *pDecoder, SCheckAlterInfo *pInfo) {
+  if (tDecodeCStrTo(pDecoder, pInfo->topic) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pInfo->ntbUid) < 0) return -1;
+  int32_t sz;
+  if (tDecodeI32(pDecoder, &sz) < 0) return -1;
+  pInfo->colIdList = taosArrayInit(sz, sizeof(int16_t));
+  if (pInfo->colIdList == NULL) return -1;
+  for (int32_t i = 0; i < sz; i++) {
+    int16_t colId;
+    if (tDecodeI16(pDecoder, &colId) < 0) return -1;
+    taosArrayPush(pInfo->colIdList, &colId);
+  }
+  return 0;
+}
+
 int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) {
   int32_t nUid = taosArrayGetSize(pRes->uidList);
 

source/common/src/ttime.c

@@ -815,9 +815,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio
 
   if (pInterval->offset > 0) {
     start = taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision);
-    if (start > t) {
+
-      start = taosTimeAdd(start, -pInterval->interval, pInterval->intervalUnit, precision);
-    } else {
     // try to move current window to the left-hande-side, due to the offset effect.
     int64_t end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1;
 
@@ -829,7 +827,6 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio
 
     start = taosTimeAdd(end, -pInterval->interval, pInterval->intervalUnit, precision) + 1;
   }
-  }
 
   return start;
 }

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -224,6 +224,7 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_CHECK_ALTER_INFO_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -352,6 +352,7 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_CHECK_ALTER_INFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

source/dnode/mnode/impl/inc/mndDef.h

@@ -477,6 +477,10 @@ typedef struct {
   char*          physicalPlan;
   SSchemaWrapper schema;
   int64_t        stbUid;
+  // forbid condition
+  int64_t ntbUid;
+  SArray* ntbColIds;
+  int64_t ctbStbUid;
 } SMqTopicObj;
 
 typedef struct {

source/dnode/mnode/impl/inc/mndTopic.h

@@ -37,7 +37,7 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]);
 
 int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
 
-int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId);
+int32_t mndCheckColAndTagModifiable(SMnode *pMnode,  const char* stbname, int64_t suid, col_id_t colId);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndStb.c

@@ -1144,7 +1144,7 @@ static int32_t mndDropSuperTableTag(SMnode *pMnode, const SStbObj *pOld, SStbObj
   }
 
   col_id_t colId = pOld->pTags[tag].colId;
-  if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) {
+  if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) {
     return -1;
   }
 
@@ -1179,7 +1179,7 @@ static int32_t mndAlterStbTagName(SMnode *pMnode, const SStbObj *pOld, SStbObj *
   }
 
   col_id_t colId = pOld->pTags[tag].colId;
-  if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) {
+  if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) {
     return -1;
   }
 
@@ -1213,7 +1213,7 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj
   }
 
   col_id_t colId = pOld->pTags[tag].colId;
-  if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) {
+  if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) {
     return -1;
   }
 
@@ -1295,7 +1295,7 @@ static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStb
   }
 
   col_id_t colId = pOld->pColumns[col].colId;
-  if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) {
+  if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) {
     return -1;
   }
 
@@ -1329,7 +1329,7 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO
   }
 
   col_id_t colId = pOld->pColumns[col].colId;
-  if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) {
+  if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) {
     return -1;
   }
 

source/dnode/mnode/impl/src/mndTopic.c

@@ -57,6 +57,7 @@ int32_t mndInitTopic(SMnode *pMnode) {
   mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicReq);
   mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq);
   mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndTransProcessRsp);
+  mndSetMsgHandle(pMnode, TDMT_VND_CHECK_ALTER_INFO_RSP, mndTransProcessRsp);
 
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic);
   mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextTopic);
@@ -71,14 +72,16 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) {
   return strchr(topic, '.') + 1;
 }
 
-int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId) {
+int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char* stbname, int64_t suid, col_id_t colId) {
   SSdb *pSdb = pMnode->pSdb;
   void *pIter = NULL;
-  bool  found = false;
   while (1) {
     SMqTopicObj *pTopic = NULL;
     pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
     if (pIter == NULL) break;
+
+    mDebug("topic:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, subType:%d sql:%s", 
+          pTopic->name, stbname, suid, colId, pTopic->subType, pTopic->sql);
     if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) {
       sdbRelease(pSdb, pTopic);
       continue;
@@ -95,21 +98,25 @@ int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId
     SNode *pNode = NULL;
     FOREACH(pNode, pNodeList) {
       SColumnNode *pCol = (SColumnNode *)pNode;
-      if (pCol->tableId != suid) goto NEXT;
+      mDebug("topic:%s, check colId:%d tableId:%" PRId64 " ctbStbUid:%" PRId64, pTopic->name, pCol->colId, pCol->tableId, pTopic->ctbStbUid);
-      if (pCol->colId > 0 && pCol->colId == colId) {
+   
-        found = true;
+      if (pCol->tableId != suid && pTopic->ctbStbUid != suid) {
+        mDebug("topic:%s, check colId:%d passed", pTopic->name, pCol->colId);
         goto NEXT;
       }
-      mTrace("topic:%s, colId:%d is used", pTopic->name, pCol->colId);
+      if (pCol->colId > 0 && pCol->colId == colId) {
+        sdbRelease(pSdb, pTopic);
+        nodesDestroyNode(pAst);
+        terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC;
+        mError("topic:%s, check colId:%d conflicted", pTopic->name, pCol->colId);
+        return -1;
+      }
+      mDebug("topic:%s, check colId:%d passed", pTopic->name, pCol->colId);
     }
 
   NEXT:
     sdbRelease(pSdb, pTopic);
     nodesDestroyNode(pAst);
-    if (found) {
-      terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC;
-      return -1;
-    }
   }
 
   return 0;
@@ -127,8 +134,10 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   if (pTopic->schema.nCols) {
     schemaLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema);
   }
-  int32_t size =
+  int32_t ntbColLen = taosArrayGetSize(pTopic->ntbColIds) * sizeof(int16_t);
-      sizeof(SMqTopicObj) + physicalPlanLen + pTopic->sqlLen + pTopic->astLen + schemaLen + MND_TOPIC_RESERVE_SIZE;
+
+  int32_t size = sizeof(SMqTopicObj) + physicalPlanLen + pTopic->sqlLen + pTopic->astLen + schemaLen + ntbColLen +
+                 MND_TOPIC_RESERVE_SIZE;
   SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size);
   if (pRaw == NULL) goto TOPIC_ENCODE_OVER;
 
@@ -164,6 +173,16 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
     taosEncodeSSchemaWrapper(&aswBuf, &pTopic->schema);
     SDB_SET_BINARY(pRaw, dataPos, swBuf, schemaLen, TOPIC_ENCODE_OVER);
   }
+  SDB_SET_INT64(pRaw, dataPos, pTopic->ntbUid, TOPIC_ENCODE_OVER);
+  if (pTopic->ntbUid != 0) {
+    int32_t sz = taosArrayGetSize(pTopic->ntbColIds);
+    SDB_SET_INT32(pRaw, dataPos, sz, TOPIC_ENCODE_OVER);
+    for (int32_t i = 0; i < sz; i++) {
+      int16_t colId = *(int16_t *)taosArrayGet(pTopic->ntbColIds, i);
+      SDB_SET_INT16(pRaw, dataPos, colId, TOPIC_ENCODE_OVER);
+    }
+  }
+  SDB_SET_INT64(pRaw, dataPos, pTopic->ctbStbUid, TOPIC_ENCODE_OVER);
 
   SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER);
   SDB_SET_DATALEN(pRaw, dataPos, TOPIC_ENCODE_OVER);
@@ -259,6 +278,20 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
     pTopic->schema.version = 0;
     pTopic->schema.pSchema = NULL;
   }
+  SDB_GET_INT64(pRaw, dataPos, &pTopic->ntbUid, TOPIC_DECODE_OVER);
+  if (pTopic->ntbUid != 0) {
+    int32_t ntbColNum;
+    SDB_GET_INT32(pRaw, dataPos, &ntbColNum, TOPIC_DECODE_OVER);
+    pTopic->ntbColIds = taosArrayInit(ntbColNum, sizeof(int16_t));
+    if (pTopic->ntbColIds == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto TOPIC_DECODE_OVER;
+    }
+    int16_t colId;
+    SDB_GET_INT16(pRaw, dataPos, &colId, TOPIC_DECODE_OVER);
+    taosArrayPush(pTopic->ntbColIds, &colId);
+  }
+  SDB_GET_INT64(pRaw, dataPos, &pTopic->ctbStbUid, TOPIC_DECODE_OVER);
 
   SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER);
 
@@ -346,6 +379,26 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
   return 0;
 }
 
+static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) {
+  SNodeList *pNodeList = NULL;
+  nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList);
+  int64_t suid = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->suid;
+  int8_t  tableType = ((SRealTableNode *)((SSelectStmt *)pAst)->pFromTable)->pMeta->tableType;
+  if (tableType == TSDB_CHILD_TABLE) {
+    pTopic->ctbStbUid = suid;
+  } else if (tableType == TSDB_NORMAL_TABLE) {
+    SNode *pNode = NULL;
+    FOREACH(pNode, pNodeList) {
+      SColumnNode *pCol = (SColumnNode *)pNode;
+      if (pCol->tableType == TSDB_NORMAL_TABLE) {
+        pTopic->ntbUid = pCol->tableId;
+        taosArrayPush(pTopic->ntbColIds, &pCol->colId);
+      }
+    }
+  }
+  return 0;
+}
+
 static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) {
   mDebug("topic:%s to create", pCreate->name);
   SMqTopicObj topicObj = {0};
@@ -386,6 +439,19 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
       return -1;
     }
 
+    int64_t ntbUid;
+    topicObj.ntbColIds = taosArrayInit(0, sizeof(int16_t));
+    if (topicObj.ntbColIds == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return -1;
+    }
+    extractTopicTbInfo(pAst, &topicObj);
+
+    if (topicObj.ntbUid == 0) {
+      taosArrayDestroy(topicObj.ntbColIds);
+      topicObj.ntbColIds = NULL;
+    }
+
     if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) {
       mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
       taosMemoryFree(topicObj.ast);
@@ -433,6 +499,60 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
   }
   sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
 
+  if (topicObj.ntbUid != 0) {
+    SCheckAlterInfo info;
+    memcpy(info.topic, topicObj.name, TSDB_TOPIC_FNAME_LEN);
+    info.ntbUid = topicObj.ntbUid;
+    info.colIdList = topicObj.ntbColIds;
+    // broadcast forbid alter info
+    void   *pIter = NULL;
+    SSdb   *pSdb = pMnode->pSdb;
+    SVgObj *pVgroup = NULL;
+    while (1) {
+      // iterate vg
+      pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
+      if (pIter == NULL) break;
+      if (!mndVgroupInDb(pVgroup, topicObj.dbUid)) {
+        sdbRelease(pSdb, pVgroup);
+        continue;
+      }
+
+      // encoder check alter info
+      int32_t len;
+      int32_t code;
+      tEncodeSize(tEncodeSCheckAlterInfo, &info, len, code);
+      if (code < 0) {
+        sdbRelease(pSdb, pVgroup);
+        mndTransDrop(pTrans);
+        ASSERT(0);
+        return -1;
+      }
+      void    *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
+      void    *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+      SEncoder encoder;
+      tEncoderInit(&encoder, abuf, len);
+      if (tEncodeSCheckAlterInfo(&encoder, &info) < 0) {
+        sdbRelease(pSdb, pVgroup);
+        mndTransDrop(pTrans);
+        return -1;
+      }
+      tEncoderClear(&encoder);
+      ((SMsgHead *)buf)->vgId = htonl(pVgroup->vgId);
+      // add redo action
+      STransAction action = {0};
+      action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
+      action.pCont = buf;
+      action.contLen = sizeof(SMsgHead) + len;
+      action.msgType = TDMT_VND_CHECK_ALTER_INFO;
+      if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+        taosMemoryFree(buf);
+        sdbRelease(pSdb, pVgroup);
+        mndTransDrop(pTrans);
+        return -1;
+      }
+    }
+  }
+
   if (mndTransPrepare(pMnode, pTrans) != 0) {
     mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
     taosMemoryFreeClear(topicObj.physicalPlan);
@@ -442,7 +562,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
 
   taosMemoryFreeClear(topicObj.physicalPlan);
   mndTransDrop(pTrans);
-  return 0;
+  return TSDB_CODE_ACTION_IN_PROGRESS;
 }
 
 static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) {

source/dnode/vnode/inc/vnode.h

@@ -89,6 +89,7 @@ void        metaReaderClear(SMetaReader *pReader);
 int32_t     metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
 int32_t     metaReadNext(SMetaReader *pReader);
 const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *tagVal);
+int         metaGetTableNameByUid(void* meta, uint64_t uid, char* tbName);
 
 typedef struct SMetaFltParam {
   tb_uid_t suid;

source/dnode/vnode/src/inc/sma.h

@@ -67,8 +67,6 @@ struct STSmaStat {
 struct SRSmaStat {
   SSma     *pSma;
   int64_t   commitAppliedVer;  // vnode applied version for async commit
-  int64_t   commitSubmitVer;   // rsma submit version for async commit
-  int64_t   submitVer;         // latest submit version
   int64_t   refId;             // shared by fetch tasks
   int8_t    triggerStat;       // shared by fetch tasks
   int8_t    commitStat;        // 0 not in committing, 1 in committing
@@ -91,7 +89,6 @@ struct SSmaStat {
 #define RSMA_TRIGGER_STAT(r)  (&(r)->triggerStat)
 #define RSMA_COMMIT_STAT(r)   (&(r)->commitStat)
 #define RSMA_REF_ID(r)        ((r)->refId)
-#define RSMA_SUBMIT_VER(r)    ((r)->submitVer)
 
 struct SRSmaInfoItem {
   void   *taskInfo;  // qTaskInfo_t
@@ -223,13 +220,6 @@ struct STFInfo {
   uint32_t ftype;
   uint32_t fver;
   int64_t  fsize;
-
-  // specific fields
-  union {
-    struct {
-      int64_t submitVer;
-    } qTaskInfo;
-  };
 };
 
 enum {

source/dnode/vnode/src/inc/tq.h

@@ -110,9 +110,6 @@ typedef struct {
   // exec
   STqExecHandle execHandle;
 
-  // prevent drop
-  int64_t ntbUid;
-  SArray* colIdList;  // SArray<int32_t>
 } STqHandle;
 
 struct STQ {
@@ -120,9 +117,9 @@ struct STQ {
   SHashObj*       pushMgr;       // consumerId -> STqHandle*
   SHashObj*       handles;       // subKey -> STqHandle
   SHashObj*       pStreamTasks;  // taksId -> SStreamTask
+  SHashObj*       pAlterInfo;    // topic -> SAlterCheckInfo
   STqOffsetStore* pOffsetStore;
   SVnode*         pVnode;
-  SWal*           pWal;
   TDB*            pMetaStore;
   TTB*            pExecStore;
 };

source/dnode/vnode/src/inc/vnodeInt.h

@@ -137,12 +137,13 @@ STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableL
 // tq
 int     tqInit();
 void    tqCleanUp();
-STQ*    tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
+STQ*    tqOpen(const char* path, SVnode* pVnode);
 void    tqClose(STQ*);
 int     tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
 int     tqCommit(STQ*);
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
-int32_t tqCheckColModifiable(STQ* pTq, int32_t colId);
+int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId);
+int32_t tqProcessCheckAlterInfoReq(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen);
 int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen);
@@ -177,7 +178,6 @@ int32_t smaAsyncPostCommit(SSma* pSma);
 
 int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
 int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
-int64_t tdRSmaGetMaxSubmitVer(SSma* pSma, int8_t level);
 
 int32_t tdProcessRSmaCreate(SSma* pSma, SVCreateStbReq* pReq);
 int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType);

source/dnode/vnode/src/meta/metaQuery.c

@@ -98,6 +98,17 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) {
   return uid;
 }
 
+int metaGetTableNameByUid(void* meta, uint64_t uid, char* tbName) {
+  SMetaReader mr = {0};
+  metaReaderInit(&mr, (SMeta*)meta, 0);
+  metaGetTableEntryByUid(&mr, uid);
+
+  STR_TO_VARSTR(tbName, mr.me.name);
+  metaReaderClear(&mr);
+
+  return 0;
+}
+
 int metaReadNext(SMetaReader *pReader) {
   SMeta *pMeta = pReader->pMeta;
 

source/dnode/vnode/src/meta/metaTable.c

@@ -438,12 +438,15 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
 }
 
 int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) {
-  metaWLock(pMeta);
   int ret = metaTtlSmaller(pMeta, ttl, tbUids);
   if (ret != 0) {
-    metaULock(pMeta);
     return ret;
   }
+  if (taosArrayGetSize(tbUids) == 0){
+    return 0;
+  }
+
+  metaWLock(pMeta);
   for (int i = 0; i < taosArrayGetSize(tbUids); ++i) {
     tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i);
     metaDropTableByUid(pMeta, *uid, NULL);
@@ -637,6 +640,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
         terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
         goto _err;
       }
+      if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) {
+        terrno = TSDB_CODE_VND_COL_SUBSCRIBED;
+        goto _err;
+      }
       pSchema->version++;
       tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema);
       if (tlen) {
@@ -653,6 +660,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
         terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
         goto _err;
       }
+      if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) {
+        terrno = TSDB_CODE_VND_COL_SUBSCRIBED;
+        goto _err;
+      }
       pSchema->version++;
       pColumn->bytes = pAlterTbReq->colModBytes;
       break;
@@ -661,6 +672,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
         terrno = TSDB_CODE_VND_TABLE_COL_NOT_EXISTS;
         goto _err;
       }
+      if (tqCheckColModifiable(pMeta->pVnode->pTq, uid, pColumn->colId) != 0) {
+        terrno = TSDB_CODE_VND_COL_SUBSCRIBED;
+        goto _err;
+      }
       pSchema->version++;
       strcpy(pColumn->name, pAlterTbReq->colNewName);
       break;

source/dnode/vnode/src/sma/smaCommit.c

@@ -146,7 +146,6 @@ static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma) {
 
   // step 3: perform persist task for qTaskInfo
   pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied;
-  pRSmaStat->commitSubmitVer = pRSmaStat->submitVer;
   tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat));
 
   smaDebug("vgId:%d, rsma pre commit success", SMA_VID(pSma));
@@ -317,7 +316,6 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) {
 
   // step 4: others
   pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied;
-  pRSmaStat->commitSubmitVer = pRSmaStat->submitVer;
 
   return TSDB_CODE_SUCCESS;
 }

source/dnode/vnode/src/sma/smaRollup.c

@@ -560,17 +560,6 @@ static void tdDestroySDataBlockArray(SArray *pArray) {
   taosArrayDestroy(pArray);
 }
 
-int64_t tdRSmaGetMaxSubmitVer(SSma *pSma, int8_t level) {
-  if (level == TSDB_RETENTION_L0) {
-    return pSma->pVnode->state.applied;
-  }
-
-  SSmaEnv   *pRSmaEnv = SMA_RSMA_ENV(pSma);
-  SRSmaStat *pRSmaStat = (SRSmaStat *)(SMA_ENV_STAT(pRSmaEnv));
-
-  return atomic_load_64(&pRSmaStat->submitVer);
-}
-
 static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat,
                                           int8_t blkType) {
   SArray *pResult = NULL;
@@ -615,13 +604,16 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche
         goto _err;
       }
 
-      if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pStat->submitVer, 1), pReq) < 0) {
+      if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) {
         taosMemoryFreeClear(pReq);
         smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s",
                  SMA_VID(pSma), suid, pItem->level, terrstr());
         goto _err;
       }
 
+      smaDebug("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " version:%"PRIi64, SMA_VID(pSma),
+               suid, pItem->level, output->info.version);
+
       taosMemoryFreeClear(pReq);
       taosArrayClear(pResult);
     } else if (terrno == 0) {
@@ -908,12 +900,8 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed) {
     goto _err;
   }
 
-  ASSERT(tFileInfo.qTaskInfo.submitVer > 0);
-
   SSmaEnv   *pRSmaEnv = pSma->pRSmaEnv;
   SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pRSmaEnv);
-  atomic_store_64(&pRSmaStat->submitVer, tFileInfo.qTaskInfo.submitVer);
-  smaDebug("%s:%d tFileInfo.qTaskInfo.submitVer = %" PRIi64, __func__, __LINE__, tFileInfo.qTaskInfo.submitVer);
 
   SRSmaQTaskInfoIter fIter = {0};
   if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
@@ -1266,7 +1254,6 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
   }
 
   if (isFileCreated) {
-    tFile.info.qTaskInfo.submitVer = atomic_load_64(&pRSmaStat->commitSubmitVer);
     if (tdUpdateTFileHeader(&tFile) < 0) {
       smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile),
                tstrerror(terrno));
@@ -1346,6 +1333,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
       tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK);
 
       tdUnRefRSmaInfo(pSma, pRSmaInfo);
+      // atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
+      // taosTmrReset(tdRSmaFetchTrigger, 5000, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
     } break;
     case TASK_TRIGGER_STAT_PAUSED: {
       smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",

source/dnode/vnode/src/sma/smaUtil.c

@@ -32,9 +32,6 @@ static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) {
   tlen += taosEncodeFixedU32(buf, pInfo->ftype);
   tlen += taosEncodeFixedU32(buf, pInfo->fver);
   tlen += taosEncodeFixedI64(buf, pInfo->fsize);
-  if (pInfo->ftype == TD_FTYPE_RSMA_QTASKINFO) {
-    tlen += taosEncodeFixedI64(buf, pInfo->qTaskInfo.submitVer);
-  }
 
   return tlen;
 }
@@ -44,10 +41,6 @@ static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) {
   buf = taosDecodeFixedU32(buf, &(pInfo->ftype));
   buf = taosDecodeFixedU32(buf, &(pInfo->fver));
   buf = taosDecodeFixedI64(buf, &(pInfo->fsize));
-  // specific
-  if (pInfo->ftype == TD_FTYPE_RSMA_QTASKINFO) {
-    buf = taosDecodeFixedI64(buf, &(pInfo->qTaskInfo.submitVer));
-  }
 
   return buf;
 }

source/dnode/vnode/src/tq/tq.c

@@ -51,7 +51,7 @@ void tqCleanUp() {
   }
 }
 
-STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
+STQ* tqOpen(const char* path, SVnode* pVnode) {
   STQ* pTq = taosMemoryCalloc(1, sizeof(STQ));
   if (pTq == NULL) {
     terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
@@ -59,7 +59,6 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
   }
   pTq->path = strdup(path);
   pTq->pVnode = pVnode;
-  pTq->pWal = pWal;
 
   pTq->handles = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
 
@@ -67,6 +66,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
 
   pTq->pushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
 
+  pTq->pAlterInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
+
   if (tqMetaOpen(pTq) < 0) {
     ASSERT(0);
   }
@@ -91,6 +92,7 @@ void tqClose(STQ* pTq) {
     }
     taosHashCleanup(pTq->pStreamTasks);
     taosHashCleanup(pTq->pushMgr);
+    taosHashCleanup(pTq->pAlterInfo);
     taosMemoryFree(pTq->path);
     tqMetaClose(pTq);
     taosMemoryFree(pTq);
@@ -208,18 +210,18 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
   return 0;
 }
 
-int32_t tqCheckColModifiable(STQ* pTq, int32_t colId) {
+int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) {
   void* pIter = NULL;
   while (1) {
-    pIter = taosHashIterate(pTq->handles, pIter);
+    pIter = taosHashIterate(pTq->pAlterInfo, pIter);
     if (pIter == NULL) break;
-    STqHandle* pExec = (STqHandle*)pIter;
+    SCheckAlterInfo* pCheck = (SCheckAlterInfo*)pIter;
-    if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
+    if (pCheck->ntbUid == tbUid) {
-      int32_t sz = pExec->execHandle.pSchemaWrapper->nCols;
+      int32_t sz = taosArrayGetSize(pCheck->colIdList);
       for (int32_t i = 0; i < sz; i++) {
-        SSchema* pSchema = &pExec->execHandle.pSchemaWrapper->pSchema[i];
+        int16_t forbidColId = *(int16_t*)taosArrayGet(pCheck->colIdList, i);
-        if (pSchema->colId == colId) {
+        if (forbidColId == colId) {
-          taosHashCancelIterate(pTq->handles, pIter);
+          taosHashCancelIterate(pTq->pAlterInfo, pIter);
           return -1;
         }
       }
@@ -270,6 +272,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   int32_t      code = 0;
   STqOffsetVal reqOffset = pReq->reqOffset;
   STqOffsetVal fetchOffsetNew;
+  SWalCkHead*  pCkHead = NULL;
 
   // 1.find handle
   STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey));
@@ -459,6 +462,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   }
 
 OVER:
+  if (pCkHead) taosMemoryFree(pCkHead);
   // TODO wrap in destroy func
   taosArrayDestroy(dataRsp.blockDataLen);
   taosArrayDestroyP(dataRsp.blockData, (FDelete)taosMemoryFree);
@@ -488,6 +492,22 @@ int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) {
   return 0;
 }
 
+int32_t tqProcessCheckAlterInfoReq(STQ* pTq, char* msg, int32_t msgLen) {
+  SCheckAlterInfo info = {0};
+  SDecoder        decoder;
+  tDecoderInit(&decoder, msg, msgLen);
+  if (tDecodeSCheckAlterInfo(&decoder, &info) < 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  tDecoderClear(&decoder);
+  if (taosHashPut(pTq->pAlterInfo, info.topic, strlen(info.topic), &info, sizeof(SCheckAlterInfo)) < 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  return 0;
+}
+
 int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
   SMqRebVgReq req = {0};
   tDecodeSMqRebVgReq(msg, &req);
@@ -549,8 +569,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
       taosArrayDestroy(tbUidList);
     }
     taosHashPut(pTq->handles, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle));
+    tqDebug("try to persist handle %s consumer %ld", req.subKey, pHandle->consumerId);
     if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
       // TODO
+      ASSERT(0);
     }
   } else {
     /*ASSERT(pExec->consumerId == req.oldConsumerId);*/

source/dnode/vnode/src/tq/tqExec.c

@@ -201,10 +201,12 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
       if (pRsp->withTbName) {
         int64_t uid = pExec->pExecReader->msgIter.uid;
         if (tqAddTbNameToRsp(pTq, uid, pRsp) < 0) {
+          blockDataFreeRes(&block);
           continue;
         }
       }
       tqAddBlockDataToRsp(&block, pRsp, taosArrayGetSize(block.pDataBlock));
+      blockDataFreeRes(&block);
       tqAddBlockSchemaToRsp(pExec, pRsp);
       pRsp->blockNum++;
     }
@@ -220,10 +222,12 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR
       if (pRsp->withTbName) {
         int64_t uid = pExec->pExecReader->msgIter.uid;
         if (tqAddTbNameToRsp(pTq, uid, pRsp) < 0) {
+          blockDataFreeRes(&block);
           continue;
         }
       }
       tqAddBlockDataToRsp(&block, pRsp, taosArrayGetSize(block.pDataBlock));
+      blockDataFreeRes(&block);
       tqAddBlockSchemaToRsp(pExec, pRsp);
       pRsp->blockNum++;
     }

source/dnode/vnode/src/tq/tqMeta.c

@@ -89,8 +89,8 @@ int32_t tqMetaOpen(STQ* pTq) {
           .version = handle.snapshotVer,
       };
 
-      handle.execHandle.execCol.task =
+      handle.execHandle.execCol.task = qCreateQueueExecTaskInfo(
-          qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, &handle.execHandle.numOfCols, &handle.execHandle.pSchemaWrapper);
+          handle.execHandle.execCol.qmsg, &reader, &handle.execHandle.numOfCols, &handle.execHandle.pSchemaWrapper);
       ASSERT(handle.execHandle.execCol.task);
       void* scanner = NULL;
       qExtractStreamScanner(handle.execHandle.execCol.task, &scanner);
@@ -101,6 +101,7 @@ int32_t tqMetaOpen(STQ* pTq) {
       handle.execHandle.execDb.pFilterOutTbUid =
           taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
     }
+    tqDebug("tq restore %s consumer %ld", handle.subKey, handle.consumerId);
     taosHashPut(pTq->handles, pKey, kLen, &handle, sizeof(STqHandle));
   }
 

source/dnode/vnode/src/tq/tqRead.c

@@ -340,29 +340,30 @@ FAIL:
 
 void tqReaderSetColIdList(STqReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; }
 
-int tqReaderSetTbUidList(STqReader* pHandle, const SArray* tbUidList) {
+int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) {
-  if (pHandle->tbIdHash) {
+  if (pReader->tbIdHash) {
-    taosHashClear(pHandle->tbIdHash);
+    taosHashClear(pReader->tbIdHash);
+  } else {
+    pReader->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
   }
 
-  pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+  if (pReader->tbIdHash == NULL) {
-  if (pHandle->tbIdHash == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 
   for (int i = 0; i < taosArrayGetSize(tbUidList); i++) {
     int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
-    taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
+    taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
   }
 
   return 0;
 }
 
-int tqReaderAddTbUidList(STqReader* pHandle, const SArray* tbUidList) {
+int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) {
-  if (pHandle->tbIdHash == NULL) {
+  if (pReader->tbIdHash == NULL) {
-    pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+    pReader->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
-    if (pHandle->tbIdHash == NULL) {
+    if (pReader->tbIdHash == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
     }
@@ -370,18 +371,18 @@ int tqReaderAddTbUidList(STqReader* pHandle, const SArray* tbUidList) {
 
   for (int i = 0; i < taosArrayGetSize(tbUidList); i++) {
     int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
-    taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
+    taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0);
   }
 
   return 0;
 }
 
-int tqReaderRemoveTbUidList(STqReader* pHandle, const SArray* tbUidList) {
+int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) {
-  ASSERT(pHandle->tbIdHash != NULL);
+  ASSERT(pReader->tbIdHash != NULL);
 
   for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
     int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
-    taosHashRemove(pHandle->tbIdHash, pKey, sizeof(int64_t));
+    taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t));
   }
 
   return 0;

source/dnode/vnode/src/tq/tqSink.c

@@ -110,7 +110,6 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo
   // TODO
   ret = rpcMallocCont(cap);
   ret->header.vgId = vgId;
-  ret->version = htonl(1);
   ret->length = sizeof(SSubmitReq);
   ret->numOfBlocks = htonl(sz);
 

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -2265,10 +2265,6 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret
 SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) {
   int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion;
 
-  if (VND_IS_RSMA(pVnode)) {
-    return (SVersionRange){.minVer = startVer, .maxVer = tdRSmaGetMaxSubmitVer(pVnode->pSma, level)};
-  }
-
   int64_t endVer = 0;
   if (pCond->endVersion ==
       -1) {  // user not specified end version, set current maximum version of vnode as the endVersion

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -95,16 +95,19 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
   // save info to a vnode_tmp.json
   pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
+    vError("failed to open info file: %s for write: %s", fname, terrstr());
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
   if (taosWriteFile(pFile, data, strlen(data)) < 0) {
+    vError("failed to write info file: %s data: %s", fname, terrstr());
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
 
   if (taosFsyncFile(pFile) < 0) {
+    vError("failed to fsync info file: %s error: %s", fname, terrstr());
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -135,7 +135,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   // open tq
   sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR);
   taosRealPath(tdir, NULL, sizeof(tdir));
-  pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal);
+  pVnode->pTq = tqOpen(tdir, pVnode);
   if (pVnode->pTq == NULL) {
     vError("vgId:%d, failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -112,8 +112,8 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
       tEncodeSize(tEncodeDeleteRes, &res, size, ret);
       pCont = rpcMallocCont(size + sizeof(SMsgHead));
 
-      ((SMsgHead *)pCont)->contLen = htonl(size + sizeof(SMsgHead));
+      ((SMsgHead *)pCont)->contLen = size + sizeof(SMsgHead);
-      ((SMsgHead *)pCont)->vgId = htonl(TD_VID(pVnode));
+      ((SMsgHead *)pCont)->vgId = TD_VID(pVnode);
 
       tEncoderInit(pCoder, pCont + sizeof(SMsgHead), size);
       tEncodeDeleteRes(pCoder, &res);
@@ -206,6 +206,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
         goto _err;
       }
       break;
+    case TDMT_VND_CHECK_ALTER_INFO:
+      if (tqProcessCheckAlterInfoReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
+                                     pMsg->contLen - sizeof(SMsgHead)) < 0) {
+        goto _err;
+      }
+      break;
     case TDMT_STREAM_TASK_DEPLOY: {
       if (tqProcessTaskDeployReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
                                  pMsg->contLen - sizeof(SMsgHead)) < 0) {

source/libs/executor/inc/executil.h

@@ -108,6 +108,9 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode);
 
 EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
 int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo);
+int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId);
+size_t  getTableTagsBufLen(const SNodeList* pGroups);
+
 SArray*  createSortInfo(SNodeList* pNodeList);
 SArray*  extractPartitionColInfo(SNodeList* pNodeList);
 SArray*  extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
@@ -129,6 +132,6 @@ int32_t convertFillType(int32_t mode);
 
 int32_t resultrowComparAsc(const void* p1, const void* p2);
 
-int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified);
+int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified);
 
 #endif  // TDENGINE_QUERYUTIL_H

source/libs/executor/inc/executorimpl.h

@@ -55,8 +55,9 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int
 #define START_TS_COLUMN_INDEX             0
 #define END_TS_COLUMN_INDEX               1
 #define UID_COLUMN_INDEX                  2
-#define GROUPID_COLUMN_INDEX        UID_COLUMN_INDEX
+#define GROUPID_COLUMN_INDEX              3
-#define DELETE_GROUPID_COLUMN_INDEX 2
+#define CALCULATE_START_TS_COLUMN_INDEX   4
+#define CALCULATE_END_TS_COLUMN_INDEX     5
 
 enum {
   // when this task starts to execute, this status will set
@@ -175,6 +176,7 @@ typedef struct SExecTaskInfo {
   int64_t          owner;  // if it is in execution
   int32_t          code;
 
+  int64_t          version; // used for stream to record wal version
   SStreamTaskInfo  streamInfo;
   SSchemaInfo      schemaInfo;
   STableListInfo   tableqinfoList;  // this is a table list
@@ -346,7 +348,6 @@ typedef enum EStreamScanMode {
   STREAM_SCAN_FROM_READERHANDLE = 1,
   STREAM_SCAN_FROM_RES,
   STREAM_SCAN_FROM_UPDATERES,
-  STREAM_SCAN_FROM_DATAREADER, // todo(liuyao) delete it
   STREAM_SCAN_FROM_DATAREADER_RETRIEVE,
   STREAM_SCAN_FROM_DATAREADER_RANGE,
 } EStreamScanMode;
@@ -366,7 +367,7 @@ typedef struct SStreamAggSupporter {
   char*          pKeyBuf;              // window key buffer
   SDiskbasedBuf* pResultBuf;           // query result buffer based on blocked-wised disk file
   int32_t        resultRowSize;        // the result buffer size for each result row, with the meta data size for each row
-  SArray*        pScanWindow;
+  SSDataBlock*   pScanBlock;
 } SStreamAggSupporter;
 
 typedef struct SessionWindowSupporter {
@@ -401,8 +402,6 @@ typedef struct SStreamScanInfo {
   uint64_t     numOfExec;        // execution times
   STqReader*   tqReader;
 
-  int32_t      tsArrayIndex;
-  SArray*      tsArray;
   uint64_t     groupId;
   SUpdateInfo* pUpdateInfo;
 
@@ -419,10 +418,11 @@ typedef struct SStreamScanInfo {
   int32_t                deleteDataIndex;
   STimeWindow            updateWin;
   STimeWindowAggSupp     twAggSup;
-
+  SSDataBlock*           pUpdateDataRes;
   // status for tmq
   // SSchemaWrapper schema;
   STqOffset              offset;
+  SNodeList*             pGroupTags;
   SNode*                 pTagCond;
   SNode*                 pTagIndexCond;
 } SStreamScanInfo;
@@ -545,9 +545,10 @@ typedef struct SProjectOperatorInfo {
   SOptrBasicInfo     binfo;
   SAggSupporter      aggSup;
   SNode*             pFilterNode;  // filter info, which is push down by optimizer
-  SSDataBlock*       existDataBlock;
   SArray*            pPseudoColInfo;
   SLimitInfo         limitInfo;
+  bool               mergeDataBlocks;
+  SSDataBlock*       pFinalRes;
 } SProjectOperatorInfo;
 
 typedef struct SIndefOperatorInfo {
@@ -712,7 +713,6 @@ typedef struct SStreamStateAggOperatorInfo {
   SSDataBlock*         pDelRes;
   SHashObj*            pSeDeleted;
   void*                pDelIterator;
-  SArray*              pScanWindow;
   SArray*              pChildren;       // cache for children's result;
   bool                 ignoreExpiredData;
 } SStreamStateAggOperatorInfo;
@@ -805,7 +805,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scan
 int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
 
 void    doSetOperatorCompleted(SOperatorInfo* pOperator);
-void    doFilter(const SNode* pFilterNode, SSDataBlock* pBlock);
+void    doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo);
 int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr,
                                SSDataBlock* pBlock, const char* idStr);
 
@@ -869,7 +869,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* re
                                                SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
-                                            STimeWindowAggSupp* pTwAggSup, SExecTaskInfo* pTaskInfo);
+                                            SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo);
 
@@ -879,8 +879,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf
 
 SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
 
-SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, /*SExprInfo* pExprInfo, int32_t numOfCols,
+SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
-                                           SSDataBlock* pResultBlock, const SNodeListNode* pValNode, */SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SJoinPhysiNode* pJoinNode,
                                            SExecTaskInfo* pTaskInfo);
@@ -954,6 +953,7 @@ int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs,
     TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted);
 bool functionNeedToExecute(SqlFunctionCtx* pCtx);
 bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup);
+void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid);
 
 int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition,
                                        SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset,
@@ -970,7 +970,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
 void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
 
 int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey);
-SSDataBlock* createPullDataBlock();
+SSDataBlock* createSpecialDataBlock(EStreamType type);
 
 #ifdef __cplusplus
 }

source/libs/executor/src/executil.c

@@ -265,7 +265,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
   return DEAL_RES_CONTINUE;
 }
 
-int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) {
+int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) {
   int32_t     code = TSDB_CODE_SUCCESS;
   SMetaReader mr = {0};
 
@@ -356,7 +356,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
       STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i);
 
       bool qualified = true;
-      code = isTableOk(info, pTagCond, metaHandle, &qualified);
+      code = isQualifiedTable(info, pTagCond, metaHandle, &qualified);
       if (code != TSDB_CODE_SUCCESS) {
         return code;
       }
@@ -379,6 +379,82 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
   return code;
 }
 
+size_t getTableTagsBufLen(const SNodeList* pGroups) {
+  size_t keyLen = 0;
+
+  SNode* node;
+  FOREACH(node, pGroups) {
+    SExprNode* pExpr = (SExprNode*)node;
+    keyLen += pExpr->resType.bytes;
+  }
+
+  keyLen += sizeof(int8_t) * LIST_LENGTH(pGroups);
+  return keyLen;
+}
+
+int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) {
+  SMetaReader    mr = {0};
+  metaReaderInit(&mr, pMeta, 0);
+  metaGetTableEntryByUid(&mr, uid);
+
+  SNodeList* groupNew = nodesCloneList(pGroupNode);
+
+  nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr);
+  char* isNull = (char*)keyBuf;
+  char* pStart = (char*)keyBuf + sizeof(int8_t)*LIST_LENGTH(pGroupNode);
+
+  SNode*  pNode;
+  int32_t index = 0;
+  FOREACH(pNode, groupNew) {
+    SNode*  pNew = NULL;
+    int32_t code = scalarCalculateConstants(pNode, &pNew);
+    if (TSDB_CODE_SUCCESS == code) {
+      REPLACE_NODE(pNew);
+    } else {
+      taosMemoryFree(keyBuf);
+      nodesDestroyList(groupNew);
+      metaReaderClear(&mr);
+      return code;
+    }
+
+    ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
+    SValueNode* pValue = (SValueNode*)pNew;
+
+    if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL || pValue->isNull) {
+      isNull[index++] = 1;
+      continue;
+    } else {
+      isNull[index++] = 0;
+      char* data = nodesGetValueFromNode(pValue);
+      if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) {
+        if (tTagIsJson(data)) {
+          terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
+          taosMemoryFree(keyBuf);
+          nodesDestroyList(groupNew);
+          metaReaderClear(&mr);
+          return terrno;
+        }
+        int32_t len = getJsonValueLen(data);
+        memcpy(pStart, data, len);
+        pStart += len;
+      } else if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) {
+        memcpy(pStart, data, varDataTLen(data));
+        pStart += varDataTLen(data);
+      } else {
+        memcpy(pStart, data, pValue->node.resType.bytes);
+        pStart += pValue->node.resType.bytes;
+      }
+    }
+  }
+
+  int32_t  len = (int32_t)(pStart - (char*)keyBuf);
+  *pGroupId = calcGroupId(keyBuf, len);
+
+  nodesDestroyList(groupNew);
+  metaReaderClear(&mr);
+  return TSDB_CODE_SUCCESS;
+}
+
 SArray* extractPartitionColInfo(SNodeList* pNodeList) {
   if (!pNodeList) {
     return NULL;

source/libs/executor/src/executor.c

@@ -14,10 +14,21 @@
  */
 
 #include "executor.h"
+#include "tref.h"
 #include "executorimpl.h"
 #include "planner.h"
 #include "tdatablock.h"
 #include "vnode.h"
+#include "tudf.h"
+
+static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT;
+int32_t             exchangeObjRefPool = -1;
+
+static void initRefPool() { exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo); }
+static void cleanupRefPool() {
+  int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0);
+  taosCloseRef(ref);
+}
 
 static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, bool assignUid,
                                 char* id) {
@@ -120,8 +131,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO
   return code;
 }
 
-qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols,
+qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema) {
-                                     SSchemaWrapper** pSchemaWrapper) {
   if (msg == NULL) {
     // TODO create raw scan
     return NULL;
@@ -155,7 +165,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n
     }
   }
 
-  *pSchemaWrapper = tCloneSSchemaWrapper(((SExecTaskInfo*)pTaskInfo)->schemaInfo.qsw);
+  *pSchema = tCloneSSchemaWrapper(((SExecTaskInfo*)pTaskInfo)->schemaInfo.qsw);
   return pTaskInfo;
 }
 
@@ -185,8 +195,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) {
   return pTaskInfo;
 }
 
-static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList,
+static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) {
-                                          const char* idstr) {
   SArray* qa = taosArrayInit(4, sizeof(tb_uid_t));
 
   // let's discard the tables those are not created according to the queried super table.
@@ -209,7 +218,7 @@ static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, cons
     if (pScanInfo->pTagCond != NULL) {
       bool          qualified = false;
       STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid};
-      code = isTableOk(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified);
+      code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified);
       if (code != TSDB_CODE_SUCCESS) {
         qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr);
         continue;
@@ -240,7 +249,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
   int32_t          code = 0;
   SStreamScanInfo* pScanInfo = pInfo->info;
   if (isAdd) {  // add new table id
-    SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo));
+    SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo));
 
     qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa));
     code = tqReaderAddTbUidList(pScanInfo->tqReader, qa);
@@ -248,17 +257,35 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo
       return code;
     }
 
-    // add to qTaskInfo
     // todo refactor STableList
+    size_t bufLen = (pScanInfo->pGroupTags != NULL)? getTableTagsBufLen(pScanInfo->pGroupTags):0;
+    char* keyBuf = NULL;
+    if (bufLen > 0) {
+      keyBuf = taosMemoryMalloc(bufLen);
+      if (keyBuf == NULL) {
+        return TSDB_CODE_OUT_OF_MEMORY;
+      }
+    }
+
     for(int32_t i = 0; i < taosArrayGetSize(qa); ++i) {
       uint64_t*     uid = taosArrayGet(qa, i);
-
-      qDebug("table %ld added to task info", *uid);
-
       STableKeyInfo keyInfo = {.uid = *uid, .groupId = 0};
+
+      if (bufLen > 0) {
+        code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf,
+                                       &keyInfo.groupId);
+        if (code != TSDB_CODE_SUCCESS) {
+          return code;
+        }
+      }
+
       taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo);
     }
 
+    if (keyBuf != NULL) {
+      taosMemoryFree(keyBuf);
+    }
+
     taosArrayDestroy(qa);
   } else {  // remove the table id in current list
     qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList));
@@ -292,3 +319,396 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table
 
   return 0;
 }
+
+int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan,
+                        qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model) {
+  assert(pSubplan != NULL);
+  SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo;
+
+  taosThreadOnce(&initPoolOnce, initRefPool);
+  atexit(cleanupRefPool);
+
+  int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100};
+  code = dsDataSinkMgtInit(&cfg);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  if (handle) {
+    void* pSinkParam = NULL;
+    code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _error;
+    }
+
+    code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam);
+  }
+
+  _error:
+  // if failed to add ref for all tables in this query, abort current query
+  return code;
+}
+
+#ifdef TEST_IMPL
+// wait moment
+int waitMoment(SQInfo* pQInfo) {
+  if (pQInfo->sql) {
+    int   ms = 0;
+    char* pcnt = strstr(pQInfo->sql, " count(*)");
+    if (pcnt) return 0;
+
+    char* pos = strstr(pQInfo->sql, " t_");
+    if (pos) {
+      pos += 3;
+      ms = atoi(pos);
+      while (*pos >= '0' && *pos <= '9') {
+        pos++;
+      }
+      char unit_char = *pos;
+      if (unit_char == 'h') {
+        ms *= 3600 * 1000;
+      } else if (unit_char == 'm') {
+        ms *= 60 * 1000;
+      } else if (unit_char == 's') {
+        ms *= 1000;
+      }
+    }
+    if (ms == 0) return 0;
+    printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql);
+
+    if (ms < 1000) {
+      taosMsleep(ms);
+    } else {
+      int used_ms = 0;
+      while (used_ms < ms) {
+        taosMsleep(1000);
+        used_ms += 1000;
+        if (isTaskKilled(pQInfo)) {
+          printf("test check query is canceled, sleep break.%s\n", pQInfo->sql);
+          break;
+        }
+      }
+    }
+  }
+  return 1;
+}
+#endif
+
+int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  int64_t        threadId = taosGetSelfPthreadId();
+
+  *pRes = NULL;
+  int64_t curOwner = 0;
+  if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) {
+    qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner);
+    pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC;
+    return pTaskInfo->code;
+  }
+
+  if (pTaskInfo->cost.start == 0) {
+    pTaskInfo->cost.start = taosGetTimestampMs();
+  }
+
+  if (isTaskKilled(pTaskInfo)) {
+    qDebug("%s already killed, abort", GET_TASKID(pTaskInfo));
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // error occurs, record the error code and return to client
+  int32_t ret = setjmp(pTaskInfo->env);
+  if (ret != TSDB_CODE_SUCCESS) {
+    pTaskInfo->code = ret;
+    cleanUpUdfs();
+    qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
+    atomic_store_64(&pTaskInfo->owner, 0);
+
+    return pTaskInfo->code;
+  }
+
+  qDebug("%s execTask is launched", GET_TASKID(pTaskInfo));
+
+  int64_t st = taosGetTimestampUs();
+
+  *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot);
+  uint64_t el = (taosGetTimestampUs() - st);
+
+  pTaskInfo->cost.elapsedTime += el;
+  if (NULL == *pRes) {
+    *useconds = pTaskInfo->cost.elapsedTime;
+  }
+
+  cleanUpUdfs();
+
+  int32_t  current = (*pRes != NULL) ? (*pRes)->info.rows : 0;
+  uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows;
+
+  qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms",
+         GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0);
+
+  atomic_store_64(&pTaskInfo->owner, 0);
+  return pTaskInfo->code;
+}
+
+int32_t qKillTask(qTaskInfo_t qinfo) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
+  if (pTaskInfo == NULL) {
+    return TSDB_CODE_QRY_INVALID_QHANDLE;
+  }
+
+  qAsyncKillTask(qinfo);
+
+  // Wait for the query executing thread being stopped/
+  // Once the query is stopped, the owner of qHandle will be cleared immediately.
+  while (pTaskInfo->owner != 0) {
+    taosMsleep(100);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
+
+  if (pTaskInfo == NULL) {
+    return TSDB_CODE_QRY_INVALID_QHANDLE;
+  }
+
+  qDebug("%s execTask async killed", GET_TASKID(pTaskInfo));
+  setTaskKilled(pTaskInfo);
+  return TSDB_CODE_SUCCESS;
+}
+
+void qDestroyTask(qTaskInfo_t qTaskHandle) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle;
+  if (pTaskInfo == NULL) {
+    return;
+  }
+
+  qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows);
+
+  queryCostStatis(pTaskInfo);  // print the query cost summary
+  doDestroyTask(pTaskInfo);
+}
+
+int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  int32_t        capacity = 0;
+
+  return getOperatorExplainExecInfo(pTaskInfo->pRoot, pRes, &capacity, resNum);
+}
+
+int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) {
+  SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo;
+  if (pTaskInfo->pRoot == NULL) {
+    return TSDB_CODE_INVALID_PARA;
+  }
+
+  int32_t nOptrWithVal = 0;
+  int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
+  if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) {
+    taosMemoryFreeClear(*pOutput);
+    *len = 0;
+  }
+  return code;
+}
+
+int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) {
+  SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo;
+
+  if (pTaskInfo == NULL || pInput == NULL || len == 0) {
+    return TSDB_CODE_INVALID_PARA;
+  }
+
+  return decodeOperator(pTaskInfo->pRoot, pInput, len);
+}
+
+int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  SOperatorInfo* pOperator = pTaskInfo->pRoot;
+
+  while (1) {
+    uint8_t type = pOperator->operatorType;
+    if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
+      *scanner = pOperator->info;
+      return 0;
+    } else {
+      ASSERT(pOperator->numOfDownstream == 1);
+      pOperator = pOperator->pDownstream[0];
+    }
+  }
+}
+
+#if 0
+int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM);
+  taosWriteQitem(pTaskInfo->streamInfo.inputQueue->queue, pItem);
+  return 0;
+}
+#endif
+
+int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM);
+  pTaskInfo->streamInfo.recoverStartVer = startVer;
+  pTaskInfo->streamInfo.recoverEndVer = endVer;
+  pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE;
+  return 0;
+}
+
+void* qExtractReaderFromStreamScanner(void* scanner) {
+  SStreamScanInfo* pInfo = scanner;
+  return (void*)pInfo->tqReader;
+}
+
+const SSchemaWrapper* qExtractSchemaFromStreamScanner(void* scanner) {
+  SStreamScanInfo* pInfo = scanner;
+  return pInfo->tqReader->pSchemaWrapper;
+}
+
+const STqOffset* qExtractStatusFromStreamScanner(void* scanner) {
+  SStreamScanInfo* pInfo = scanner;
+  return &pInfo->offset;
+}
+
+void* qStreamExtractMetaMsg(qTaskInfo_t tinfo) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
+  return pTaskInfo->streamInfo.metaBlk;
+}
+
+int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
+  memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal));
+  return 0;
+}
+
+int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+  SOperatorInfo* pOperator = pTaskInfo->pRoot;
+  ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
+  pTaskInfo->streamInfo.prepareStatus = *pOffset;
+  if (!tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) {
+    while (1) {
+      uint8_t type = pOperator->operatorType;
+      pOperator->status = OP_OPENED;
+      if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
+        SStreamScanInfo* pInfo = pOperator->info;
+        if (pOffset->type == TMQ_OFFSET__LOG) {
+          STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
+          tsdbReaderClose(pTSInfo->dataReader);
+          pTSInfo->dataReader = NULL;
+#if 0
+          if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) &&
+              pInfo->tqReader->pWalReader->curVersion != pOffset->version) {
+            qError("prepare scan ver %ld actual ver %ld, last %ld", pOffset->version,
+                   pInfo->tqReader->pWalReader->curVersion, pTaskInfo->streamInfo.lastStatus.version);
+            ASSERT(0);
+          }
+#endif
+          if (tqSeekVer(pInfo->tqReader, pOffset->version + 1) < 0) {
+            return -1;
+          }
+          ASSERT(pInfo->tqReader->pWalReader->curVersion == pOffset->version + 1);
+        } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+          /*pInfo->blockType = STREAM_INPUT__TABLE_SCAN;*/
+          int64_t uid = pOffset->uid;
+          int64_t ts = pOffset->ts;
+
+          if (uid == 0) {
+            if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) {
+              STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0);
+              uid = pTableInfo->uid;
+              ts = INT64_MIN;
+            } else {
+              return -1;
+            }
+          }
+
+          /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/
+          /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/
+          STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
+          int32_t         tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
+
+#ifndef NDEBUG
+
+          qDebug("switch to next table %ld (cursor %d), %ld rows returned", uid, pTableScanInfo->currentTable,
+                 pInfo->pTableScanOp->resultInfo.totalRows);
+          pInfo->pTableScanOp->resultInfo.totalRows = 0;
+#endif
+
+          bool found = false;
+          for (int32_t i = 0; i < tableSz; i++) {
+            STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i);
+            if (pTableInfo->uid == uid) {
+              found = true;
+              pTableScanInfo->currentTable = i;
+              break;
+            }
+          }
+
+          // TODO after dropping table, table may be not found
+          ASSERT(found);
+
+          if (pTableScanInfo->dataReader == NULL) {
+            if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond,
+                               pTaskInfo->tableqinfoList.pTableList, &pTableScanInfo->dataReader, NULL) < 0 ||
+                pTableScanInfo->dataReader == NULL) {
+              ASSERT(0);
+            }
+          }
+
+          tsdbSetTableId(pTableScanInfo->dataReader, uid);
+          int64_t oldSkey = pTableScanInfo->cond.twindows.skey;
+          pTableScanInfo->cond.twindows.skey = ts + 1;
+          tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
+          pTableScanInfo->cond.twindows.skey = oldSkey;
+          pTableScanInfo->scanTimes = 0;
+
+          qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts,
+                 pTableScanInfo->currentTable, tableSz);
+          /*}*/
+
+        } else {
+          ASSERT(0);
+        }
+        return 0;
+      } else {
+        ASSERT(pOperator->numOfDownstream == 1);
+        pOperator = pOperator->pDownstream[0];
+      }
+    }
+  }
+  return 0;
+}
+
+
+#if 0
+int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+
+  if (uid == 0) {
+    if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) {
+      STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0);
+      uid = pTableInfo->uid;
+      ts = INT64_MIN;
+    }
+  }
+
+  return doPrepareScan(pTaskInfo->pRoot, uid, ts);
+}
+
+int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) {
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
+
+  return doGetScanStatus(pTaskInfo->pRoot, uid, ts);
+}
+#endif
+

source/libs/executor/src/executorMain.c

@@ -1,425 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "dataSinkMgt.h"
-#include "os.h"
-#include "tmsg.h"
-#include "tref.h"
-#include "tudf.h"
-
-#include "executor.h"
-#include "executorimpl.h"
-#include "query.h"
-
-static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT;
-int32_t             exchangeObjRefPool = -1;
-
-static void initRefPool() { exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo); }
-static void cleanupRefPool() {
-  int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0);
-  taosCloseRef(ref);
-}
-
-int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan,
-                        qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model) {
-  assert(pSubplan != NULL);
-  SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo;
-
-  taosThreadOnce(&initPoolOnce, initRefPool);
-  atexit(cleanupRefPool);
-
-  int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100};
-  code = dsDataSinkMgtInit(&cfg);
-  if (code != TSDB_CODE_SUCCESS) {
-    goto _error;
-  }
-
-  if (handle) {
-    void* pSinkParam = NULL;
-    code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle);
-    if (code != TSDB_CODE_SUCCESS) {
-      goto _error;
-    }
-
-    code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam);
-  }
-
-_error:
-  // if failed to add ref for all tables in this query, abort current query
-  return code;
-}
-
-#ifdef TEST_IMPL
-// wait moment
-int waitMoment(SQInfo* pQInfo) {
-  if (pQInfo->sql) {
-    int   ms = 0;
-    char* pcnt = strstr(pQInfo->sql, " count(*)");
-    if (pcnt) return 0;
-
-    char* pos = strstr(pQInfo->sql, " t_");
-    if (pos) {
-      pos += 3;
-      ms = atoi(pos);
-      while (*pos >= '0' && *pos <= '9') {
-        pos++;
-      }
-      char unit_char = *pos;
-      if (unit_char == 'h') {
-        ms *= 3600 * 1000;
-      } else if (unit_char == 'm') {
-        ms *= 60 * 1000;
-      } else if (unit_char == 's') {
-        ms *= 1000;
-      }
-    }
-    if (ms == 0) return 0;
-    printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql);
-
-    if (ms < 1000) {
-      taosMsleep(ms);
-    } else {
-      int used_ms = 0;
-      while (used_ms < ms) {
-        taosMsleep(1000);
-        used_ms += 1000;
-        if (isTaskKilled(pQInfo)) {
-          printf("test check query is canceled, sleep break.%s\n", pQInfo->sql);
-          break;
-        }
-      }
-    }
-  }
-  return 1;
-}
-#endif
-
-int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  int64_t        threadId = taosGetSelfPthreadId();
-
-  *pRes = NULL;
-  int64_t curOwner = 0;
-  if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) {
-    qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner);
-    pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC;
-    return pTaskInfo->code;
-  }
-
-  if (pTaskInfo->cost.start == 0) {
-    pTaskInfo->cost.start = taosGetTimestampMs();
-  }
-
-  if (isTaskKilled(pTaskInfo)) {
-    qDebug("%s already killed, abort", GET_TASKID(pTaskInfo));
-    return TSDB_CODE_SUCCESS;
-  }
-
-  // error occurs, record the error code and return to client
-  int32_t ret = setjmp(pTaskInfo->env);
-  if (ret != TSDB_CODE_SUCCESS) {
-    pTaskInfo->code = ret;
-    cleanUpUdfs();
-    qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));
-    return pTaskInfo->code;
-  }
-
-  qDebug("%s execTask is launched", GET_TASKID(pTaskInfo));
-
-  int64_t st = taosGetTimestampUs();
-
-  *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot);
-  uint64_t el = (taosGetTimestampUs() - st);
-
-  pTaskInfo->cost.elapsedTime += el;
-  if (NULL == *pRes) {
-    *useconds = pTaskInfo->cost.elapsedTime;
-  }
-
-  cleanUpUdfs();
-
-  int32_t  current = (*pRes != NULL) ? (*pRes)->info.rows : 0;
-  uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows;
-
-  qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms",
-         GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0);
-
-  atomic_store_64(&pTaskInfo->owner, 0);
-  return pTaskInfo->code;
-}
-
-int32_t qKillTask(qTaskInfo_t qinfo) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
-
-  if (pTaskInfo == NULL) {
-    return TSDB_CODE_QRY_INVALID_QHANDLE;
-  }
-
-  qDebug("%s execTask killed", GET_TASKID(pTaskInfo));
-  setTaskKilled(pTaskInfo);
-
-  // Wait for the query executing thread being stopped/
-  // Once the query is stopped, the owner of qHandle will be cleared immediately.
-  while (pTaskInfo->owner != 0) {
-    taosMsleep(100);
-  }
-
-  return TSDB_CODE_SUCCESS;
-}
-
-int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
-
-  if (pTaskInfo == NULL) {
-    return TSDB_CODE_QRY_INVALID_QHANDLE;
-  }
-
-  qDebug("%s execTask async killed", GET_TASKID(pTaskInfo));
-  setTaskKilled(pTaskInfo);
-
-  return TSDB_CODE_SUCCESS;
-}
-
-void qDestroyTask(qTaskInfo_t qTaskHandle) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle;
-  if (pTaskInfo == NULL) {
-    return;
-  }
-
-  qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows);
-
-  queryCostStatis(pTaskInfo);  // print the query cost summary
-  doDestroyTask(pTaskInfo);
-}
-
-int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  int32_t        capacity = 0;
-
-  return getOperatorExplainExecInfo(pTaskInfo->pRoot, pRes, &capacity, resNum);
-}
-
-int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) {
-  SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo;
-  if (pTaskInfo->pRoot == NULL) {
-    return TSDB_CODE_INVALID_PARA;
-  }
-
-  int32_t nOptrWithVal = 0;
-  int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
-  if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) {
-    taosMemoryFreeClear(*pOutput);
-    *len = 0;
-  }
-  return code;
-}
-
-int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) {
-  SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo;
-
-  if (pTaskInfo == NULL || pInput == NULL || len == 0) {
-    return TSDB_CODE_INVALID_PARA;
-  }
-
-  return decodeOperator(pTaskInfo->pRoot, pInput, len);
-}
-
-int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  SOperatorInfo* pOperator = pTaskInfo->pRoot;
-
-  while (1) {
-    uint8_t type = pOperator->operatorType;
-    if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
-      *scanner = pOperator->info;
-      return 0;
-    } else {
-      ASSERT(pOperator->numOfDownstream == 1);
-      pOperator = pOperator->pDownstream[0];
-    }
-  }
-}
-
-#if 0
-int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM);
-  taosWriteQitem(pTaskInfo->streamInfo.inputQueue->queue, pItem);
-  return 0;
-}
-#endif
-
-int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM);
-  pTaskInfo->streamInfo.recoverStartVer = startVer;
-  pTaskInfo->streamInfo.recoverEndVer = endVer;
-  pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE;
-  return 0;
-}
-
-void* qExtractReaderFromStreamScanner(void* scanner) {
-  SStreamScanInfo* pInfo = scanner;
-  return (void*)pInfo->tqReader;
-}
-
-const SSchemaWrapper* qExtractSchemaFromStreamScanner(void* scanner) {
-  SStreamScanInfo* pInfo = scanner;
-  return pInfo->tqReader->pSchemaWrapper;
-}
-
-const STqOffset* qExtractStatusFromStreamScanner(void* scanner) {
-  SStreamScanInfo* pInfo = scanner;
-  return &pInfo->offset;
-}
-
-void* qStreamExtractMetaMsg(qTaskInfo_t tinfo) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
-  return pTaskInfo->streamInfo.metaBlk;
-}
-
-int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
-  memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal));
-  return 0;
-}
-
-int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-  SOperatorInfo* pOperator = pTaskInfo->pRoot;
-  ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE);
-  pTaskInfo->streamInfo.prepareStatus = *pOffset;
-  if (!tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) {
-    while (1) {
-      uint8_t type = pOperator->operatorType;
-      pOperator->status = OP_OPENED;
-      if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
-        SStreamScanInfo* pInfo = pOperator->info;
-        if (pOffset->type == TMQ_OFFSET__LOG) {
-          STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
-          tsdbReaderClose(pTSInfo->dataReader);
-          pTSInfo->dataReader = NULL;
-#if 0
-          if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) &&
-              pInfo->tqReader->pWalReader->curVersion != pOffset->version) {
-            qError("prepare scan ver %ld actual ver %ld, last %ld", pOffset->version,
-                   pInfo->tqReader->pWalReader->curVersion, pTaskInfo->streamInfo.lastStatus.version);
-            ASSERT(0);
-          }
-#endif
-          if (tqSeekVer(pInfo->tqReader, pOffset->version + 1) < 0) {
-            return -1;
-          }
-          ASSERT(pInfo->tqReader->pWalReader->curVersion == pOffset->version + 1);
-        } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
-          /*pInfo->blockType = STREAM_INPUT__TABLE_SCAN;*/
-          int64_t uid = pOffset->uid;
-          int64_t ts = pOffset->ts;
-
-          if (uid == 0) {
-            if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) {
-              STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0);
-              uid = pTableInfo->uid;
-              ts = INT64_MIN;
-            } else {
-              return -1;
-            }
-          }
-
-          /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/
-          /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/
-          STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
-          int32_t         tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
-
-#ifndef NDEBUG
-
-          qDebug("switch to next table %ld (cursor %d), %ld rows returned", uid, pTableScanInfo->currentTable,
-                 pInfo->pTableScanOp->resultInfo.totalRows);
-          pInfo->pTableScanOp->resultInfo.totalRows = 0;
-#endif
-
-          bool found = false;
-          for (int32_t i = 0; i < tableSz; i++) {
-            STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i);
-            if (pTableInfo->uid == uid) {
-              found = true;
-              pTableScanInfo->currentTable = i;
-              break;
-            }
-          }
-
-          // TODO after dropping table, table may be not found
-          ASSERT(found);
-
-          if (pTableScanInfo->dataReader == NULL) {
-            if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond,
-                               pTaskInfo->tableqinfoList.pTableList, &pTableScanInfo->dataReader, NULL) < 0 ||
-                pTableScanInfo->dataReader == NULL) {
-              ASSERT(0);
-            }
-          }
-
-          tsdbSetTableId(pTableScanInfo->dataReader, uid);
-          int64_t oldSkey = pTableScanInfo->cond.twindows.skey;
-          pTableScanInfo->cond.twindows.skey = ts + 1;
-          tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond);
-          pTableScanInfo->cond.twindows.skey = oldSkey;
-          pTableScanInfo->scanTimes = 0;
-
-          qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts,
-                 pTableScanInfo->currentTable, tableSz);
-          /*}*/
-
-        } else {
-          ASSERT(0);
-        }
-        return 0;
-      } else {
-        ASSERT(pOperator->numOfDownstream == 1);
-        pOperator = pOperator->pDownstream[0];
-      }
-    }
-  }
-  return 0;
-}
-
-#if 0
-int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-
-  if (uid == 0) {
-    if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) {
-      STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0);
-      uid = pTableInfo->uid;
-      ts = INT64_MIN;
-    }
-  }
-
-  return doPrepareScan(pTaskInfo->pRoot, uid, ts);
-}
-
-int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) {
-  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
-
-  return doGetScanStatus(pTaskInfo->pRoot, uid, ts);
-}
-#endif

source/libs/executor/src/executorimpl.c

@@ -521,7 +521,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt
 
         // NOTE: the last parameter is the primary timestamp column
         // todo: refactor this
-        if (fmIsTimelineFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) {
+        if (fmIsImplicitTsFunc(pCtx[i].functionId) && (j == pOneExpr->base.numOfParams - 1)) {
           pInput->pPTS = pInput->pData[j];  // in case of merge function, this is not always the ts column data.
                                             //          ASSERT(pInput->pPTS->info.type == TSDB_DATA_TYPE_TIMESTAMP);
         }
@@ -1333,7 +1333,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO
 
 static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep);
 
-void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
+void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) {
   if (pFilterNode == NULL || pBlock->info.rows == 0) {
     return;
   }
@@ -1354,6 +1354,20 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) {
   filterFreeInfo(filter);
 
   extractQualifiedTupleByFilterResult(pBlock, rowRes, keep);
+
+  if (pColMatchInfo != NULL) {
+    for(int32_t i = 0; i < taosArrayGetSize(pColMatchInfo); ++i) {
+      SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, i);
+      if (pInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+        SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pInfo->targetSlotId);
+        if (pColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
+          blockDataUpdateTsWindow(pBlock, pInfo->targetSlotId);
+          break;
+        }
+      }
+    }
+  }
+
   taosMemoryFree(rowRes);
 }
 
@@ -1602,6 +1616,9 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
   SSDataBlock*    pBlock = pbInfo->pRes;
   SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
 
+  // set output datablock version
+  pBlock->info.version = pTaskInfo->version;
+
   blockDataCleanup(pBlock);
   if (!hasDataInGroupInfo(pGroupResInfo)) {
     return;
@@ -3040,7 +3057,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
   blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
   while (1) {
     doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf);
-    doFilter(pAggInfo->pCondition, pInfo->pRes);
+    doFilter(pAggInfo->pCondition, pInfo->pRes, NULL);
 
     if (!hasDataInGroupInfo(&pAggInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
@@ -3206,6 +3223,7 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa
     pLimitInfo->currentGroupId = pBlock->info.groupId;
   }
 
+  // here check for a new group data, we need to handle the data of the previous group.
   if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) {
     pLimitInfo->numOfOutputGroups += 1;
     if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
@@ -3218,6 +3236,11 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa
     // reset the value for a new group data
     pLimitInfo->numOfOutputRows = 0;
     pLimitInfo->remainOffset = pLimitInfo->limit.offset;
+
+    // existing rows that belongs to previous group.
+    if (pBlock->info.rows > 0) {
+      return PROJECT_RETRIEVE_DONE;
+    }
   }
 
   // here we reach the start position, according to the limit/offset requirements.
@@ -3262,7 +3285,9 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
 
   SExprSupp*   pSup = &pOperator->exprSupp;
   SSDataBlock* pRes = pInfo->pRes;
-  blockDataCleanup(pRes);
+  SSDataBlock* pFinalRes = pProjectInfo->pFinalRes;
+
+  blockDataCleanup(pFinalRes);
 
   SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
   if (pOperator->status == OP_EXEC_DONE) {
@@ -3273,24 +3298,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
     return NULL;
   }
 
-#if 0
-  if (pProjectInfo->existDataBlock) {  // TODO refactor
-    SSDataBlock* pBlock = pProjectInfo->existDataBlock;
-    pProjectInfo->existDataBlock = NULL;
-
-    // the pDataBlock are always the same one, no need to call this again
-    setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC);
-
-    blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows);
-    projectApplyFunctions(pOperator->exprSupp.pExprInfo, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->exprSupp.numOfExprs);
-    if (pRes->info.rows >= pProjectInfo->binfo.capacity * 0.8) {
-      copyTsColoum(pRes, pInfo->pCtx, pOperator->exprSupp.numOfExprs);
-      resetResultRowEntryResult(pInfo->pCtx, pOperator->exprSupp.numOfExprs);
-      return pRes;
-    }
-  }
-#endif
-
   int64_t st = 0;
   int32_t order = 0;
   int32_t scanFlag = 0;
@@ -3300,26 +3307,58 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
   }
 
   SOperatorInfo* downstream = pOperator->pDownstream[0];
+  SLimitInfo* pLimitInfo = &pProjectInfo->limitInfo;
 
   while(1) {
+    while (1) {
+      blockDataCleanup(pRes);
+
       // The downstream exec may change the value of the newgroup, so use a local variable instead.
-    qDebug("projection call next");
       SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
       if (pBlock == NULL) {
-      qDebug("projection get null");
-
-      /*if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH) {*/
         doSetOperatorCompleted(pOperator);
-      /*} else if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) {*/
-      /*pOperator->status = OP_RES_TO_RETURN;*/
-      /*}*/
         break;
       }
+
       if (pBlock->info.type == STREAM_RETRIEVE) {
         // for stream interval
         return pBlock;
       }
 
+      if (pLimitInfo->remainGroupOffset > 0) {
+        if (pLimitInfo->currentGroupId == 0 || pLimitInfo->currentGroupId == pBlock->info.groupId) {  // it is the first group
+          pLimitInfo->currentGroupId = pBlock->info.groupId;
+          continue;
+        } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) {
+          // now it is the data from a new group
+          pLimitInfo->remainGroupOffset -= 1;
+          pLimitInfo->currentGroupId = pBlock->info.groupId;
+
+          // ignore data block in current group
+          if (pLimitInfo->remainGroupOffset > 0) {
+            continue;
+          }
+        }
+
+        // set current group id of the project operator
+        pLimitInfo->currentGroupId = pBlock->info.groupId;
+      }
+
+      // remainGroupOffset == 0
+      // here check for a new group data, we need to handle the data of the previous group.
+      if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) {
+        pLimitInfo->numOfOutputGroups += 1;
+        if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
+          doSetOperatorCompleted(pOperator);
+          break;
+        }
+
+        // reset the value for a new group data
+        // existing rows that belongs to previous group.
+        pLimitInfo->numOfOutputRows = 0;
+        pLimitInfo->remainOffset = pLimitInfo->limit.offset;
+      }
+
       // the pDataBlock are always the same one, no need to call this again
       int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);
       if (code != TSDB_CODE_SUCCESS) {
@@ -3335,32 +3374,65 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
         longjmp(pTaskInfo->env, code);
       }
 
-    int32_t status = handleLimitOffset(pOperator, &pProjectInfo->limitInfo, pInfo->pRes, true);
+      // set current group id
+      pLimitInfo->currentGroupId = pBlock->info.groupId;
 
-    // filter shall be applied after apply functions and limit/offset on the result
+      if (pLimitInfo->remainOffset >= pInfo->pRes->info.rows) {
-    doFilter(pProjectInfo->pFilterNode, pInfo->pRes);
+        pLimitInfo->remainOffset -= pInfo->pRes->info.rows;
-
+        blockDataCleanup(pInfo->pRes);
-    if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
-      break;
-    }
-
-    if (status == PROJECT_RETRIEVE_CONTINUE || pInfo->pRes->info.rows == 0) {
         continue;
-    } else if (status == PROJECT_RETRIEVE_DONE) {
+      } else if (pLimitInfo->remainOffset < pInfo->pRes->info.rows && pLimitInfo->remainOffset > 0) {
+        blockDataTrimFirstNRows(pInfo->pRes, pLimitInfo->remainOffset);
+        pLimitInfo->remainOffset = 0;
+      }
+
+      // check for the limitation in each group
+      if (pLimitInfo->limit.limit >= 0 &&
+          pLimitInfo->numOfOutputRows + pInfo->pRes->info.rows >= pLimitInfo->limit.limit) {
+        int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows);
+        blockDataKeepFirstNRows(pInfo->pRes, keepRows);
+        if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) {
+          pOperator->status = OP_EXEC_DONE;
+        }
+      }
+
+      pLimitInfo->numOfOutputRows += pInfo->pRes->info.rows;
+      break;
+    }
+
+    // no results generated
+    if (pInfo->pRes->info.rows == 0 || (!pProjectInfo->mergeDataBlocks)) {
+      break;
+    }
+
+    if (pProjectInfo->mergeDataBlocks) {
+      pFinalRes->info.groupId = pInfo->pRes->info.groupId;
+      pFinalRes->info.version = pInfo->pRes->info.version;
+
+      // continue merge data, ignore the group id
+      blockDataMerge(pFinalRes, pInfo->pRes);
+
+      if (pFinalRes->info.rows + pInfo->pRes->info.rows <= pOperator->resultInfo.threshold) {
+        continue;
+      }
+    }
+
+    // do apply filter
+    SSDataBlock* p = pProjectInfo->mergeDataBlocks ? pFinalRes : pRes;
+    doFilter(pProjectInfo->pFilterNode, p, NULL);
+    if (p->info.rows > 0) {
       break;
     }
   }
 
-  size_t rows = pInfo->pRes->info.rows;
+  SSDataBlock* p = pProjectInfo->mergeDataBlocks ? pFinalRes : pRes;
-  pProjectInfo->limitInfo.numOfOutputRows += rows;
+  pOperator->resultInfo.totalRows += p->info.rows;
-
-  pOperator->resultInfo.totalRows += rows;
 
   if (pOperator->cost.openCost == 0) {
     pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
   }
 
-  return (rows > 0) ? pInfo->pRes : NULL;
+  return (p->info.rows > 0) ? p : NULL;
 }
 
 static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo* pInfo, SResultInfo* pResultInfo,
@@ -3489,7 +3561,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
       break;
     }
 
-    doFilter(pInfo->pCondition, fillResult);
+    doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo);
     if (fillResult->info.rows > 0) {
       break;
     }
@@ -3752,6 +3824,7 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
   cleanupAggSup(&pInfo->aggSup);
   taosArrayDestroy(pInfo->pPseudoColInfo);
 
+  blockDataDestroy(pInfo->pFinalRes);
   taosMemoryFreeClear(param);
 }
 
@@ -3811,7 +3884,10 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
   initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo);
 
   pInfo->binfo.pRes = pResBlock;
+  pInfo->pFinalRes = createOneDataBlock(pResBlock, false);
+
   pInfo->pFilterNode = pProjPhyNode->node.pConditions;
+  pInfo->mergeDataBlocks = pProjPhyNode->mergeDataBlock;
 
   int32_t numOfRows = 4096;
   size_t  keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
@@ -3947,7 +4023,7 @@ static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
       }
     }
 
-    doFilter(pIndefInfo->pCondition, pInfo->pRes);
+    doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL);
     size_t rows = pInfo->pRes->info.rows;
     if (rows > 0 || pOperator->status == OP_EXEC_DONE) {
       break;
@@ -4131,9 +4207,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT
   return pTaskInfo;
 }
 
-static STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
-                                       STableListInfo* pTableListInfo, const char* idstr);
-
 static SArray* extractColumnInfo(SNodeList* pNodeList);
 
 SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode);
@@ -4175,8 +4248,10 @@ int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode,
 
 SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) {
   int32_t numOfCols = LIST_LENGTH(pScanNode->pScanCols);
+  int32_t numOfTags = LIST_LENGTH(pScanNode->pScanPseudoCols);
+
   SSchemaWrapper* pqSw = taosMemoryCalloc(1, sizeof(SSchemaWrapper));
-  pqSw->pSchema = taosMemoryCalloc(numOfCols, sizeof(SSchema));
+  pqSw->pSchema = taosMemoryCalloc(numOfCols + numOfTags, sizeof(SSchema));
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanCols, i);
@@ -4189,6 +4264,22 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) {
     strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name));
   }
 
+  // this the tags and pseudo function columns, we only keep the tag columns
+  for(int32_t i = 0; i < numOfTags; ++i) {
+    STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanPseudoCols, i);
+
+    int32_t type = nodeType(pNode->pExpr);
+    if (type == QUERY_NODE_COLUMN) {
+      SColumnNode* pColNode = (SColumnNode*)pNode->pExpr;
+
+      SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++];
+      pSchema->colId = pColNode->colId;
+      pSchema->type = pColNode->node.resType.type;
+      pSchema->type = pColNode->node.resType.bytes;
+      strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name));
+    }
+  }
+
   return pqSw;
 }
 
@@ -4290,69 +4381,15 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
   int32_t groupNum = 0;
   for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) {
     STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
-    SMetaReader    mr = {0};
+    int32_t code = getGroupIdFromTagsVal(pHandle->meta, info->uid, group, keyBuf, &info->groupId);
-    metaReaderInit(&mr, pHandle->meta, 0);
+    if (code != TSDB_CODE_SUCCESS) {
-    metaGetTableEntryByUid(&mr, info->uid);
-
-    SNodeList* groupNew = nodesCloneList(group);
-
-    nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr);
-    char* isNull = (char*)keyBuf;
-    char* pStart = (char*)keyBuf + nullFlagSize;
-
-    SNode*  pNode;
-    int32_t index = 0;
-    FOREACH(pNode, groupNew) {
-      SNode*  pNew = NULL;
-      int32_t code = scalarCalculateConstants(pNode, &pNew);
-      if (TSDB_CODE_SUCCESS == code) {
-        REPLACE_NODE(pNew);
-      } else {
-        taosMemoryFree(keyBuf);
-        nodesDestroyList(groupNew);
-        metaReaderClear(&mr);
       return code;
     }
 
-      ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
+    taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t));
-      SValueNode* pValue = (SValueNode*)pNew;
-
-      if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL || pValue->isNull) {
-        isNull[index++] = 1;
-        continue;
-      } else {
-        isNull[index++] = 0;
-        char* data = nodesGetValueFromNode(pValue);
-        if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) {
-          if (tTagIsJson(data)) {
-            terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
-            taosMemoryFree(keyBuf);
-            nodesDestroyList(groupNew);
-            metaReaderClear(&mr);
-            return terrno;
-          }
-          int32_t len = getJsonValueLen(data);
-          memcpy(pStart, data, len);
-          pStart += len;
-        } else if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) {
-          memcpy(pStart, data, varDataTLen(data));
-          pStart += varDataTLen(data);
-        } else {
-          memcpy(pStart, data, pValue->node.resType.bytes);
-          pStart += pValue->node.resType.bytes;
-        }
-      }
-    }
-
-    int32_t  len = (int32_t)(pStart - (char*)keyBuf);
-    uint64_t groupId = calcGroupId(keyBuf, len);
-    taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t));
-    info->groupId = groupId;
     groupNum++;
-
-    nodesDestroyList(groupNew);
-    metaReaderClear(&mr);
   }
+
   taosMemoryFree(keyBuf);
 
   if (pTableListInfo->needSortTableByGroupId) {
@@ -4440,12 +4477,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo);
     } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
-      STimeWindowAggSupp aggSup = (STimeWindowAggSupp){
-          .waterMark = pTableScanNode->watermark,
-          .calTrigger = pTableScanNode->triggerType,
-          .maxTs = INT64_MIN,
-      };
-
       if (pHandle->vnode) {
         int32_t code =
             createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort,
@@ -4465,7 +4496,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
 #endif
 
       pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan);
-      SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, &aggSup, pTaskInfo);
+      SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo);
       return pOperator;
 
     } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
@@ -5136,8 +5167,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF
   }
   pSup->valueSize = size;
 
-  pSup->pScanWindow = taosArrayInit(4, sizeof(STimeWindow));
+  pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR);
-
   int32_t pageSize = 4096;
   while (pageSize < pSup->resultRowSize * 4) {
     pageSize <<= 1u;

source/libs/executor/src/groupoperator.c

@@ -299,7 +299,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) {
   SSDataBlock* pRes = pInfo->binfo.pRes;
   while(1) {
     doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
-    doFilter(pInfo->pCondition, pRes);
+    doFilter(pInfo->pCondition, pRes, NULL);
 
     bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
     if (!hasRemain) {

source/libs/executor/src/joinoperator.c

@@ -211,7 +211,7 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
       break;
     }
     if (pJoinInfo->pCondAfterMerge != NULL) {
-      doFilter(pJoinInfo->pCondAfterMerge, pRes);
+      doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL);
     }
     if (pRes->info.rows >= pOperator->resultInfo.threshold) {
       break;

source/libs/executor/src/scanoperator.c

@@ -25,7 +25,6 @@
 #include "tdatablock.h"
 #include "tmsg.h"
 
-#include "executorimpl.h"
 #include "query.h"
 #include "tcompare.h"
 #include "thash.h"
@@ -265,7 +264,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
   }
 
   int64_t st = taosGetTimestampMs();
-  doFilter(pTableScanInfo->pFilterNode, pBlock);
+  doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo);
 
   int64_t et = taosGetTimestampMs();
   pTableScanInfo->readRecorder.filterTime += (et - st);
@@ -274,6 +273,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
     pCost->filterOutBlocks += 1;
     qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
            pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
+  } else {
+    qDebug("%s data block filter out, elapsed time:%"PRId64, GET_TASKID(pTaskInfo), (et - st));
   }
 
   return TSDB_CODE_SUCCESS;
@@ -812,6 +813,10 @@ static bool isSignleIntervalWindow(SStreamScanInfo* pInfo) {
   return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
 }
 
+static bool isSlidingWindow(SStreamScanInfo* pInfo) {
+  return isIntervalWindow(pInfo) && pInfo->interval.interval != pInfo->interval.sliding;
+}
+
 static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) {
   uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t));
   if (groupId) {
@@ -834,17 +839,10 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) {
 }
 
 static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
-  ASSERT(rowIndex < pBlock->info.rows);
-  switch (pBlock->info.type) {
-    case STREAM_DELETE_DATA:
-    case STREAM_RETRIEVE: {
   SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
   uint64_t*        groupCol = (uint64_t*)pColInfo->pData;
+  ASSERT(rowIndex < pBlock->info.rows);
   pInfo->groupId = groupCol[rowIndex];
-    } break;
-    default:
-      break;
-  }
 }
 
 void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) {
@@ -864,7 +862,17 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
   SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
   TSKEY*           endData = (TSKEY*)pEndTsCol->pData;
   STimeWindow      win = {.skey = startData[*pRowIndex], .ekey = endData[*pRowIndex]};
+
+  SColumnInfoData* pCalStartTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
+  TSKEY*           calStartData = (TSKEY*)pCalStartTsCol->pData;
+  SColumnInfoData* pCalEndTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
+  TSKEY*           calEndData = (TSKEY*)pCalEndTsCol->pData;
+
   setGroupId(pInfo, pBlock, GROUPID_COLUMN_INDEX, *pRowIndex);
+  if (isSlidingWindow(pInfo)) {
+    pInfo->updateWin.skey = calStartData[*pRowIndex];
+    pInfo->updateWin.ekey = calEndData[*pRowIndex];
+  }
   (*pRowIndex)++;
 
   for (; *pRowIndex < pBlock->info.rows; (*pRowIndex)++) {
@@ -876,8 +884,8 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_
       win.skey = TMIN(win.skey, startData[*pRowIndex]);
       continue;
     }
-    ASSERT((win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) ||
+    ASSERT(!(win.skey > startData[*pRowIndex] && win.ekey < endData[*pRowIndex]) ||
-           (isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0)));
+           !(isInTimeWindow(&win, startData[*pRowIndex], 0) || isInTimeWindow(&win, endData[*pRowIndex], 0)));
     break;
   }
 
@@ -908,68 +916,6 @@ static STimeWindow getSlidingWindow(TSKEY* tsCol, SInterval* pInterval, SDataBlo
     win.ekey = endWin.ekey;
   }
 }
-static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
-  STimeWindow win = {
-      .skey = INT64_MIN,
-      .ekey = INT64_MAX,
-  };
-  bool needRead = false;
-  if (!isStateWindow(pInfo) && (*pRowIndex) < pSDB->info.rows) {
-    SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, tsColIndex);
-    TSKEY*           tsCols = (TSKEY*)pColDataInfo->pData;
-    SResultRowInfo   dumyInfo;
-    dumyInfo.cur.pageId = -1;
-    if (isSessionWindow(pInfo)) {
-      SStreamAggSupporter* pAggSup = pInfo->sessionSup.pStreamAggSup;
-      int64_t              gap = pInfo->sessionSup.gap;
-      int32_t              winIndex = 0;
-      SResultWindowInfo*   pCurWin =
-          getSessionTimeWindow(pAggSup, tsCols[*pRowIndex], INT64_MIN, pSDB->info.groupId, gap, &winIndex);
-      win = pCurWin->win;
-      setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex);
-      (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, *pRowIndex, gap, NULL);
-    } else {
-      setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex);
-      pInfo->updateWin.skey = tsCols[*pRowIndex];
-      win = getSlidingWindow(tsCols, &pInfo->interval, &pSDB->info, pRowIndex);
-      pInfo->updateWin.ekey = tsCols[*pRowIndex - 1];
-      // win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, TSDB_ORDER_ASC);
-      // (*pRowIndex) +=
-      //     getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL,
-      //     TSDB_ORDER_ASC);
-    }
-    needRead = true;
-  } else if (isStateWindow(pInfo)) {
-    SArray* pWins = pInfo->sessionSup.pStreamAggSup->pScanWindow;
-    int32_t size = taosArrayGetSize(pWins);
-    if (pInfo->scanWinIndex < size) {
-      win = *(STimeWindow*)taosArrayGet(pWins, pInfo->scanWinIndex);
-      pInfo->scanWinIndex++;
-      needRead = true;
-    } else {
-      pInfo->scanWinIndex = 0;
-      taosArrayClear(pWins);
-    }
-  }
-  if (!needRead) {
-    return false;
-  }
-  resetTableScanInfo(pInfo->pTableScanOp->info, &win);
-  return true;
-}
-
-static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) {
-  for (int32_t j = 0; j < taosArrayGetSize(source->pDataBlock); j++) {
-    SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j);
-    SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j);
-    if (colDataIsNull_s(pSourceCol, sourceRowId)) {
-      colDataAppendNULL(pDestCol, dest->info.rows);
-    } else {
-      colDataAppend(pDestCol, dest->info.rows, colDataGetData(pSourceCol, sourceRowId), false);
-    }
-  }
-  dest->info.rows++;
-}
 
 static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
   while (1) {
@@ -982,29 +928,6 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
     if (!pResult) {
       blockDataCleanup(pSDB);
       *pRowIndex = 0;
-      STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
-      tsdbReaderClose(pTableScanInfo->dataReader);
-      pTableScanInfo->dataReader = NULL;
-      return NULL;
-    }
-
-    if (pResult->info.groupId == pInfo->groupId) {
-      return pResult;
-    }
-  }
-}
-
-static SSDataBlock* doDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
-  while (1) {
-    SSDataBlock* pResult = NULL;
-    pResult = doTableScan(pInfo->pTableScanOp);
-    if (pResult == NULL) {
-      if (prepareDataScan(pInfo, pSDB, tsColIndex, pRowIndex)) {
-        // scan next window data
-        pResult = doTableScan(pInfo->pTableScanOp);
-      }
-    }
-    if (!pResult) {
       pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
       STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
       tsdbReaderClose(pTableScanInfo->dataReader);
@@ -1017,77 +940,31 @@ static SSDataBlock* doDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_
       return pResult;
     }
   }
+}
 
-  /* Todo(liuyao) for partition by column
+static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
-    SSDataBlock* pBlock = createOneDataBlock(pResult, true);
+  if (pSrcBlock->info.rows == 0) {
-    blockDataCleanup(pResult);
+    return TSDB_CODE_SUCCESS;
-    for (int32_t i = 0; i < pBlock->info.rows; i++) {
-      uint64_t id = getGroupId(pInfo->pOperatorDumy, pBlock->info.uid);
-      if (id == pInfo->groupId) {
-        copyOneRow(pResult, pBlock, i);
   }
+  blockDataCleanup(pDestBlock);
+  int32_t code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows);
+    if (code != TSDB_CODE_SUCCESS) {
+    return code;
   }
-    return pResult;
+  ASSERT(taosArrayGetSize(pSrcBlock->pDataBlock) >= 3);
-  */
+  SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
-}
-static void generateIntervalTs(SStreamScanInfo* pInfo, SSDataBlock* pDelBlock, SOperatorInfo* pOperator,
-                               SSDataBlock* pUpdateRes) {
-  if (pDelBlock->info.rows == 0) {
-    return;
-  }
-  blockDataCleanup(pUpdateRes);
-  blockDataEnsureCapacity(pUpdateRes, 64);
-  ASSERT(taosArrayGetSize(pDelBlock->pDataBlock) >= 3);
-  SColumnInfoData* pStartTsCol = taosArrayGet(pDelBlock->pDataBlock, START_TS_COLUMN_INDEX);
   TSKEY*           startData = (TSKEY*)pStartTsCol->pData;
-  SColumnInfoData* pEndTsCol = taosArrayGet(pDelBlock->pDataBlock, END_TS_COLUMN_INDEX);
+  SColumnInfoData* pEndTsCol = taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
   TSKEY*           endData = (TSKEY*)pEndTsCol->pData;
-  SColumnInfoData* pGpCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX);
+  SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
-  uint64_t*        uidCol = (uint64_t*)pGpCol->pData;
+  uint64_t*        uidCol = (uint64_t*)pUidCol->pData;
 
-  SColumnInfoData* pDestTsCol = taosArrayGet(pUpdateRes->pDataBlock, START_TS_COLUMN_INDEX);
+  SColumnInfoData* pDestStartCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX);
-  SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, GROUPID_COLUMN_INDEX);
+  SColumnInfoData* pDestEndCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX);
-  for (int32_t i = pInfo->deleteDataIndex;
+  SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX);
-       i < pDelBlock->info.rows &&
-       i < pDelBlock->info.capacity - (endData[i] - startData[i]) / pInfo->interval.interval - 1;
-       i++) {
-    uint64_t groupId = getGroupId(pOperator, uidCol[i]);
-    for (TSKEY startTs = startData[i]; startTs <= endData[i];) {
-      colDataAppend(pDestTsCol, pUpdateRes->info.rows, (const char*)&startTs, false);
-      colDataAppend(pDestGpCol, pUpdateRes->info.rows, (const char*)&groupId, false);
-      pUpdateRes->info.rows++;
-      startTs = taosTimeAdd(startTs, pInfo->interval.interval, pInfo->interval.intervalUnit, pInfo->interval.precision);
-    }
-    pInfo->deleteDataIndex++;
-  }
-
-  if (pInfo->deleteDataIndex > 0 && pInfo->deleteDataIndex == pDelBlock->info.rows) {
-    blockDataCleanup(pDelBlock);
-    pInfo->deleteDataIndex = 0;
-  }
-}
-
-static void generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SOperatorInfo* pOperator,
-                              SSDataBlock* pUpdateRes) {
-  if (pBlock->info.rows == 0) {
-    return;
-  }
-  blockDataCleanup(pUpdateRes);
-  blockDataEnsureCapacity(pUpdateRes, pBlock->info.rows);
-  ASSERT(taosArrayGetSize(pBlock->pDataBlock) >= 3);
-  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
-  TSKEY*           startData = (TSKEY*)pStartTsCol->pData;
-  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
-  TSKEY*           endData = (TSKEY*)pEndTsCol->pData;
-  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
-  uint64_t*        uidCol = (uint64_t*)pGpCol->pData;
-
-  SColumnInfoData* pDestStartCol = taosArrayGet(pUpdateRes->pDataBlock, START_TS_COLUMN_INDEX);
-  SColumnInfoData* pDestEndCol = taosArrayGet(pUpdateRes->pDataBlock, END_TS_COLUMN_INDEX);
-  SColumnInfoData* pDestGpCol = taosArrayGet(pUpdateRes->pDataBlock, GROUPID_COLUMN_INDEX);
   int32_t          dummy = 0;
-  for (int32_t i = 0; i < pBlock->info.rows; i++) {
+  for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
-    uint64_t groupId = getGroupId(pOperator, uidCol[i]);
+    uint64_t groupId = getGroupId(pInfo->pTableScanOp, uidCol[i]);
     // gap must be 0.
     SResultWindowInfo* pStartWin =
         getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy);
@@ -1101,46 +978,75 @@ static void generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SOper
     colDataAppend(pDestStartCol, i, (const char*)&pStartWin->win.skey, false);
     colDataAppend(pDestEndCol, i, (const char*)&pEndWin->win.ekey, false);
     colDataAppend(pDestGpCol, i, (const char*)&groupId, false);
-    pUpdateRes->info.rows++;
+    pDestBlock->info.rows++;
   }
+  return TSDB_CODE_SUCCESS;
 }
-static void setUpdateData(SStreamScanInfo* pInfo, SSDataBlock* pBlock, SSDataBlock* pUpdateBlock) {
-  blockDataCleanup(pUpdateBlock);
-  int32_t size = taosArrayGetSize(pInfo->tsArray);
-  if (pInfo->tsArrayIndex < size) {
-    SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex);
-    ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
-    blockDataEnsureCapacity(pUpdateBlock, size);
 
-    int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex);
+static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
-    pInfo->groupId = getGroupId(pInfo->pTableScanOp, pBlock->info.uid);
+  blockDataCleanup(pDestBlock);
-    int32_t i = 0;
+  int32_t rows = pSrcBlock->info.rows;
-    for (; i < size; i++) {
+  if (rows == 0) {
-      rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, i + pInfo->tsArrayIndex);
+    return TSDB_CODE_SUCCESS;
-      uint64_t id = getGroupId(pInfo->pTableScanOp, pBlock->info.uid);
-      if (pInfo->groupId != id) {
-        break;
   }
-      copyOneRow(pUpdateBlock, pBlock, rowId);
+  int32_t code = blockDataEnsureCapacity(pDestBlock, rows);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
   }
-    pUpdateBlock->info.rows = i;
+
-    pInfo->tsArrayIndex += i;
+  SColumnInfoData* pTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
-    pUpdateBlock->info.groupId = pInfo->groupId;
+  SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
-    pUpdateBlock->info.type = STREAM_CLEAR;
+  uint64_t* uidCol = (uint64_t*)pUidCol->pData;
-    blockDataUpdateTsWindow(pUpdateBlock, 0);
+  ASSERT(pTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP);
+  TSKEY* tsCol = (TSKEY*)pTsCol->pData;
+  SColumnInfoData* pStartTsCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  SColumnInfoData* pEndTsCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX);
+  SColumnInfoData* pGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+  SColumnInfoData* pCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
+  SColumnInfoData* pCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
+  uint64_t groupId = getGroupId(pInfo->pTableScanOp, uidCol[0]);
+  for (int32_t i = 0; i < rows; ) {
+    colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(tsCol + i), false);
+    STimeWindow win = getSlidingWindow(tsCol, &pInfo->interval, &pSrcBlock->info, &i);
+    colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(tsCol + i - 1), false);
+
+    colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false);
+    colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false);
+    colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false);
+    pDestBlock->info.rows++;
   }
   // all rows have same group id
-  ASSERT(pInfo->tsArrayIndex >= size);
+  pDestBlock->info.groupId = groupId;
-  if (size > 0 && pInfo->tsArrayIndex == size) {
+  return TSDB_CODE_SUCCESS;
-    taosArrayClear(pInfo->tsArray);
 }
 
-  if (size == 0) {
+static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
-    generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pUpdateBlock);
+  int32_t code = TSDB_CODE_SUCCESS;
+  if (isIntervalWindow(pInfo)) {
+    code = generateIntervalScanRange(pInfo, pSrcBlock, pDestBlock);
+  } else {
+    code = generateSessionScanRange(pInfo, pSrcBlock, pDestBlock);
   }
+  pDestBlock->info.type = STREAM_CLEAR;
+  blockDataUpdateTsWindow(pDestBlock, 0);
+  return code;
+}
+
+void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid) {
+  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
+  SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
+  colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)pStartTs, false);
+  colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)pEndTs, false);
+  colDataAppend(pUidCol, pBlock->info.rows, (const char*)pUid, false);
+  pBlock->info.rows++;
 }
 
 static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) {
+  if (out) {
+    blockDataCleanup(pInfo->pUpdateDataRes);
+    blockDataEnsureCapacity(pInfo->pUpdateDataRes, pBlock->info.rows);
+  }
   SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
   ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
   TSKEY* tsCol = (TSKEY*)pColDataInfo->pData;
@@ -1151,9 +1057,13 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
     // must check update info first.
     bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]);
     if ((update || (isSignleIntervalWindow(pInfo) && isCloseWindow(&win, &pInfo->twAggSup))) && out) {
-      taosArrayPush(pInfo->tsArray, &rowId);
+      appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid);
     }
   }
+  if (out) {
+    blockDataUpdateTsWindow(pInfo->pUpdateDataRes, 0);
+    pInfo->pUpdateDataRes->info.type = STREAM_CLEAR;
+  }
 }
 
 static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t uidColIndex) {
@@ -1176,6 +1086,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
   pInfo->pRes->info.rows = pBlock->info.rows;
   pInfo->pRes->info.uid = pBlock->info.uid;
   pInfo->pRes->info.type = STREAM_NORMAL;
+  pInfo->pRes->info.version = pBlock->info.version;
 
   uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t));
   if (groupIdPre) {
@@ -1225,7 +1136,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
     }
   }
 
-  doFilter(pInfo->pCondition, pInfo->pRes);
+  doFilter(pInfo->pCondition, pInfo->pRes, NULL);
   blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
   blockDataFreeRes((SSDataBlock*)pBlock);
   return 0;
@@ -1319,26 +1230,18 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
       case STREAM_RETRIEVE: {
         pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
         pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE;
-        copyDataBlock(pInfo->pPullDataRes, pBlock);
+        copyDataBlock(pInfo->pUpdateRes, pBlock);
-        pInfo->pullDataResIndex = 0;
+        prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
-        prepareDataScan(pInfo, pInfo->pPullDataRes, START_TS_COLUMN_INDEX, &pInfo->pullDataResIndex);
         updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
       } break;
       case STREAM_DELETE_DATA: {
         pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
         pInfo->updateResIndex = 0;
-        if (isIntervalWindow(pInfo)) {
+        generateScanRange(pInfo, pBlock, pInfo->pUpdateRes);
-          copyDataBlock(pInfo->pDeleteDataRes, pBlock);
-          generateIntervalTs(pInfo, pInfo->pDeleteDataRes, pInfo->pTableScanOp, pInfo->pUpdateRes);
-          prepareDataScan(pInfo, pInfo->pUpdateRes, START_TS_COLUMN_INDEX, &pInfo->updateResIndex);
-          pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
-        } else {
-          generateScanRange(pInfo, pBlock, pInfo->pTableScanOp, pInfo->pUpdateRes);
         prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
+        copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes);
         pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
-        }
+        return pInfo->pDeleteDataRes;
-        pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA;
-        return pInfo->pUpdateRes;
       } break;
       default:
         break;
@@ -1346,56 +1249,40 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
     return pBlock;
   } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) {
     qDebug("scan mode %d", pInfo->scanMode);
-    if (pInfo->scanMode == STREAM_SCAN_FROM_RES) {
+    switch (pInfo->scanMode) {
+      case STREAM_SCAN_FROM_RES: {
         blockDataDestroy(pInfo->pUpdateRes);
         pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
         return pInfo->pRes;
-    } else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) {
+      } break;
-      if (isStateWindow(pInfo)) {
+      case STREAM_SCAN_FROM_UPDATERES: {
-        pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
+        generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes);
-      } else {
+        prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
-        pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
+        pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
-        prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
-      }
         return pInfo->pUpdateRes;
-    } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RETRIEVE) {
+      } break;
-      SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex);
+      case STREAM_SCAN_FROM_DATAREADER_RANGE:
-      if (pSDB != NULL) {
+      case STREAM_SCAN_FROM_DATAREADER_RETRIEVE: {
-        checkUpdateData(pInfo, true, pSDB, false);
-        pSDB->info.type = STREAM_PULL_DATA;
-        return pSDB;
-      }
-      pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
-    } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) {
-      SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
-      if (pSDB) {
-        pSDB->info.type = STREAM_NORMAL;
-        checkUpdateData(pInfo, true, pSDB, false);
-        return pSDB;
-      }
-      setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes);
-      if (pInfo->pUpdateRes->info.rows > 0) {
-        prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
-        return pInfo->pUpdateRes;
-      }
-      pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
-    } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE) {
         SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
         if (pSDB) {
-        pSDB->info.type = STREAM_NORMAL;
+          pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA;
           checkUpdateData(pInfo, true, pSDB, false);
           return pSDB;
         }
         pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
-    } else if (isStateWindow(pInfo)) {
+      } break;
-      pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
+      default:
-      pInfo->updateResIndex = pInfo->pUpdateRes->info.rows;
+        break;
-      if (prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex)) {
-        blockDataCleanup(pInfo->pUpdateRes);
-        // return empty data blcok
-        return pInfo->pUpdateRes;
     }
-      pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
+    
+    SStreamAggSupporter* pSup = pInfo->sessionSup.pStreamAggSup;
+    if (isStateWindow(pInfo) && pSup->pScanBlock->info.rows > 0) {
+      pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
+      pInfo->updateResIndex = 0;
+      copyDataBlock(pInfo->pUpdateRes, pSup->pScanBlock);
+      blockDataCleanup(pSup->pScanBlock);
+      prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex);
+      return pInfo->pUpdateRes;
     }
 
     SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
@@ -1452,17 +1339,15 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
       updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
       /*pOperator->status = OP_EXEC_DONE;*/
     } else if (pInfo->pUpdateInfo) {
-      pInfo->tsArrayIndex = 0;
       checkUpdateData(pInfo, true, pInfo->pRes, true);
-      setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes);
       pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlockInfo->window.ekey);
-      if (pInfo->pUpdateRes->info.rows > 0) {
+      if (pInfo->pUpdateDataRes->info.rows > 0) {
-        if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) {
+        if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) {
           pInfo->updateResIndex = 0;
           pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES;
-        } else if (pInfo->pUpdateRes->info.type == STREAM_INVERT) {
+        } else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) {
           pInfo->scanMode = STREAM_SCAN_FROM_RES;
-          return pInfo->pUpdateRes;
+          return pInfo->pUpdateDataRes;
         }
       }
     }
@@ -1511,7 +1396,8 @@ static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) {
 #if 1
   if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) {
     STableScanInfo* pTableScanInfo = pStreamScan->pTableScanOp->info;
-    destroyTableScanOperatorInfo(pTableScanInfo, 1);
+    destroyTableScanOperatorInfo(pTableScanInfo, numOfOutput);
+    taosMemoryFreeClear(pStreamScan->pTableScanOp);
   }
 #endif
   if (pStreamScan->tqReader) {
@@ -1525,13 +1411,13 @@ static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) {
   blockDataDestroy(pStreamScan->pUpdateRes);
   blockDataDestroy(pStreamScan->pPullDataRes);
   blockDataDestroy(pStreamScan->pDeleteDataRes);
+  blockDataDestroy(pStreamScan->pUpdateDataRes);
   taosArrayDestroy(pStreamScan->pBlockLists);
-  taosArrayDestroy(pStreamScan->tsArray);
   taosMemoryFree(pStreamScan);
 }
 
 SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
-                                            STimeWindowAggSupp* pTwSup, SExecTaskInfo* pTaskInfo) {
+                                            SExecTaskInfo* pTaskInfo) {
   SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo));
   SOperatorInfo*   pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
 
@@ -1544,8 +1430,12 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
   SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc;
 
   pInfo->pTagCond = pTagCond;
-
+  pInfo->pGroupTags = pTableScanNode->pGroupTags;
-  pInfo->twAggSup = *pTwSup;
+  pInfo->twAggSup = (STimeWindowAggSupp){
+      .waterMark = pTableScanNode->watermark,
+      .calTrigger = pTableScanNode->triggerType,
+      .maxTs = INT64_MIN,
+  };
 
   int32_t numOfCols = 0;
   pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
@@ -1568,11 +1458,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
     goto _error;
   }
 
-  pInfo->tsArray = taosArrayInit(4, sizeof(int32_t));
-  if (pInfo->tsArray == NULL) {
-    goto _error;
-  }
-
   if (pHandle->vnode) {
     SOperatorInfo*  pTableScanOp = createTableScanOperatorInfo(pTableScanNode, pHandle, pTaskInfo);
     STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info;
@@ -1628,17 +1513,18 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
   }
 
   pInfo->pRes = createResDataBlock(pDescNode);
-  pInfo->pUpdateRes = createResDataBlock(pDescNode);
+  pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
   pInfo->pCondition = pScanPhyNode->node.pConditions;
   pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
   pInfo->sessionSup =
       (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN};
   pInfo->groupId = 0;
-  pInfo->pPullDataRes = createPullDataBlock();
+  pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
   pInfo->pStreamScanOp = pOperator;
   pInfo->deleteDataIndex = 0;
-  pInfo->pDeleteDataRes = createPullDataBlock();
+  pInfo->pDeleteDataRes = createSpecialDataBlock(STREAM_DELETE_DATA);
   pInfo->updateWin = (STimeWindow){.skey = INT64_MAX, .ekey = INT64_MAX};
+  pInfo->pUpdateDataRes = createSpecialDataBlock(STREAM_CLEAR);
 
   pOperator->name = "StreamScanOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
@@ -1761,55 +1647,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) {
     return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes;
   }
 
-  doFilter(pInfo->pCondition, pInfo->pRes);
+  doFilter(pInfo->pCondition, pInfo->pRes, NULL);
-#if 0
-  SFilterInfo* filter = NULL;
-
-  int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0);
-
-  SFilterColumnParam param1 = {.numOfCols = pInfo->pRes->info.numOfCols, .pDataBlock = pInfo->pRes->pDataBlock};
-  code = filterSetDataFromSlotId(filter, &param1);
-
-  int8_t* rowRes = NULL;
-  bool    keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols);
-  filterFreeInfo(filter);
-
-  SSDataBlock* px = createOneDataBlock(pInfo->pRes, false);
-  blockDataEnsureCapacity(px, pInfo->pRes->info.rows);
-
-  // TODO refactor
-  int32_t numOfRow = 0;
-  for (int32_t i = 0; i < pInfo->pRes->info.numOfCols; ++i) {
-    SColumnInfoData* pDest = taosArrayGet(px->pDataBlock, i);
-    SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i);
-
-    if (keep) {
-      colDataAssign(pDest, pSrc, pInfo->pRes->info.rows, &px->info);
-      numOfRow = pInfo->pRes->info.rows;
-    } else if (NULL != rowRes) {
-      numOfRow = 0;
-      for (int32_t j = 0; j < pInfo->pRes->info.rows; ++j) {
-        if (rowRes[j] == 0) {
-          continue;
-        }
-
-        if (colDataIsNull_s(pSrc, j)) {
-          colDataAppendNULL(pDest, numOfRow);
-        } else {
-          colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false);
-        }
-
-        numOfRow += 1;
-      }
-    } else {
-      numOfRow = 0;
-    }
-  }
-
-  px->info.rows = numOfRow;
-  pInfo->pRes = px;
-#endif
-
   return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes;
 }
 
@@ -2777,7 +2615,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
   }
 
   int64_t st = taosGetTimestampMs();
-  doFilter(pTableScanInfo->pFilterNode, pBlock);
+  doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo);
 
   int64_t et = taosGetTimestampMs();
   pTableScanInfo->readRecorder.filterTime += (et - st);
@@ -2895,6 +2733,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   int32_t tableEndIdx = pInfo->tableEndIndex;
 
   STableListInfo* tableListInfo = pInfo->tableListInfo;
+  pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
   createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx,
                             pInfo->dataReaders, GET_TASKID(pTaskInfo));
 
@@ -3067,7 +2906,13 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla
 int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) {
   const STableKeyInfo* info1 = p1;
   const STableKeyInfo* info2 = p2;
-  return info1->groupId - info2->groupId;
+  if (info1->groupId < info2->groupId) {
+    return -1;
+  } else if (info1->groupId > info2->groupId) {
+    return 1;
+  } else {
+    return 0;
+  }
 }
 
 SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo,
@@ -3109,7 +2954,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   pInfo->pColMatchInfo = pColList;
 
   pInfo->pResBlock = createResDataBlock(pDescNode);
-  pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES);
   pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
 
   pInfo->pSortInfo = generateSortByTsInfo(pInfo->pColMatchInfo, pInfo->cond.order);

source/libs/executor/src/sortoperator.c

@@ -216,7 +216,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
       return NULL;
     }
 
-    doFilter(pInfo->pCondition, pBlock);
+    doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo);
     if (blockDataGetNumOfRows(pBlock) == 0) {
       continue;
     }

source/libs/executor/src/timewindowoperator.c

@@ -772,6 +772,41 @@ int32_t binarySearch(void* keyList, int num, TSKEY key, int order, __get_value_f
   return midPos;
 }
 
+int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {
+  SArray*     res = (SArray*)data;
+  SPullWindowInfo* pos = taosArrayGet(res, index);
+  SPullWindowInfo* pData = (SPullWindowInfo*) pKey;
+  if (pData->window.skey == pos->window.skey) {
+    if (pData->groupId > pos->groupId) {
+      return 1;
+    } else if (pData->groupId < pos->groupId) {
+      return -1;
+    }
+    return 0;
+  } else if (pData->window.skey > pos->window.skey) {
+    return 1;
+  }
+  return -1;
+}
+
+static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {
+  int32_t size = taosArrayGetSize(pPullWins);
+  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);
+  if (index == -1) {
+    index = 0;
+  } else {
+    if (comparePullWinKey(pPullInfo, pPullWins, index) > 0) {
+      index++;
+    } else {
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t compareResKey(void* pKey, void* data, int32_t index) {
   SArray*     res = (SArray*)data;
   SResKeyPos* pos = taosArrayGetP(res, index);
@@ -1178,7 +1213,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_RES_TO_RETURN) {
     while (1) {
       doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
-      doFilter(pInfo->pCondition, pBInfo->pRes);
+      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
 
       bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
       if (!hasRemain) {
@@ -1219,7 +1254,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
   blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
   while (1) {
     doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
-    doFilter(pInfo->pCondition, pBInfo->pRes);
+    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
 
     bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
     if (!hasRemain) {
@@ -1256,7 +1291,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
     blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
     while (1) {
       doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
-      doFilter(pInfo->pCondition, pBlock);
+      doFilter(pInfo->pCondition, pBlock, NULL);
 
       bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
       if (!hasRemain) {
@@ -1373,8 +1408,10 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock,
 
 static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput,
                            SSDataBlock* pBlock, SArray* pUpWins) {
-  SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
-  TSKEY*           tsCols = (TSKEY*)pTsCol->pData;
+  TSKEY*           startTsCols = (TSKEY*)pStartTsCol->pData;
+  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
+  TSKEY*           endTsCols = (TSKEY*)pEndTsCol->pData;
   uint64_t*        pGpDatas = NULL;
   if (pBlock->info.type == STREAM_RETRIEVE) {
     SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
@@ -1382,22 +1419,18 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval*
   }
   int32_t        step = 0;
   int32_t        startPos = 0;
+  for (int32_t i = 0; i < pBlock->info.rows; i++) {
     SResultRowInfo dumyInfo;
     dumyInfo.cur.pageId = -1;
-  STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[0], pInterval, TSDB_ORDER_ASC);
+    STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
-  while (1) {
+    while (win.ekey <= endTsCols[i]) {
-    step =
-        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
       uint64_t winGpId = pGpDatas ? pGpDatas[startPos] : pBlock->info.groupId;
       bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput);
       if (pUpWins && res) {
         SWinRes winRes = {.ts = win.skey, .groupId = winGpId};
         taosArrayPush(pUpWins, &winRes);
       }
-    int32_t prevEndPos = step - 1 + startPos;
+      getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
-    startPos = getNextQualifiedWindow(pInterval, &win, &pBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);
-    if (startPos < 0) {
-      break;
     }
   }
 }
@@ -1501,7 +1534,7 @@ static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlo
   }
   blockDataEnsureCapacity(pBlock, size - *index);
   SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
-  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, DELETE_GROUPID_COLUMN_INDEX);
+  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
   for (int32_t i = *index; i < size; i++) {
     SWinRes* pWin = taosArrayGet(pWins, i);
     colDataAppend(pTsCol, pBlock->info.rows, (const char*)&pWin->ts, false);
@@ -1562,6 +1595,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
       continue;
     }
 
+    if (pBlock->info.type == STREAM_NORMAL) {
+      //set input version
+      pTaskInfo->version = pBlock->info.version;
+    }
+
     if (pInfo->scalarSupp.pExprInfo != NULL) {
       SExprSupp* pExprSup = &pInfo->scalarSupp;
       projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
@@ -1645,6 +1683,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) {
     }
   }
   nodesDestroyNode((SNode*)pInfo->pPhyNode);
+  colDataDestroy(&pInfo->twAggSup.timeWindowData);
 
   taosMemoryFreeClear(param);
 }
@@ -1793,10 +1832,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
   pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
   pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes));
   pInfo->delIndex = 0;
-  // pInfo->pDelRes = createPullDataBlock(); todo(liuyao) for delete
+  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
-  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
-  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete
-
   initResultRowInfo(&pInfo->binfo.resultRowInfo);
 
   pOperator->name = "TimeIntervalAggOperator";
@@ -1969,7 +2005,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_RES_TO_RETURN) {
     while (1) {
       doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
-      doFilter(pInfo->pCondition, pBInfo->pRes);
+      doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
 
       bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
       if (!hasRemain) {
@@ -2013,7 +2049,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
   blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
   while (1) {
     doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
-    doFilter(pInfo->pCondition, pBInfo->pRes);
+    doFilter(pInfo->pCondition, pBInfo->pRes, NULL);
 
     bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo);
     if (!hasRemain) {
@@ -2585,7 +2621,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
       if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) {
         SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId};
         // add pull data request
-        taosArrayPush(pInfo->pPullWins, &pull);
+        savePullWindow(&pull, pInfo->pPullWins);
         int32_t size = taosArrayGetSize(pInfo->pChildren);
         addPullWindow(pInfo->pPullDataMap, &winRes, size);
         qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size);
@@ -2598,14 +2634,6 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
           chId = getChildIndex(pSDataBlock);
           index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);
         }
-        // if (index != -1 && pSDataBlock->info.type == STREAM_PULL_DATA) {
-        //   qDebug("===stream===delete child id %d", chId);
-        //   taosArrayRemove(chArray, index);
-        //   if (taosArrayGetSize(chArray) == 0) {
-        //     // pull data is over
-        //     taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
-        //   }
-        // }
         if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {
           ignore = false;
         }
@@ -2681,14 +2709,6 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol
   blockDataUpdateTsWindow(pDest, 0);
 }
 
-static bool needBreak(SStreamFinalIntervalOperatorInfo* pInfo) {
-  int32_t size = taosArrayGetSize(pInfo->pPullWins);
-  if (pInfo->pullIndex < size) {
-    return true;
-  }
-  return false;
-}
-
 static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {
   clearSpecialDataBlock(pBlock);
   int32_t size = taosArrayGetSize(array);
@@ -2697,16 +2717,18 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB
   }
   blockDataEnsureCapacity(pBlock, size - (*pIndex));
   ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock));
+  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
+  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
+  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
+  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
+  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
   for (; (*pIndex) < size; (*pIndex)++) {
     SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));
-    SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
     colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
-
-    SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
     colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
-
-    SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
     colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);
+    colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);
+    colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);
     pBlock->info.rows++;
   }
   if ((*pIndex) == size) {
@@ -2753,6 +2775,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
   } else if (pOperator->status == OP_RES_TO_RETURN) {
+    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
+    if (pInfo->pPullDataRes->info.rows != 0) {
+      // process the rest of the data
+      ASSERT(IS_FINAL_OP(pInfo));
+      printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+      return pInfo->pPullDataRes;
+    }
+
     doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
     if (pInfo->binfo.pRes->info.rows == 0) {
       pOperator->status = OP_EXEC_DONE;
@@ -2767,11 +2797,13 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
     printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
     return pInfo->binfo.pRes;
   } else {
+    if (!IS_FINAL_OP(pInfo)) {
       doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
       if (pInfo->binfo.pRes->info.rows != 0) {
         printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
         return pInfo->binfo.pRes;
       }
+    }
     if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) {
       pInfo->returnUpdate = false;
       ASSERT(!IS_FINAL_OP(pInfo));
@@ -2779,13 +2811,13 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
       // process the rest of the data
       return pInfo->pUpdateRes;
     }
-    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
+    // doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
-    if (pInfo->pPullDataRes->info.rows != 0) {
+    // if (pInfo->pPullDataRes->info.rows != 0) {
-      // process the rest of the data
+    //   // process the rest of the data
-      ASSERT(IS_FINAL_OP(pInfo));
+    //   ASSERT(IS_FINAL_OP(pInfo));
-      printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+    //   printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
-      return pInfo->pPullDataRes;
+    //   return pInfo->pPullDataRes;
-    }
+    // }
     doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
     if (pInfo->pDelRes->info.rows != 0) {
       // process the rest of the data
@@ -2825,7 +2857,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
         continue;
       }
       removeResults(pUpWins, pUpdated);
-      copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
+      copyDataBlock(pInfo->pUpdateRes, pBlock);
+      // copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex);
       pInfo->returnUpdate = true;
       taosArrayDestroy(pUpWins);
       break;
@@ -2884,10 +2917,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
       SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info;
       setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true);
       doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL);
-
-      if (needBreak(pInfo)) {
-        break;
-      }
     }
   }
 
@@ -2901,6 +2930,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
   finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
   initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+
+  doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
+  if (pInfo->pPullDataRes->info.rows != 0) {
+    // process the rest of the data
+    ASSERT(IS_FINAL_OP(pInfo));
+    printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
+    return pInfo->pPullDataRes;
+  }
+
   doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
   if (pInfo->binfo.pRes->info.rows != 0) {
     printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
@@ -2915,14 +2953,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
     return pInfo->pUpdateRes;
   }
 
-  doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);
-  if (pInfo->pPullDataRes->info.rows != 0) {
-    // process the rest of the data
-    ASSERT(IS_FINAL_OP(pInfo));
-    printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi");
-    return pInfo->pPullDataRes;
-  }
-
   doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
   if (pInfo->pDelRes->info.rows != 0) {
     // process the rest of the data
@@ -2933,15 +2963,16 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
   return NULL;
 }
 
-SSDataBlock* createPullDataBlock() {
+SSDataBlock* createSpecialDataBlock(EStreamType type) {
   SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
   pBlock->info.hasVarCol = false;
   pBlock->info.groupId = 0;
   pBlock->info.rows = 0;
-  pBlock->info.type = STREAM_RETRIEVE;
+  pBlock->info.type = type;
-  pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t);
+  pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) +
+      sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY);
 
-  pBlock->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData));
+  pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData));
   SColumnInfoData infoData = {0};
   infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP;
   infoData.info.bytes = sizeof(TSKEY);
@@ -2952,6 +2983,14 @@ SSDataBlock* createPullDataBlock() {
 
   infoData.info.type = TSDB_DATA_TYPE_UBIGINT;
   infoData.info.bytes = sizeof(uint64_t);
+  // uid
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+  // group id
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  // calculate start ts
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+  // calculate end ts
   taosArrayPush(pBlock->pDataBlock, &infoData);
 
   return pBlock;
@@ -3019,8 +3058,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
       goto _error;
     }
   }
-  pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
+  pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
-  pInfo->pUpdateRes->info.type = STREAM_CLEAR;
   blockDataEnsureCapacity(pInfo->pUpdateRes, 128);
   pInfo->returnUpdate = false;
 
@@ -3042,11 +3080,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   pInfo->pullIndex = 0;
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
-  pInfo->pPullDataRes = createPullDataBlock();
+  pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
   pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
-  // pInfo->pDelRes = createPullDataBlock(); // todo(liuyao) for delete
+  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
-  pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
-  pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete
   pInfo->delIndex = 0;
   pInfo->pDelWins = taosArrayInit(4, sizeof(SWinRes));
 
@@ -3061,7 +3097,9 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
   pOperator->fpSet =
       createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo,
                           aggEncodeResultRow, aggDecodeResultRow, NULL);
-
+  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
+    initIntervalDownStream(downstream, pPhyNode->type);
+  }
   code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
@@ -3086,6 +3124,7 @@ void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {
   }
   taosHashCleanup(pSup->pResultRows);
   destroyDiskbasedBuf(pSup->pResultBuf);
+  blockDataDestroy(pSup->pScanBlock);
 }
 
 void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
@@ -3103,6 +3142,10 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) {
       taosMemoryFreeClear(pChInfo);
     }
   }
+  colDataDestroy(&pInfo->twAggSup.timeWindowData);
+  blockDataDestroy(pInfo->pDelRes);
+  blockDataDestroy(pInfo->pWinBlock);
+  blockDataDestroy(pInfo->pUpdateRes);
 
   taosMemoryFreeClear(param);
 }
@@ -3200,7 +3243,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
   pInfo->pDelIterator = NULL;
-  // pInfo->pDelRes = createPullDataBlock();
+  // pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
   pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
   pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete
   pInfo->pChildren = NULL;
@@ -3559,7 +3602,7 @@ static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc
   TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;
   SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
   TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;
-  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
+  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
   uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;
   for (int32_t i = 0; i < pBlock->info.rows; i++) {
     int32_t winIndex = 0;
@@ -4255,7 +4298,6 @@ static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc
     step = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCol, pKeyColInfo, pBlock->info.rows, i, &allEqual,
                                  pSeDeleted);
     ASSERT(isTsInWindow(pCurWin, tsCol[i]) || isEqualStateKey(pCurWin, pKeyData));
-    taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win);
     taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
     deleteWindow(pAggSup->pCurWins, winIndex);
   }
@@ -4282,6 +4324,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
   }
   
   SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
+  blockDataEnsureCapacity(pAggSup->pScanBlock, pSDataBlock->info.rows);
   SColumnInfoData*     pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
   for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
     if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
@@ -4296,7 +4339,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
     winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, pKeyColInfo, pSDataBlock->info.rows, i,
                                     &allEqual, pInfo->pSeDeleted);
     if (!allEqual) {
-      taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win);
+      appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey,
+          &pSDataBlock->info.groupId);
       taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition));
       deleteWindow(pAggSup->pCurWins, winIndex);
       continue;
@@ -4460,7 +4504,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
   pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
   pInfo->pDelIterator = NULL;
-  // pInfo->pDelRes = createPullDataBlock(); // todo(liuyao) for delete
+  // pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
   pInfo->pDelRes = createOneDataBlock(pInfo->binfo.pRes, false);  // todo(liuyao) for delete
   pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;               // todo(liuyao) for delete
   pInfo->pChildren = NULL;
@@ -4626,7 +4670,7 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) {
       getTableScanInfo(pOperator, &iaInfo->order, &scanFlag);
       setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->order, scanFlag, true);
       doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes);
-      doFilter(miaInfo->pCondition, pRes);
+      doFilter(miaInfo->pCondition, pRes, NULL);
       if (pRes->info.rows >= pOperator->resultInfo.capacity) {
         break;
       }

source/libs/function/src/builtins.c

@@ -2242,7 +2242,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "last_row",
     .type = FUNCTION_TYPE_LAST_ROW,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateFirstLast,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -2253,7 +2253,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_cache_last_row",
     .type = FUNCTION_TYPE_CACHE_LAST_ROW,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateFirstLast,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -2263,7 +2263,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "first",
     .type = FUNCTION_TYPE_FIRST,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateFirstLast,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -2277,7 +2277,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_first_partial",
     .type = FUNCTION_TYPE_FIRST_PARTIAL,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateFirstLastPartial,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -2288,7 +2288,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_first_merge",
     .type = FUNCTION_TYPE_FIRST_MERGE,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateFirstLastMerge,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -2299,7 +2299,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "last",
     .type = FUNCTION_TYPE_LAST,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateFirstLast,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -2313,7 +2313,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_last_partial",
     .type = FUNCTION_TYPE_LAST_PARTIAL,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateFirstLastPartial,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,
@@ -2324,7 +2324,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_last_merge",
     .type = FUNCTION_TYPE_LAST_MERGE,
-    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
+    .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateFirstLastMerge,
     .getEnvFunc   = getFirstLastFuncEnv,
     .initFunc     = functionSetup,

source/libs/function/src/builtinsimpl.c

@@ -476,16 +476,16 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
 
 int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
   SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx);
-  char*                pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
+  SFirstLastRes*       pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
   int32_t              type = pDestCtx->input.pData[0]->info.type;
   int32_t              bytes = pDestCtx->input.pData[0]->info.bytes;
 
   SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
-  char*                pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
+  SFirstLastRes*       pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
 
-  if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || *(TSKEY*)(pDBuf + bytes) > *(TSKEY*)(pSBuf + bytes))) {
+  if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || pDBuf->ts > pSBuf->ts)) {
-    memcpy(pDBuf, pSBuf, bytes);
+    memcpy(pDBuf->buf, pSBuf->buf, bytes);
-    *(TSKEY*)(pDBuf + bytes) = *(TSKEY*)(pSBuf + bytes);
+    pDBuf->ts = pSBuf->ts;
     pDResInfo->numOfRes = 1;
   }
   return TSDB_CODE_SUCCESS;
@@ -2552,8 +2552,7 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo*
       memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN);
       pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo));
 
-      qDebug("merge histo, total:%"PRId64", entry:%d, %p", pHisto->numOfElems, pHisto->numOfEntries,
+      qDebug("merge histo, total:%" PRId64 ", entry:%d, %p", pHisto->numOfElems, pHisto->numOfEntries, pHisto);
-          pHisto);
       tHistogramDestroy(&pRes);
     }
   }
@@ -2580,7 +2579,8 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) {
   }
 
   if (pInfo->algo != APERCT_ALGO_TDIGEST) {
-    qDebug("after merge, total:%d, numOfEntry:%d, %p", pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto);
+    qDebug("after merge, total:%d, numOfEntry:%d, %p", pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries,
+           pInfo->pHisto);
   }
 
   SET_VAL(pResInfo, 1, 1);
@@ -2602,7 +2602,8 @@ int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
   } else {
     buildHistogramInfo(pInfo);
     if (pInfo->pHisto->numOfElems > 0) {
-      qDebug("get the final res:%d, elements:%"PRId64", entry:%d", pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries);
+      qDebug("get the final res:%d, elements:%" PRId64 ", entry:%d", pInfo->pHisto->numOfElems,
+             pInfo->pHisto->numOfEntries);
 
       double  ratio[] = {pInfo->percent};
       double* res = tHistogramUniform(pInfo->pHisto, ratio, 1);
@@ -2656,7 +2657,6 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx)
 
   SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
   SAPercentileInfo*    pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
-  ASSERT(pDBuf->algo == pSBuf->algo);
 
   qDebug("start to combine apercentile, %p", pDBuf->pHisto);
 
@@ -2994,16 +2994,16 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
 // todo rewrite:
 int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
   SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx);
-  char*                pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
+  SFirstLastRes*       pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
   int32_t              type = pDestCtx->input.pData[0]->info.type;
   int32_t              bytes = pDestCtx->input.pData[0]->info.bytes;
 
   SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
-  char*                pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
+  SFirstLastRes*       pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
 
-  if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || *(TSKEY*)(pDBuf + bytes) < *(TSKEY*)(pSBuf + bytes))) {
+  if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || pDBuf->ts < pSBuf->ts)) {
-    memcpy(pDBuf, pSBuf, bytes);
+    memcpy(pDBuf->buf, pSBuf->buf, bytes);
-    *(TSKEY*)(pDBuf + bytes) = *(TSKEY*)(pSBuf + bytes);
+    pDBuf->ts = pSBuf->ts;
     pDResInfo->numOfRes = 1;
   }
   return TSDB_CODE_SUCCESS;
@@ -3879,11 +3879,11 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) {
       if (pCtx->end.key != INT64_MIN) {
         pInfo->min = pCtx->end.key;
       } else {
-        pInfo->min = ptsList[0];
+        pInfo->min = ptsList[start];
       }
     } else {
       if (pCtx->start.key == INT64_MIN) {
-        pInfo->min = (pInfo->min > ptsList[0]) ? ptsList[0] : pInfo->min;
+        pInfo->min = (pInfo->min > ptsList[start]) ? ptsList[start] : pInfo->min;
       } else {
         pInfo->min = pCtx->start.key;
       }
@@ -4665,10 +4665,8 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) {
   SSumRes*             pSumRes = GET_ROWCELL_INTERBUF(pResInfo);
 
   SInputColumnInfoData* pInput = &pCtx->input;
-  TSKEY*                tsList = (int64_t*)pInput->pPTS->pData;
 
   SColumnInfoData* pInputCol = pInput->pData[0];
-  SColumnInfoData* pTsOutput = pCtx->pTsOutput;
   SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
 
   int32_t numOfElems = 0;
@@ -4704,11 +4702,6 @@ int32_t csumFunction(SqlFunctionCtx* pCtx) {
       }
     }
 
-    // TODO: remove this after pTsOutput is handled
-    if (pTsOutput != NULL) {
-      colDataAppendInt64(pTsOutput, pos, &tsList[i]);
-    }
-
     numOfElems++;
   }
 

source/libs/function/src/tudf.c

@@ -875,7 +875,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf);
 void udfcUvHandleRsp(SClientUvConn *conn);
 void udfcUvHandleError(SClientUvConn *conn);
 void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf);
-void onUdfcPipetWrite(uv_write_t *write, int status);
+void onUdfcPipeWrite(uv_write_t *write, int status);
 void onUdfcPipeConnect(uv_connect_t *connect, int status);
 int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask);
 int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask);
@@ -1244,6 +1244,9 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf
       buf->base = NULL;
       buf->len = 0;
     }
+  } else if (connBuf->total == -1 && connBuf->len < msgHeadSize) {
+    buf->base = connBuf->buf + connBuf->len;
+    buf->len = msgHeadSize - connBuf->len;
   } else {
     connBuf->cap = connBuf->total > connBuf->cap ? connBuf->total : connBuf->cap;
     void *resultBuf = taosMemoryRealloc(connBuf->buf, connBuf->cap);
@@ -1258,8 +1261,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf
     }
   }
 
-  fnTrace("conn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total);
+  fnDebug("udfc uv alloc buffer: cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total);
-
 }
 
 bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) {
@@ -1267,7 +1269,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) {
     connBuf->total = *(int32_t *) (connBuf->buf);
   }
   if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) {
-    fnTrace("udfc complete message is received, now handle it");
+    fnDebug("udfc complete message is received, now handle it");
     return true;
   }
   return false;
@@ -1278,7 +1280,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
   int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen then seqnum
 
   if (QUEUE_EMPTY(&conn->taskQueue)) {
-    fnError("udfc no task waiting for response on connection");
+    fnError("udfc no task waiting on connection. response seqnum:%"PRId64, seqNum);
     return;
   }
   bool found = false;
@@ -1287,6 +1289,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
   SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue);
 
   while (h != &conn->taskQueue) {
+    fnDebug("udfc handle response iterate through queue. uvTask:%d-%p", task->seqNum, task);
     if (task->seqNum == seqNum) {
       if (found == false) {
         found = true;
@@ -1315,6 +1318,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
 }
 
 void udfcUvHandleError(SClientUvConn *conn) {
+  fnDebug("handle error on conn: %p, pipe: %p", conn, conn->pipe);
   while (!QUEUE_EMPTY(&conn->taskQueue)) {
     QUEUE* h = QUEUE_HEAD(&conn->taskQueue);
     SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue);
@@ -1328,7 +1332,7 @@ void udfcUvHandleError(SClientUvConn *conn) {
 }
 
 void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
-  fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread);
+  fnDebug("udfc client %p, client read from pipe. nread: %zd", client, nread);
   if (nread == 0) return;
 
   SClientUvConn *conn = client->data;
@@ -1338,31 +1342,25 @@ void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
     if (isUdfcUvMsgComplete(connBuf)) {
       udfcUvHandleRsp(conn);
     }
-
   }
   if (nread < 0) {
-    fnError("udfc client pipe %p read error: %zd, %s.", client, nread, uv_strerror(nread));
+    fnError("udfc client pipe %p read error: %zd(%s).", client, nread, uv_strerror(nread));
     if (nread == UV_EOF) {
       fnError("\tudfc client pipe %p closed", client);
     }
     udfcUvHandleError(conn);
   }
-
 }
 
-void onUdfcPipetWrite(uv_write_t *write, int status) {
+void onUdfcPipeWrite(uv_write_t *write, int status) {
-  SClientUvTaskNode *uvTask = write->data;
+  SClientUvConn *conn = write->data;
-  uv_pipe_t *pipe = uvTask->pipe;
+  if (status < 0) {
-  fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len);
+    fnError("udfc client connection %p write failed. status: %d(%s)", conn, status, uv_strerror(status));
-  SClientUvConn *conn = pipe->data;
-  if (status == 0) {
-    QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue);
-  } else {
-    fnError("udfc client %p write error.", pipe);
     udfcUvHandleError(conn);
+  } else {
+    fnDebug("udfc client connection %p write succeed", conn);
   }
   taosMemoryFree(write);
-  taosMemoryFree(uvTask->reqBuf.base);
 }
 
 void onUdfcPipeConnect(uv_connect_t *connect, int status) {
@@ -1419,7 +1417,7 @@ int32_t udfcCreateUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN
 }
 
 int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) {
-  fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask);
+  fnDebug("queue uv task to event loop, uvTask: %d-%p", uvTask->type, uvTask);
   SUdfcProxy *udfc = uvTask->udfc;
   uv_mutex_lock(&udfc->taskQueueMutex);
   QUEUE_INSERT_TAIL(&udfc->taskQueue, &uvTask->recvTaskQueue);
@@ -1427,14 +1425,14 @@ int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) {
   uv_async_send(&udfc->loopTaskAync);
 
   uv_sem_wait(&uvTask->taskSem);
-  fnInfo("udfc uv task finished. task: %d, %p", uvTask->type, uvTask);
+  fnInfo("udfc uvTask finished. uvTask:%"PRId64"-%d-%p", uvTask->seqNum, uvTask->type, uvTask);
   uv_sem_destroy(&uvTask->taskSem);
 
   return 0;
 }
 
 int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
-  fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask);
+  fnDebug("event loop start uv task. uvTask: %"PRId64"-%d-%p", uvTask->seqNum, uvTask->type, uvTask);
   int32_t code = 0;
 
   switch (uvTask->type) {
@@ -1465,10 +1463,12 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
         code = TSDB_CODE_UDF_PIPE_NO_PIPE;
       } else {
         uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t));
-        write->data = uvTask;
+        write->data = pipe->data;
-        int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipetWrite);
+        QUEUE* connTaskQueue = &((SClientUvConn*)pipe->data)->taskQueue;
+        QUEUE_INSERT_TAIL(connTaskQueue, &uvTask->connTaskQueue);
+        int err = uv_write(write, (uv_stream_t *)pipe, &uvTask->reqBuf, 1, onUdfcPipeWrite);
         if (err != 0) {
-          fnError("udfc event loop start req/rsp task uv_write failed. code: %s", uv_strerror(err));
+          fnError("udfc event loop start req_rsp task uv_write failed. uvtask: %p, code: %s", uvTask, uv_strerror(err));
         }
         code = err;
       }
@@ -1618,6 +1618,7 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) {
   SClientUvTaskNode *uvTask = NULL;
 
   udfcCreateUvTask(task, uvTaskType, &uvTask);
+  fnDebug("udfc client task: %p created uvTask: %p. pipe: %p", task, uvTask, task->session->udfUvPipe);
   udfcQueueUvTask(uvTask);
   udfcGetUdfTaskResultFromUvTask(task, uvTask);
   if (uvTaskType == UV_TASK_CONNECT) {
@@ -1625,6 +1626,8 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) {
     SClientUvConn *conn = uvTask->pipe->data;
     conn->session = task->session;
   }
+  taosMemoryFree(uvTask->reqBuf.base);
+  uvTask->reqBuf.base = NULL;
   taosMemoryFree(uvTask);
   uvTask = NULL;
   return task->errCode;
@@ -1670,7 +1673,7 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
 
 int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
                 SSDataBlock* output, SUdfInterBuf *newState) {
-  fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle);
+  fnDebug("udfc call udf. callType: %d, funcHandle: %p", callType, handle);
   SUdfcUvSession *session = (SUdfcUvSession *) handle;
   if (session->udfUvPipe == NULL) {
     fnError("No pipe to udfd");

source/libs/function/src/udfd.c

@@ -671,6 +671,9 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
       fnError("udfd can not allocate enough memory") buf->base = NULL;
       buf->len = 0;
     }
+  } else if (ctx->inputTotal == -1 && ctx->inputLen < msgHeadSize) {
+      buf->base = ctx->inputBuf + ctx->inputLen;
+      buf->len = msgHeadSize - ctx->inputLen;
   } else {
     ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap;
     void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap);

source/libs/nodes/src/nodesCodeFuncs.c

@@ -2667,6 +2667,7 @@ static int32_t jsonToExprNode(const SJson* pJson, void* pObj) {
 }
 
 static const char* jkColumnTableId = "TableId";
+static const char* jkColumnTableType = "TableType";
 static const char* jkColumnColId = "ColId";
 static const char* jkColumnColType = "ColType";
 static const char* jkColumnDbName = "DbName";
@@ -2683,6 +2684,9 @@ static int32_t columnNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkColumnTableId, pNode->tableId);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkColumnTableType, pNode->tableType);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkColumnColId, pNode->colId);
   }
@@ -2718,6 +2722,9 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetUBigIntValue(pJson, jkColumnTableId, &pNode->tableId);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetTinyIntValue(pJson, jkColumnTableType, &pNode->tableType);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetSmallIntValue(pJson, jkColumnColId, &pNode->colId);
   }

source/libs/parser/src/parTranslater.c

@@ -1237,7 +1237,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) {
     pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType);
     pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType);
     pSelect->hasLastRowFunc = pSelect->hasLastRowFunc ? true : (FUNCTION_TYPE_LAST_ROW == pFunc->funcType);
-    pSelect->hasTimeLineFunc = pSelect->hasLastRowFunc ? true : fmIsTimelineFunc(pFunc->funcId);
+    pSelect->hasTimeLineFunc = pSelect->hasTimeLineFunc ? true : fmIsTimelineFunc(pFunc->funcId);
   }
 }
 

source/libs/planner/src/planLogicCreater.c

@@ -478,8 +478,9 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
   }
 
   pAgg->hasLastRow = pSelect->hasLastRowFunc;
+  pAgg->hasTimeLineFunc = pSelect->hasTimeLineFunc;
   pAgg->node.groupAction = GROUP_ACTION_SET;
-  pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
+  pAgg->node.requireDataOrder = pAgg->hasTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE;
   pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE;
 
   int32_t code = TSDB_CODE_SUCCESS;
@@ -786,6 +787,14 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
   return code;
 }
 
+static bool isPrimaryKeySort(SNodeList* pOrderByList) {
+  SNode* pExpr = ((SOrderByExprNode*)nodesListGetNode(pOrderByList, 0))->pExpr;
+  if (QUERY_NODE_COLUMN != nodeType(pExpr)) {
+    return false;
+  }
+  return PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pExpr)->colId;
+}
+
 static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
   if (NULL == pSelect->pOrderByList) {
     return TSDB_CODE_SUCCESS;
@@ -799,7 +808,9 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
   pSort->groupSort = pSelect->groupSort;
   pSort->node.groupAction = pSort->groupSort ? GROUP_ACTION_KEEP : GROUP_ACTION_CLEAR;
   pSort->node.requireDataOrder = DATA_ORDER_LEVEL_NONE;
-  pSort->node.resultDataOrder = pSort->groupSort ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL;
+  pSort->node.resultDataOrder = isPrimaryKeySort(pSelect->pOrderByList)
+                                    ? (pSort->groupSort ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL)
+                                    : DATA_ORDER_LEVEL_NONE;
 
   int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets);
   if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) {

source/libs/planner/src/planSpliter.c

@@ -215,6 +215,14 @@ static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) {
   return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild));
 }
 
+static bool stbSplIsMultiTbScanChild(bool streamQuery, SLogicNode* pNode) {
+  if (1 != LIST_LENGTH(pNode->pChildren)) {
+    return false;
+  }
+  SNode* pChild = nodesListGetNode(pNode->pChildren, 0);
+  return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild));
+}
+
 static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) {
   SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode;
   if (WINDOW_TYPE_INTERVAL == pWindow->winType) {
@@ -247,7 +255,7 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) {
     case QUERY_NODE_LOGIC_PLAN_JOIN:
       return !(((SJoinLogicNode*)pNode)->isSingleTableJoin);
     case QUERY_NODE_LOGIC_PLAN_PARTITION:
-      return stbSplHasMultiTbScan(streamQuery, pNode);
+      return stbSplIsMultiTbScanChild(streamQuery, pNode);
     case QUERY_NODE_LOGIC_PLAN_AGG:
       return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
     case QUERY_NODE_LOGIC_PLAN_WINDOW:
@@ -904,14 +912,6 @@ static int32_t stbSplSplitScanNodeWithPartTags(SSplitContext* pCxt, SStableSplit
   return code;
 }
 
-static int32_t stbSplSplitScanNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
-  SScanLogicNode* pScan = (SScanLogicNode*)pInfo->pSplitNode;
-  if (NULL != pScan->pGroupTags) {
-    return stbSplSplitScanNodeWithPartTags(pCxt, pInfo);
-  }
-  return stbSplSplitScanNodeWithoutPartTags(pCxt, pInfo);
-}
-
 static SNode* stbSplFindPrimaryKeyFromScan(SScanLogicNode* pScan) {
   SNode* pCol = NULL;
   FOREACH(pCol, pScan->pScanCols) {
@@ -922,11 +922,12 @@ static SNode* stbSplFindPrimaryKeyFromScan(SScanLogicNode* pScan) {
   return NULL;
 }
 
-static int32_t stbSplSplitScanNodeForJoin(SSplitContext* pCxt, SLogicSubplan* pSubplan, SScanLogicNode* pScan) {
+static int32_t stbSplSplitMergeScanNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SScanLogicNode* pScan,
+                                        bool groupSort) {
   SNodeList* pMergeKeys = NULL;
   int32_t    code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pScan), &pMergeKeys);
   if (TSDB_CODE_SUCCESS == code) {
-    code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan, false);
+    code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan, groupSort);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesListMakeStrictAppend(&pSubplan->pChildren,
@@ -937,12 +938,24 @@ static int32_t stbSplSplitScanNodeForJoin(SSplitContext* pCxt, SLogicSubplan* pS
   return code;
 }
 
+static int32_t stbSplSplitScanNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
+  SScanLogicNode* pScan = (SScanLogicNode*)pInfo->pSplitNode;
+  if (SCAN_TYPE_TABLE_MERGE == pScan->scanType) {
+    pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
+    return stbSplSplitMergeScanNode(pCxt, pInfo->pSubplan, pScan, true);
+  }
+  if (NULL != pScan->pGroupTags) {
+    return stbSplSplitScanNodeWithPartTags(pCxt, pInfo);
+  }
+  return stbSplSplitScanNodeWithoutPartTags(pCxt, pInfo);
+}
+
 static int32_t stbSplSplitJoinNodeImpl(SSplitContext* pCxt, SLogicSubplan* pSubplan, SJoinLogicNode* pJoin) {
   int32_t code = TSDB_CODE_SUCCESS;
   SNode*  pChild = NULL;
   FOREACH(pChild, pJoin->node.pChildren) {
     if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild)) {
-      code = stbSplSplitScanNodeForJoin(pCxt, pSubplan, (SScanLogicNode*)pChild);
+      code = stbSplSplitMergeScanNode(pCxt, pSubplan, (SScanLogicNode*)pChild, false);
     } else if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pChild)) {
       code = stbSplSplitJoinNodeImpl(pCxt, pSubplan, (SJoinLogicNode*)pChild);
     } else {
@@ -964,8 +977,28 @@ static int32_t stbSplSplitJoinNode(SSplitContext* pCxt, SStableSplitInfo* pInfo)
   return code;
 }
 
+static int32_t stbSplCreateMergeKeysForPartitionNode(SLogicNode* pPart, SNodeList** pMergeKeys) {
+  SNode* pPrimaryKey =
+      nodesCloneNode(stbSplFindPrimaryKeyFromScan((SScanLogicNode*)nodesListGetNode(pPart->pChildren, 0)));
+  if (NULL == pPrimaryKey) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  int32_t code = nodesListAppend(pPart->pTargets, pPrimaryKey);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = stbSplCreateMergeKeysByPrimaryKey(pPrimaryKey, pMergeKeys);
+  }
+  return code;
+}
+
 static int32_t stbSplSplitPartitionNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
-  int32_t code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, NULL, pInfo->pSplitNode, true);
+  int32_t    code = TSDB_CODE_SUCCESS;
+  SNodeList* pMergeKeys = NULL;
+  if (pInfo->pSplitNode->requireDataOrder >= DATA_ORDER_LEVEL_IN_GROUP) {
+    code = stbSplCreateMergeKeysForPartitionNode(pInfo->pSplitNode, &pMergeKeys);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pInfo->pSplitNode, true);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
                                      (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT));

source/libs/planner/src/planUtil.c

@@ -124,7 +124,7 @@ int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode*
 }
 
 static int32_t adjustScanDataRequirement(SScanLogicNode* pScan, EDataOrderLevel requirement) {
-  if (SCAN_TYPE_TABLE != pScan->scanType || SCAN_TYPE_TABLE_MERGE != pScan->scanType) {
+  if (SCAN_TYPE_TABLE != pScan->scanType && SCAN_TYPE_TABLE_MERGE != pScan->scanType) {
     return TSDB_CODE_SUCCESS;
   }
   // The lowest sort level of scan output data is DATA_ORDER_LEVEL_IN_BLOCK
@@ -161,7 +161,9 @@ static int32_t adjustAggDataRequirement(SAggLogicNode* pAgg, EDataOrderLevel req
     return TSDB_CODE_PLAN_INTERNAL_ERROR;
   }
   pAgg->node.resultDataOrder = requirement;
-  pAgg->node.requireDataOrder = requirement;
+  if (pAgg->hasTimeLineFunc) {
+    pAgg->node.requireDataOrder = requirement < DATA_ORDER_LEVEL_IN_GROUP ? DATA_ORDER_LEVEL_IN_GROUP : requirement;
+  }
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/planner/test/planBasicTest.cpp

@@ -139,6 +139,10 @@ TEST_F(PlanBasicTest, timeLineFunc) {
   run("SELECT CSUM(c1) FROM t1");
 
   run("SELECT CSUM(c1) FROM st1");
+
+  run("SELECT TWA(c1) FROM t1");
+
+  run("SELECT TWA(c1) FROM st1");
 }
 
 TEST_F(PlanBasicTest, multiResFunc) {

source/libs/planner/test/planPartByTest.cpp

@@ -58,7 +58,21 @@ TEST_F(PlanPartitionByTest, withInterval) {
 TEST_F(PlanPartitionByTest, withGroupBy) {
   useDb("root", "test");
 
-  run("select count(*) from t1 partition by c1 group by c2");
+  run("SELECT COUNT(*) FROM t1 PARTITION BY c1 GROUP BY c2");
 
   run("SELECT TBNAME, c1 FROM st1 PARTITION BY TBNAME GROUP BY c1");
 }
+
+TEST_F(PlanPartitionByTest, withTimeLineFunc) {
+  useDb("root", "test");
+
+  run("SELECT TWA(c1) FROM st1 PARTITION BY c1");
+
+  run("SELECT MAVG(c1, 2) FROM st1 PARTITION BY c1");
+}
+
+TEST_F(PlanPartitionByTest, withSlimit) {
+  useDb("root", "test");
+
+  run("SELECT CSUM(c1) FROM st1 PARTITION BY TBNAME SLIMIT 1");
+}

source/libs/scalar/src/filter.c

@@ -3671,6 +3671,22 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) {
         SNode *t = node->pLeft;
         node->pLeft = node->pRight;
         node->pRight = t;
+        switch (node->opType) {
+          case OP_TYPE_GREATER_THAN:
+            node->opType = OP_TYPE_LOWER_THAN;
+            break;
+          case OP_TYPE_LOWER_THAN:
+            node->opType = OP_TYPE_GREATER_THAN;
+            break;
+          case OP_TYPE_GREATER_EQUAL:
+            node->opType = OP_TYPE_LOWER_EQUAL;
+            break;
+          case OP_TYPE_LOWER_EQUAL:
+            node->opType = OP_TYPE_GREATER_EQUAL;
+            break;
+          default:
+            break;
+        }
       }
 
       if (OP_TYPE_IN == node->opType && QUERY_NODE_NODE_LIST != nodeType(node->pRight)) {

source/libs/scalar/src/sclfunc.c

@@ -5,6 +5,7 @@
 #include "tdatablock.h"
 #include "tjson.h"
 #include "ttime.h"
+#include "cJSON.h"
 #include "vnode.h"
 
 typedef float (*_float_fn)(float);
@@ -1722,15 +1723,10 @@ int32_t winEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p
 int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   ASSERT(inputNum == 1);
 
-  SMetaReader mr = {0};
-  metaReaderInit(&mr, pInput->param, 0);
-
   uint64_t uid = *(uint64_t *)colDataGetData(pInput->columnData, 0);
-  metaGetTableEntryByUid(&mr, uid);
 
   char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
-  STR_TO_VARSTR(str, mr.me.name);
+  metaGetTableNameByUid(pInput->param, uid, str);
-  metaReaderClear(&mr);
 
   for(int32_t i = 0; i < pInput->numOfRows; ++i) {
     colDataAppend(pOutput->columnData, pOutput->numOfRows + i, str, false);

source/libs/scalar/test/CMakeLists.txt

@@ -1,4 +1,4 @@
 enable_testing()
 
-add_subdirectory(filter)
+#add_subdirectory(filter)
 add_subdirectory(scalar)

source/libs/scalar/test/filter/CMakeLists.txt

@@ -9,7 +9,7 @@ IF(NOT TD_DARWIN)
         ADD_EXECUTABLE(filterTest ${SOURCE_LIST})
         TARGET_LINK_LIBRARIES(
                 filterTest
-                PUBLIC os util common gtest qcom function nodes scalar
+                PUBLIC os util common gtest qcom function nodes scalar parser catalog transport
         )
 
         TARGET_INCLUDE_DIRECTORIES(

source/libs/scalar/test/filter/filterTests.cpp

@@ -44,7 +44,6 @@
 #include "scalar.h"
 #include "stub.h"
 #include "taos.h"
-#include "tdatablock.h"
 #include "tdef.h"
 #include "tlog.h"
 #include "tvariant.h"

source/libs/stream/src/streamUpdate.c

@@ -207,6 +207,7 @@ void updateInfoDestroy(SUpdateInfo *pInfo) {
   }
 
   taosArrayDestroy(pInfo->pTsSBFs);
+  taosHashCleanup(pInfo->pMap);
   taosMemoryFree(pInfo);
 }
 

source/libs/sync/test/sh/a.sh

@@ -22,7 +22,7 @@ done
 
 echo ""
 echo "generate vgId ..."
-cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | awk -F, '{print $1}' | sort | uniq > ${logpath}/log.vgIds.tmp
+cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | sort | uniq > ${logpath}/log.vgIds.tmp
 echo "all vgIds:" > ${logpath}/log.vgIds
 cat ${logpath}/log.dnode* | grep "vgId:" | grep -v ERROR | awk '{print $5}' | awk -F, '{print $1}' | sort | uniq >> ${logpath}/log.vgIds
 for dnode in `ls ${logpath} | grep dnode | grep -v log`;do

source/libs/transport/inc/transComm.h

@@ -396,6 +396,7 @@ typedef struct SDelayQueue {
 int         transDQCreate(uv_loop_t* loop, SDelayQueue** queue);
 void        transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg));
 SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs);
+void        transDQCancel(SDelayQueue* queue, SDelayTask* task);
 
 bool transEpSetIsEqual(SEpSet* a, SEpSet* b);
 /*

source/libs/transport/src/transCli.c

@@ -27,7 +27,6 @@ typedef struct SCliConn {
   SConnBuffer readBuf;
   STransQueue cliMsgs;
   queue       q;
-  uint64_t    expireTime;
 
   STransCtx  ctx;
   bool       broken;  // link broken or not
@@ -37,6 +36,7 @@ typedef struct SCliConn {
   char*    ip;
   uint32_t port;
 
+  SDelayTask* task;
   // debug and log info
   struct sockaddr_in addr;
   struct sockaddr_in localAddr;
@@ -65,6 +65,7 @@ typedef struct SCliThrd {
   queue         msg;
   TdThreadMutex msgMtx;
   SDelayQueue*  delayQueue;
+  SDelayQueue*  timeoutQueue;
   uint64_t      nextTimeout;  // next timeout
   void*         pTransInst;   //
 
@@ -92,9 +93,10 @@ static void*     createConnPool(int size);
 static void*     destroyConnPool(void* pool);
 static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port);
 static void      addConnToPool(void* pool, SCliConn* conn);
+static void      doCloseIdleConn(void* param);
 
 // register timer in each thread to clear expire conn
-static void cliTimeoutCb(uv_timer_t* handle);
+// static void cliTimeoutCb(uv_timer_t* handle);
 // alloc buf for recv
 static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
 // callback after  read nbytes from socket
@@ -184,7 +186,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
       pThrd = (SCliThrd*)(exh)->pThrd;                \
     }                                                 \
   } while (0)
-#define CONN_PERSIST_TIME(para)    (para * 1000 * 10)
+#define CONN_PERSIST_TIME(para)    ((para) == 0 ? 3 * 1000 : (para))
 #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL)
 #define CONN_GET_INST_LABEL(conn)  (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label)
 #define CONN_SHOULD_RELEASE(conn, head)                                                                           \
@@ -194,6 +196,10 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
       CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle);                                                                  \
       transClearBuffer(&conn->readBuf);                                                                           \
       transFreeMsg(transContFromHead((char*)head));                                                               \
+      if (transQueueSize(&conn->cliMsgs) > 0 && ahandle == 0) {                                                   \
+        SCliMsg* cliMsg = transQueueGet(&conn->cliMsgs, 0);                                                       \
+        if (cliMsg->type == Release) return;                                                                      \
+      }                                                                                                           \
       tDebug("%s conn %p receive release request, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \
       if (T_REF_VAL_GET(conn) > 1) {                                                                              \
         transUnrefCliHandle(conn);                                                                                \
@@ -384,10 +390,6 @@ void cliHandleResp(SCliConn* conn) {
   }
 
   uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb);
-  // start thread's timer of conn pool if not active
-  if (!uv_is_active((uv_handle_t*)&pThrd->timer) && pTransInst->idleTime > 0) {
-    // uv_timer_start((uv_timer_t*)&pThrd->timer, cliTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
-  }
 }
 
 void cliHandleExcept(SCliConn* pConn) {
@@ -441,30 +443,30 @@ void cliHandleExcept(SCliConn* pConn) {
   transUnrefCliHandle(pConn);
 }
 
-void cliTimeoutCb(uv_timer_t* handle) {
+// void cliTimeoutCb(uv_timer_t* handle) {
-  SCliThrd* pThrd = handle->data;
+//   SCliThrd* pThrd = handle->data;
-  STrans*   pTransInst = pThrd->pTransInst;
+//   STrans*   pTransInst = pThrd->pTransInst;
-  int64_t   currentTime = pThrd->nextTimeout;
+//   int64_t   currentTime = pThrd->nextTimeout;
-  tTrace("%s conn timeout, try to remove expire conn from conn pool", pTransInst->label);
+//   tTrace("%s conn timeout, try to remove expire conn from conn pool", pTransInst->label);
-
+//
-  SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL);
+//   SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL);
-  while (p != NULL) {
+//   while (p != NULL) {
-    while (!QUEUE_IS_EMPTY(&p->conn)) {
+//     while (!QUEUE_IS_EMPTY(&p->conn)) {
-      queue*    h = QUEUE_HEAD(&p->conn);
+//       queue*    h = QUEUE_HEAD(&p->conn);
-      SCliConn* c = QUEUE_DATA(h, SCliConn, q);
+//       SCliConn* c = QUEUE_DATA(h, SCliConn, q);
-      if (c->expireTime < currentTime) {
+//       if (c->expireTime < currentTime) {
-        QUEUE_REMOVE(h);
+//         QUEUE_REMOVE(h);
-        transUnrefCliHandle(c);
+//         transUnrefCliHandle(c);
-      } else {
+//       } else {
-        break;
+//         break;
-      }
+//       }
-    }
+//     }
-    p = taosHashIterate((SHashObj*)pThrd->pool, p);
+//     p = taosHashIterate((SHashObj*)pThrd->pool, p);
-  }
+//   }
-
+//
-  pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
+//   pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
-  uv_timer_start(handle, cliTimeoutCb, CONN_PERSIST_TIME(pTransInst->idleTime) / 2, 0);
+//   uv_timer_start(handle, cliTimeoutCb, CONN_PERSIST_TIME(pTransInst->idleTime) / 2, 0);
-}
+// }
 
 void* createConnPool(int size) {
   // thread local, no lock
@@ -506,6 +508,10 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) {
   QUEUE_REMOVE(&conn->q);
   QUEUE_INIT(&conn->q);
   assert(h == &conn->q);
+
+  transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task);
+  conn->task = NULL;
+
   return conn;
 }
 static int32_t allocConnRef(SCliConn* conn, bool update) {
@@ -537,6 +543,7 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) {
   transReleaseExHandle(transGetRefMgt(), handle);
   return 0;
 }
+
 static void addConnToPool(void* pool, SCliConn* conn) {
   if (conn->status == ConnInPool) {
     return;
@@ -547,7 +554,6 @@ static void addConnToPool(void* pool, SCliConn* conn) {
   allocConnRef(conn, true);
 
   STrans* pTransInst = thrd->pTransInst;
-  conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime);
   cliReleaseUnfinishedMsg(conn);
   transQueueClear(&conn->cliMsgs);
   transCtxCleanup(&conn->ctx);
@@ -562,7 +568,13 @@ static void addConnToPool(void* pool, SCliConn* conn) {
   assert(plist != NULL);
   QUEUE_INIT(&conn->q);
   QUEUE_PUSH(&plist->conn, &conn->q);
+
   assert(!QUEUE_IS_EMPTY(&plist->conn));
+
+  STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg));
+  arg->param1 = conn;
+  arg->param2 = thrd;
+  conn->task = transDQSched(thrd->timeoutQueue, doCloseIdleConn, arg, CONN_PERSIST_TIME(pTransInst->idleTime));
 }
 static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
   SCliConn*    conn = handle->data;
@@ -631,6 +643,8 @@ static void cliDestroyConn(SCliConn* conn, bool clear) {
   transRemoveExHandle(transGetRefMgt(), conn->refId);
   conn->refId = -1;
 
+  if (conn->task != NULL) transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task);
+
   if (clear) {
     if (!uv_is_closing((uv_handle_t*)conn->stream)) {
       uv_read_stop(conn->stream);
@@ -997,6 +1011,8 @@ static SCliThrd* createThrdObj() {
   pThrd->pool = createConnPool(4);
   transDQCreate(pThrd->loop, &pThrd->delayQueue);
 
+  transDQCreate(pThrd->loop, &pThrd->timeoutQueue);
+
   pThrd->quit = false;
   return pThrd;
 }
@@ -1012,6 +1028,7 @@ static void destroyThrdObj(SCliThrd* pThrd) {
   transAsyncPoolDestroy(pThrd->asyncPool);
 
   transDQDestroy(pThrd->delayQueue, destroyCmsg);
+  transDQDestroy(pThrd->timeoutQueue, NULL);
   taosMemoryFree(pThrd->loop);
   taosMemoryFree(pThrd);
 }
@@ -1058,6 +1075,10 @@ static void doCloseIdleConn(void* param) {
   STaskArg* arg = param;
   SCliConn* conn = arg->param1;
   SCliThrd* pThrd = arg->param2;
+  tTrace("%s conn %p idle, close it", CONN_GET_INST_LABEL(conn), conn);
+  conn->task = NULL;
+  cliDestroyConn(conn, true);
+  taosMemoryFree(arg);
 }
 
 static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) {
@@ -1248,11 +1269,17 @@ int transReleaseCliHandle(void* handle) {
   if (pThrd == NULL) {
     return -1;
   }
+
   STransMsg tmsg = {.info.handle = handle};
+  TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64());
+
   SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg));
   cmsg->msg = tmsg;
   cmsg->type = Release;
 
+  STraceId* trace = &tmsg.info.traceId;
+  tGDebug("send release request at thread:%08" PRId64 "", pThrd->pid);
+
   if (0 != transAsyncSend(pThrd->asyncPool, &cmsg->q)) {
     return -1;
   }

source/libs/transport/src/transComm.c

@@ -480,7 +480,7 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)) {
     SDelayTask* task = container_of(minNode, SDelayTask, node);
 
     STaskArg* arg = task->arg;
-    freeFunc(arg->param1);
+    if (freeFunc) freeFunc(arg->param1);
     taosMemoryFree(arg);
 
     taosMemoryFree(task);
@@ -491,9 +491,16 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)) {
 void transDQCancel(SDelayQueue* queue, SDelayTask* task) {
   uv_timer_stop(queue->timer);
 
-  if (heapSize(queue->heap) <= 0) return;
+  if (heapSize(queue->heap) <= 0) {
+    taosMemoryFree(task->arg);
+    taosMemoryFree(task);
+    return;
+  }
   heapRemove(queue->heap, &task->node);
 
+  taosMemoryFree(task->arg);
+  taosMemoryFree(task);
+
   if (heapSize(queue->heap) != 0) {
     HeapNode* minNode = heapMin(queue->heap);
     if (minNode != NULL) return;

source/libs/transport/src/transSvr.c

@@ -152,17 +152,20 @@ static bool addHandleToAcceptloop(void* arg);
 #define CONN_SHOULD_RELEASE(conn, head)                                                                   \
   do {                                                                                                    \
     if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) {                                        \
+      reallocConnRef(conn);                                                                               \
+      tTrace("conn %p received release request", conn);                                                   \
+                                                                                                          \
+      STraceId traceId = head->traceId;                                                                   \
       conn->status = ConnRelease;                                                                         \
       transClearBuffer(&conn->readBuf);                                                                   \
       transFreeMsg(transContFromHead((char*)head));                                                       \
-      tTrace("conn %p received release request", conn);                               \
                                                                                                           \
-      STransMsg tmsg = {.code = 0, .info.handle = (void*)conn, .info.ahandle = NULL}; \
+      STransMsg tmsg = {                                                                                  \
+          .code = 0, .info.handle = (void*)conn, .info.traceId = traceId, .info.ahandle = (void*)0x9527}; \
       SSvrMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSvrMsg));                                             \
       srvMsg->msg = tmsg;                                                                                 \
       srvMsg->type = Release;                                                                             \
       srvMsg->pConn = conn;                                                                               \
-      reallocConnRef(conn);                                                           \
       if (!transQueuePush(&conn->srvMsgs, srvMsg)) {                                                      \
         return;                                                                                           \
       }                                                                                                   \
@@ -394,11 +397,11 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
 
   if (pConn->status == ConnNormal) {
     pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType);
+    if (smsg->type == Release) pHead->msgType = 0;
   } else {
     if (smsg->type == Release) {
       pHead->msgType = 0;
       pConn->status = ConnNormal;
-
       destroyConnRegArg(pConn);
       transUnrefSrvHandle(pConn);
     } else {

source/util/src/tcompression.c

@@ -58,7 +58,7 @@
 
 static const int32_t TEST_NUMBER = 1;
 #define is_bigendian()     ((*(char *)&TEST_NUMBER) == 0)
-#define SIMPLE8B_MAX_INT64 ((uint64_t)2305843009213693951LL)
+#define SIMPLE8B_MAX_INT64 ((uint64_t)1152921504606846974LL)
 
 #define safeInt64Add(a, b)  (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a)))
 #define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)  // zigzag encode

source/util/src/tconfig.c

@@ -626,6 +626,7 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) {
 int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
   char   *line = NULL, *name, *value, *value2, *value3;
   int32_t olen, vlen, vlen2, vlen3;
+  int32_t code = 0;
   ssize_t _bytes = 0;
   TdCmdPtr pCmd = taosOpenCmd("set");
   if (pCmd == NULL) {
@@ -658,9 +659,12 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
       if (vlen3 != 0) value3[vlen3] = 0;
     }
 
-    cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR);
     if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
-      cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_VAR);
+      code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_VAR);
+      if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
+    } else {
+      code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR);
+      if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
     }
   }
 
@@ -674,6 +678,7 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) {
 int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) {
   char    buf[1024], *name, *value, *value2, *value3;
   int32_t olen, vlen, vlen2, vlen3;
+  int32_t code = 0;
   int32_t index = 0;
   if (envCmd == NULL) return 0;
   while (envCmd[index]!=NULL) {
@@ -700,9 +705,12 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) {
       if (vlen3 != 0) value3[vlen3] = 0;
     }
 
-    cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD);
     if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
-      cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD);
+      code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD);
+      if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
+    } else {
+      code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD);
+      if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
     }
   }
 
@@ -713,6 +721,7 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) {
 int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) {
   char   *line = NULL, *name, *value, *value2, *value3;
   int32_t olen, vlen, vlen2, vlen3;
+  int32_t code = 0;
   ssize_t _bytes = 0;
 
   const char *filepath = ".env";
@@ -761,9 +770,12 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) {
       if (vlen3 != 0) value3[vlen3] = 0;
     }
 
-    cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE);
     if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
-      cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_FILE);
+      code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_FILE);
+      if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
+    } else {
+      code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE);
+      if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
     }
   }
 
@@ -819,11 +831,12 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
       if (vlen3 != 0) value3[vlen3] = 0;
     }
 
-    code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
-    if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
     if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
       code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE);
       if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
+    } else {
+      code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
+      if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
     }
   }
 
@@ -839,9 +852,75 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) {
   }
 }
 
+// int32_t cfgLoadFromCfgText(SConfig *pConfig, const char *configText) {
+//   char   *line = NULL, *name, *value, *value2, *value3;
+//   int32_t olen, vlen, vlen2, vlen3;
+//   ssize_t _bytes = 0;
+//   int32_t code = 0;
+
+//   TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM);
+//   if (pFile == NULL) {
+//     // success when the file does not exist
+//     if (errno == ENOENT) {
+//       terrno = TAOS_SYSTEM_ERROR(errno);
+//       uInfo("failed to load from cfg file %s since %s, use default parameters", filepath, terrstr());
+//       return 0;
+//     } else {
+//       uError("failed to load from cfg file %s since %s", filepath, terrstr());
+//       return -1;
+//     }
+//   }
+
+//   while (!taosEOFFile(pFile)) {
+//     name = value = value2 = value3 = NULL;
+//     olen = vlen = vlen2 = vlen3 = 0;
+
+//     _bytes = taosGetLineFile(pFile, &line);
+//     if (_bytes <= 0) {
+//       break;
+//     }
+
+//     if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0;
+
+//     paGetToken(line, &name, &olen);
+//     if (olen == 0) continue;
+//     name[olen] = 0;
+
+//     paGetToken(name + olen + 1, &value, &vlen);
+//     if (vlen == 0) continue;
+//     value[vlen] = 0;
+
+//     paGetToken(value + vlen + 1, &value2, &vlen2);
+//     if (vlen2 != 0) {
+//       value2[vlen2] = 0;
+//       paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
+//       if (vlen3 != 0) value3[vlen3] = 0;
+//     }
+
+//     code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE);
+//     if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
+//     if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
+//       code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE);
+//       if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
+//     }
+//   }
+
+//   taosCloseFile(&pFile);
+//   if (line != NULL) taosMemoryFreeClear(line);
+
+//   if (code == 0 || (code != 0 && terrno == TSDB_CODE_CFG_NOT_FOUND)) {
+//     uInfo("load from cfg file %s success", filepath);
+//     return 0;
+//   } else {
+//     uError("failed to load from cfg file %s since %s", filepath, terrstr());
+//     return -1;
+//   }
+// }
+
 int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
   char   *cfgLineBuf = NULL, *name, *value, *value2, *value3;
   int32_t olen, vlen, vlen2, vlen3;
+  int32_t code = 0;
   if (url == NULL || strlen(url) == 0) {
     uInfo("fail to load apoll url");
     return 0;
@@ -916,9 +995,12 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) {
           paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
           if (vlen3 != 0) value3[vlen3] = 0;
         }
-        cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL);
         if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) {
-          cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_APOLLO_URL);
+          code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_APOLLO_URL);
+          if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
+        } else {
+          code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL);
+          if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break;
         }
       }
     }

source/util/src/terror.c

@@ -336,6 +336,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_NOT_EXIST,          "Table does not exists
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TABLE_ACTION,     "Invalid table action")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS,       "Table column already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_COL_NOT_EXISTS,     "Table column not exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_SUBSCRIBED,           "Table column is subscribed")
 
 // tsdb
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID,         "Invalid table ID")

tests/pytest/topic/topicQuery.py

@@ -1,91 +0,0 @@
-###################################################################
-#           Copyright (c) 2016 by TAOS Technologies, Inc.
-#                     All rights reserved.
-#
-#  This file is proprietary and confidential to TAOS Technologies.
-#  No part of this file may be reproduced, stored, transmitted,
-#  disclosed or used in any form or by any means other than as
-#  expressly provided by the written permission from Jianhui Tao
-#
-###################################################################
-
-# -*- coding: utf-8 -*-
-
-from util.log import tdLog
-from util.cases import tdCases
-from util.sql import tdSql
-
-class TDTestCase:
-    def init(self, conn, logSql):
-        tdLog.debug("start to execute %s" % __file__)
-        tdSql.init(conn.cursor(), logSql)
-
-        self.ts = 1538548685000
-
-    def run(self):
-        tdSql.prepare()
-
-        # test case for https://jira.taosdata.com:18080/browse/TD-3679
-        print("==============step1")
-        tdSql.execute(
-            "create topic tq_test partitions 10")
-        tdSql.execute(
-            "insert into tq_test.p1(off, ts, content) values(0, %d, 'aaaa')" % self.ts)
-        tdSql.execute(
-            "insert into tq_test.p1(off, ts, content) values(1, %d, 'aaaa')" % (self.ts + 1))
-        tdSql.execute(
-            "insert into tq_test.p1(off, ts, content) values(2, %d, 'aaaa')" % (self.ts + 2))
-        tdSql.execute(
-            "insert into tq_test.p1(off, ts, content) values(3, %d, 'aaaa')" % (self.ts + 3))
-
-        print("==============step2")
-
-        tdSql.query("select * from tq_test.p1")
-        tdSql.checkRows(4)
-
-        tdSql.query("select * from tq_test.p1 where ts >= %d" % self.ts)
-        tdSql.checkRows(4)
-
-        tdSql.query("select * from tq_test.p1 where ts > %d" % self.ts)
-        tdSql.checkRows(3)
-
-        tdSql.query("select * from tq_test.p1 where ts = %d" % self.ts)
-        tdSql.checkRows(1)
-
-
-        tdSql.execute("use db")
-        tdSql.execute("create table test(ts timestamp, start timestamp, value int)")
-        tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts, self.ts))
-        tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts + 1, self.ts + 1))
-        tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts + 2, self.ts + 2))
-        tdSql.execute("insert into test values(%d, %d, 1)" % (self.ts + 3, self.ts + 3))
-
-        tdSql.query("select * from test")
-        tdSql.checkRows(4)
-
-        tdSql.query("select * from test where ts >= %d" % self.ts)
-        tdSql.checkRows(4)
-
-        tdSql.query("select * from test where ts > %d" % self.ts)
-        tdSql.checkRows(3)
-
-        tdSql.query("select * from test where ts = %d" % self.ts)
-        tdSql.checkRows(1)
-
-        tdSql.query("select * from test where start >= %d" % self.ts)
-        tdSql.checkRows(4)
-
-        tdSql.query("select * from test where start > %d" % self.ts)
-        tdSql.checkRows(3)
-
-        tdSql.query("select * from test where start = %d" % self.ts)
-        tdSql.checkRows(1)
-
-        
-    def stop(self):
-        tdSql.close()
-        tdLog.success("%s successfully executed" % __file__)
-
-
-tdCases.addWindows(__file__, TDTestCase())
-tdCases.addLinux(__file__, TDTestCase())

tests/pytest/util/common.py

@@ -457,14 +457,14 @@ class TDCom:
 
     def newcon(self,host='localhost',port=6030,user='root',password='taosdata'):
         con=taos.connect(host=host, user=user, password=password, port=port)
-        print(con)
+        # print(con)
         return con
 
     def newcur(self,host='localhost',port=6030,user='root',password='taosdata'):
         cfgPath = self.getClientCfgPath()
         con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port)
         cur=con.cursor()
-        print(cur)
+        # print(cur)
         return cur
 
     def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata'):

tests/script/jenkins/basic.txt

@@ -69,6 +69,7 @@
 ./test.sh -f tsim/insert/basic.sim
 ./test.sh -f tsim/insert/basic0.sim
 ./test.sh -f tsim/insert/basic1.sim
+./test.sh -f tsim/insert/basic2.sim
 ./test.sh -f tsim/insert/commit-merge0.sim
 ./test.sh -f tsim/insert/insert_drop.sim
 ./test.sh -f tsim/insert/insert_select.sim
@@ -88,17 +89,17 @@
 ./test.sh -f tsim/parser/alter_column.sim
 ./test.sh -f tsim/parser/alter_stable.sim
 ./test.sh -f tsim/parser/alter.sim
-# ./test.sh -f tsim/parser/alter1.sim
+# TD-17661 ./test.sh -f tsim/parser/alter1.sim
 ./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim
 ./test.sh -f tsim/parser/auto_create_tb.sim
 ./test.sh -f tsim/parser/between_and.sim
 ./test.sh -f tsim/parser/binary_escapeCharacter.sim
-# ./test.sh -f tsim/parser/col_arithmetic_operation.sim     
+# TD-17738 ./test.sh -f tsim/parser/col_arithmetic_operation.sim     
-# ./test.sh -f tsim/parser/columnValue.sim
+# TD-17661 ./test.sh -f tsim/parser/columnValue.sim
 ./test.sh -f tsim/parser/commit.sim
 # TD-17661 ./test.sh -f tsim/parser/condition.sim
 ./test.sh -f tsim/parser/constCol.sim
-./test.sh -f tsim/parser/create_db.sim
+#./test.sh -f tsim/parser/create_db.sim
 ./test.sh -f tsim/parser/create_mt.sim
 # TD-17653 ./test.sh -f tsim/parser/create_tb_with_tag_name.sim
 ./test.sh -f tsim/parser/create_tb.sim
@@ -109,61 +110,58 @@
 ./test.sh -f tsim/parser/fill.sim
 ./test.sh -f tsim/parser/first_last.sim
 ./test.sh -f tsim/parser/fourArithmetic-basic.sim
-# TD-17659 TD-17658 ./test.sh -f tsim/parser/function.sim
+# TD-17659 ./test.sh -f tsim/parser/function.sim
 ./test.sh -f tsim/parser/groupby-basic.sim
-# ./test.sh -f tsim/parser/groupby.sim
+./test.sh -f tsim/parser/groupby.sim
 # TD-17622 ./test.sh -f tsim/parser/having_child.sim
-# ./test.sh -f tsim/parser/having.sim
+# TD-17622 ./test.sh -f tsim/parser/having.sim
 ./test.sh -f tsim/parser/import_commit1.sim
 ./test.sh -f tsim/parser/import_commit2.sim
 ./test.sh -f tsim/parser/import_commit3.sim
-# TD-17663 ./test.sh -f tsim/parser/import_file.sim
+./test.sh -f tsim/parser/import_file.sim
 ./test.sh -f tsim/parser/import.sim
 ./test.sh -f tsim/parser/insert_multiTbl.sim
 ./test.sh -f tsim/parser/insert_tb.sim
-# ./test.sh -f tsim/parser/interp.sim
+# TD-17038 ./test.sh -f tsim/parser/interp.sim
 ./test.sh -f tsim/parser/join_manyblocks.sim
-# ./test.sh -f tsim/parser/join_multitables.sim
+# TD-17713 ./test.sh -f tsim/parser/join_multitables.sim
 # TD-17713 ./test.sh -f tsim/parser/join_multivnode.sim
 # TD-17707 ./test.sh -f tsim/parser/join.sim
 ./test.sh -f tsim/parser/last_cache.sim
 ./test.sh -f tsim/parser/last_groupby.sim
-# TD-17675 ./test.sh -f tsim/parser/lastrow.sim
+# TD-17722 ./test.sh -f tsim/parser/lastrow.sim
 ./test.sh -f tsim/parser/like.sim
-# ./test.sh -f tsim/parser/limit.sim
+# TD-17464 ./test.sh -f tsim/parser/limit.sim
-# ./test.sh -f tsim/parser/limit1.sim
+# TD-17464 ./test.sh -f tsim/parser/limit1.sim
-# ./test.sh -f tsim/parser/limit2.sim
+# TD-17623 ./test.sh -f tsim/parser/limit2.sim
 ./test.sh -f tsim/parser/mixed_blocks.sim
 ./test.sh -f tsim/parser/nchar.sim
 # TD-17703 ./test.sh -f tsim/parser/nestquery.sim
-# ./test.sh -f tsim/parser/null_char.sim
+# TD-17685 ./test.sh -f tsim/parser/null_char.sim
 ./test.sh -f tsim/parser/precision_ns.sim
 ./test.sh -f tsim/parser/projection_limit_offset.sim
 ./test.sh -f tsim/parser/regex.sim
 ./test.sh -f tsim/parser/select_across_vnodes.sim
 ./test.sh -f tsim/parser/select_distinct_tag.sim
 ./test.sh -f tsim/parser/select_from_cache_disk.sim
-# ./test.sh -f tsim/parser/select_with_tags.sim
+# TD-17659 ./test.sh -f tsim/parser/select_with_tags.sim
 ./test.sh -f tsim/parser/selectResNum.sim
 # TD-17685 ./test.sh -f tsim/parser/set_tag_vals.sim
 ./test.sh -f tsim/parser/single_row_in_tb.sim
 # TD-17684 ./test.sh -f tsim/parser/sliding.sim
-# ./test.sh -f tsim/parser/slimit_alter_tags.sim
+# TD-17722 ./test.sh -f tsim/parser/slimit_alter_tags.sim
-# ./test.sh -f tsim/parser/slimit.sim
+# TD-17722 ./test.sh -f tsim/parser/slimit.sim
-# ./test.sh -f tsim/parser/slimit1.sim
+# TD-17722 ./test.sh -f tsim/parser/slimit1.sim
 ./test.sh -f tsim/parser/stableOp.sim
-# ./test.sh -f tsim/parser/tags_dynamically_specifiy.sim
+# TD-17661 ./test.sh -f tsim/parser/tags_dynamically_specifiy.sim
-# ./test.sh -f tsim/parser/tags_filter.sim
+# TD-17661 ./test.sh -f tsim/parser/tags_filter.sim
 ./test.sh -f tsim/parser/tbnameIn.sim
 ./test.sh -f tsim/parser/timestamp.sim
 ./test.sh -f tsim/parser/top_groupby.sim
 ./test.sh -f tsim/parser/topbot.sim
-# ./test.sh -f tsim/parser/udf_dll_stable.sim
-# ./test.sh -f tsim/parser/udf_dll.sim
-# ./test.sh -f tsim/parser/udf.sim
 ./test.sh -f tsim/parser/union.sim
 # TD-17704 ./test.sh -f tsim/parser/union_sysinfo.sim
-# ./test.sh -f tsim/parser/where.sim
+# TD-17661 ./test.sh -f tsim/parser/where.sim
 
 # ---- query
 ./test.sh -f tsim/query/interval.sim
@@ -225,7 +223,7 @@
 
 # ---- stream
 ./test.sh -f tsim/stream/basic0.sim
-./test.sh -f tsim/stream/basic1.sim
+#./test.sh -f tsim/stream/basic1.sim
 ./test.sh -f tsim/stream/basic2.sim
 ./test.sh -f tsim/stream/drop_stream.sim
 ./test.sh -f tsim/stream/distributeInterval0.sim
@@ -324,7 +322,7 @@
 ./test.sh -f tsim/vnode/stable_replica3_vnode3.sim
 
 # --- sync
-# ./test.sh -f tsim/sync/3Replica1VgElect.sim
+./test.sh -f tsim/sync/3Replica1VgElect.sim
 ./test.sh -f tsim/sync/3Replica5VgElect.sim
 ./test.sh -f tsim/sync/oneReplica1VgElect.sim
 ./test.sh -f tsim/sync/oneReplica5VgElect.sim
@@ -422,18 +420,18 @@
 ./test.sh -f tsim/tag/bool_binary.sim
 ./test.sh -f tsim/tag/bool_int.sim
 ./test.sh -f tsim/tag/bool.sim
-# ./test.sh -f tsim/tag/change.sim
+# TD-17661 ./test.sh -f tsim/tag/change.sim
-# ./test.sh -f tsim/tag/column.sim
+./test.sh -f tsim/tag/column.sim
-# ./test.sh -f tsim/tag/commit.sim
+./test.sh -f tsim/tag/commit.sim
-# ./test.sh -f tsim/tag/create.sim
+# TD-17661 ./test.sh -f tsim/tag/create.sim
-# /test.sh -f tsim/tag/delete.sim
+# TD-17661 ./test.sh -f tsim/tag/delete.sim
-# ./test.sh -f tsim/tag/double.sim
+# TD-17661 ./test.sh -f tsim/tag/double.sim
-# ./test.sh -f tsim/tag/filter.sim
+# TD-17661 ./test.sh -f tsim/tag/filter.sim
 # TD-17407 ./test.sh -f tsim/tag/float.sim
 ./test.sh -f tsim/tag/int_binary.sim
 ./test.sh -f tsim/tag/int_float.sim
 ./test.sh -f tsim/tag/int.sim
-# ./test.sh -f tsim/tag/set.sim
+# TD-17661 ./test.sh -f tsim/tag/set.sim
 ./test.sh -f tsim/tag/smallint.sim
 ./test.sh -f tsim/tag/tinyint.sim
 

tests/script/jenkins/clusterCase.txt

@@ -0,0 +1,29 @@
+
+#======================b1-start===============
+
+# ---- mnode
+./test.sh -f tsim/mnode/basic1.sim
+./test.sh -f tsim/mnode/basic2.sim
+./test.sh -f tsim/mnode/basic3.sim
+./test.sh -f tsim/mnode/basic4.sim
+./test.sh -f tsim/mnode/basic5.sim
+
+# --- vnode
+# unsupport ./test.sh -f tsim/vnode/replica3_basic.sim
+# unsupport ./test.sh -f tsim/vnode/replica3_repeat.sim
+# unsupport ./test.sh -f tsim/vnode/replica3_vgroup.sim
+# unsupport ./test.sh -f tsim/vnode/replica3_many.sim
+# unsupport ./test.sh -f tsim/vnode/replica3_import.sim
+# unsupport ./test.sh -f tsim/vnode/stable_balance_replica1.sim
+# unsupport ./test.sh -f tsim/vnode/stable_dnode2_stop.sim
+./test.sh -f tsim/vnode/stable_dnode2.sim
+./test.sh -f tsim/vnode/stable_dnode3.sim
+./test.sh -f tsim/vnode/stable_replica3_dnode6.sim
+./test.sh -f tsim/vnode/stable_replica3_vnode3.sim
+
+# --- sync
+# jira ./test.sh -f tsim/sync/3Replica1VgElect.sim
+./test.sh -f tsim/sync/3Replica5VgElect.sim
+./test.sh -f tsim/sync/oneReplica1VgElect.sim
+./test.sh -f tsim/sync/oneReplica5VgElect.sim
+

tests/script/tsim/dnode/offline_reason.sim

@@ -64,9 +64,10 @@ if $rows != 1 then
 endi
 
 print ========== step5
-system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname port 7200
+system sh/exec.sh -n dnode2 -s start
 
+return
 $x = 0
 step5: 
 	$x = $x + 1

tests/script/tsim/insert/basic2.sim

@@ -0,0 +1,322 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+print =============== create database
+sql create database d0 keep 365000d,365000d,365000d
+sql use d0
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int unsigned, c2 double, c3 binary(10), c4 nchar(10), c5 double) tags (city binary(20),district binary(20));
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+sql create table ct1 using stb tags("BeiJing", "ChaoYang")
+sql create table ct2 using stb tags("BeiJing", "HaiDian")
+
+sql show tables
+if $rows != 2 then 
+  return -1
+endi
+
+print =============== step3-1 insert records into ct1
+sql insert into ct1 values('2022-05-03 16:59:00.010', 10, 20, 'n','n',30);
+sql insert into ct1 values('2022-05-03 16:59:00.011', 'N', 'n', 'N',"N",30);
+sql insert into ct1 values('2022-05-03 16:59:00.012', 'Nu', 'nul', 'Nul','NUL',30);
+sql insert into ct1 values('2022-05-03 16:59:00.013', NULL, 'null', 'Null',null,30);
+sql insert into ct1 values('2022-05-03 16:59:00.014', NULL, 'NuLL', 'Null',NULL,30);
+
+sql_error insert into ct1 values('2022-05-03 16:59:00.015', NULL, 20, 'Null',NUL,30);
+sql_error insert into ct1 values('2022-05-03 16:59:00.015', NULL, 20, 'Null',NU,30);
+sql_error insert into ct1 values('2022-05-03 16:59:00.015', NULL, 20, 'Null',Nu,30);
+sql_error insert into ct1 values('2022-05-03 16:59:00.015', NULL, 20, 'Null',N,30);
+sql_error insert into ct1 values('2022-05-03 16:59:00.015', N, 20, 'Null',NULL,30);
+sql_error insert into ct1 values('2022-05-03 16:59:00.015', Nu, 20, 'Null',NULL,30);
+sql_error insert into ct1 values('2022-05-03 16:59:00.015', Nul, 20, 'Null',NULL,30);
+
+print =============== step3-1 query records of ct1 from memory
+sql select * from ct1;
+print $data00 $data01 $data02  $data03  $data04  $data05
+print $data10 $data11 $data12  $data13  $data14  $data15
+print $data20 $data21 $data22  $data23  $data24  $data25
+print $data30 $data31 $data32  $data33  $data34  $data35
+print $data40 $data41 $data42  $data43  $data44  $data45
+
+if $rows != 5 then 
+  print rows $rows != 5
+  return -1
+endi
+
+if $data01 != 10 then
+  print data01 $data01 != 10
+  return -1
+endi
+
+if $data02 != 20.000000000 then
+  print data02 $data02 != 20.000000000
+  return -1
+endi
+
+if $data03 != n then
+  print data03 $data03 != n
+  return -1
+endi
+
+if $data04 != n then
+  print data04 $data04 != n
+  return -1
+endi
+
+if $data05 != 30.000000000 then
+  print data05 $data05 != 30.000000000
+  return -1
+endi
+
+if $data11 != NULL then
+  print data11 $data11 != NULL
+  return -1
+endi
+
+if $data12 != NULL then
+  print data12 $data12 != NULL
+  return -1
+endi
+
+if $data13 != N then
+  print data13 $data13 != N
+  return -1
+endi
+
+if $data14 != N then
+  print data14 $data14 != N
+  return -1
+endi
+
+if $data15 != 30.000000000 then
+  print data15 $data15 != 30.000000000
+  return -1
+endi
+
+if $data21 != NULL then
+  print data21 $data21 != NULL
+  return -1
+endi
+
+if $data22 != NULL then
+  print data22 $data22 != NULL
+  return -1
+endi
+
+if $data23 != Nul then
+  print data23 $data23 != Nul
+  return -1
+endi
+
+if $data24 != NUL then
+  print data24 $data24 != NUL
+  return -1
+endi
+
+if $data25 != 30.000000000 then
+  print data25 $data25 != 30.000000000
+  return -1
+endi
+
+if $data31 != NULL then
+  print data31 $data31 != NULL
+  return -1
+endi
+
+if $data32 != NULL then
+  print data32 $data32 != NULL
+  return -1
+endi
+
+if $data33 != Null then
+  print data33 $data33 != Null
+  return -1
+endi
+
+if $data34 != NULL then
+  print data34 $data34 != NULL
+  return -1
+endi
+
+if $data35 != 30.000000000 then
+  print data35 $data35 != 30.000000000
+  return -1
+endi
+
+if $data41 != NULL then
+  print data41 $data41 != NULL
+  return -1
+endi
+
+if $data42 != NULL then
+  print data42 $data42 != NULL
+  return -1
+endi
+
+if $data43 != Null then
+  print data43 $data43 != Null
+  return -1
+endi
+
+if $data44 != NULL then
+  print data44 $data44 != NULL
+  return -1
+endi
+
+if $data45 != 30.000000000 then
+  print data45 $data45 != 30.000000000
+  return -1
+endi
+
+#==================== reboot to trigger commit data to file
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+
+print =============== step3-2 query records of ct1 from file
+sql select * from ct1;
+print $data00 $data01 $data02  $data03  $data04  $data05
+print $data10 $data11 $data12  $data13  $data14  $data15
+print $data20 $data21 $data22  $data23  $data24  $data25
+print $data30 $data31 $data32  $data33  $data34  $data35
+print $data40 $data41 $data42  $data43  $data44  $data45
+
+if $rows != 5 then 
+  print rows $rows != 5
+  return -1
+endi
+
+if $data01 != 10 then
+  print data01 $data01 != 10
+  return -1
+endi
+
+if $data02 != 20.000000000 then
+  print data02 $data02 != 20.000000000
+  return -1
+endi
+
+if $data03 != n then
+  print data03 $data03 != n
+  return -1
+endi
+
+if $data04 != n then
+  print data04 $data04 != n
+  return -1
+endi
+
+if $data05 != 30.000000000 then
+  print data05 $data05 != 30.000000000
+  return -1
+endi
+
+if $data11 != NULL then
+  print data11 $data11 != NULL
+  return -1
+endi
+
+if $data12 != NULL then
+  print data12 $data12 != NULL
+  return -1
+endi
+
+if $data13 != N then
+  print data13 $data13 != N
+  return -1
+endi
+
+if $data14 != N then
+  print data14 $data14 != N
+  return -1
+endi
+
+if $data15 != 30.000000000 then
+  print data15 $data15 != 30.000000000
+  return -1
+endi
+
+if $data21 != NULL then
+  print data21 $data21 != NULL
+  return -1
+endi
+
+if $data22 != NULL then
+  print data22 $data22 != NULL
+  return -1
+endi
+
+if $data23 != Nul then
+  print data23 $data23 != Nul
+  return -1
+endi
+
+if $data24 != NUL then
+  print data24 $data24 != NUL
+  return -1
+endi
+
+if $data25 != 30.000000000 then
+  print data25 $data25 != 30.000000000
+  return -1
+endi
+
+if $data31 != NULL then
+  print data31 $data31 != NULL
+  return -1
+endi
+
+if $data32 != NULL then
+  print data32 $data32 != NULL
+  return -1
+endi
+
+if $data33 != Null then
+  print data33 $data33 != Null
+  return -1
+endi
+
+if $data34 != NULL then
+  print data34 $data34 != NULL
+  return -1
+endi
+
+if $data35 != 30.000000000 then
+  print data35 $data35 != 30.000000000
+  return -1
+endi
+
+if $data41 != NULL then
+  print data41 $data41 != NULL
+  return -1
+endi
+
+if $data42 != NULL then
+  print data42 $data42 != NULL
+  return -1
+endi
+
+if $data43 != Null then
+  print data43 $data43 != Null
+  return -1
+endi
+
+if $data44 != NULL then
+  print data44 $data44 != NULL
+  return -1
+endi
+
+if $data45 != 30.000000000 then
+  print data45 $data45 != 30.000000000
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/insert/commit-merge0.sim

@@ -242,3 +242,5 @@ else
     $reboot_cnt = $reboot_cnt + 1
     goto reboot_and_check
 endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/insert/update0.sim

@@ -227,3 +227,5 @@ if $data41 != NULL then
   print data41 $data41 != NULL
   return -1
 endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/insert/update1_sort_merge.sim

@@ -817,3 +817,5 @@ if $data44 != n8 then
   print data44 $data44 != n8
   return -1
 endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/parser/col_arithmetic_query.sim

@@ -36,28 +36,22 @@ sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc;
 if $rows != 10000 then
   return -1
 endi
-
 if $data00 != 0.000000000 then
   return -1
 endi
-
 #if $data01 != -nan then
 #  print expect -nan, actual: $data01
 #  return -1
 #endi
-
 if $data10 != 0.666666667 then
   return -1
 endi
-
 if $data11 != 1.000000000 then
   return -1
 endi
-
 if $data90 != 6.000000000 then
   return -1
 endi
-
 if $data91 != 1.000000000 then
   return -1
 endi
@@ -66,65 +60,49 @@ sql select (c1 * 2) % 7.9, c1*1, c1*1*1, c1*c1, c1*c1*c1 from $tb order by ts de
 if $rows != 10000 then
  return -1
 endi
-
 if $data00 != 2.200000000 then
   print expect 2.200000000, actual:$data00
   return -1
 endi
-
 if $data01 != 9.000000000 then
   return -1
 endi
-
 if $data02 != 9.000000000 then
   return -1
 endi
-
 if $data03 != 81.000000000 then
   return -1
 endi
-
 if $data04 != 729.000000000 then
   return -1
 endi
-
-
 if $data10 != 0.200000000 then
   return -1
 endi
-
 if $data11 != 8.000000000 then
   return -1
 endi
-
 if $data12 != 8.000000000 then
   return -1
 endi
-
 if $data13 != 64.000000000 then
   return -1
 endi
-
 if $data14 != 512.000000000 then
   return -1
 endi
-
 if $data90 != 0.000000000 then
  return -1
 endi
-
 if $data91 != 0.000000000 then
   return -1
 endi
-
 if $data92 != 0.000000000 then
   return -1
 endi
-
 if $data93 != 0.000000000 then
   return -1
 endi
-
 if $data94 != 0.000000000 then
   return -1
 endi
@@ -134,20 +112,16 @@ sql select c1 * c2 /4 from $tb where ts < 1537166000000 and ts > 1537156000000
 if $rows != 17 then
   return -1
 endi
-
 if $data00 != 12.250000000 then
   return -1
 endi
-
 if $data10 != 16.000000000 then
   return -1
 endi
-
 if $data20 != 20.250000000 then
   print expect 20.250000000, actual:$data21
   return -1
 endi
-
 if $data30 != 0.000000000 then
   return -1
 endi
@@ -180,47 +154,36 @@ sql select c2-c1*1.1, c3/c2, c4*c3, c5%c4, (c6+c4)%22, c2-c2 from $tb
 if $rows != 10000 then
  return -1
 endi
-
 if $data00 != 0.000000000 then
   return -1
 endi
-
 #if $data01 != -nan then
 #  return -1
 #endi
-
 if $data02 != 0.000000000 then
   return -1
 endi
-
 if $data03 != NULL then
   return -1
 endi
-
 if $data04 != 0.000000000 then
   return -1
 endi
-
 if $data05 != 0.000000000 then
   return -1
 endi
-
 if $data90 != -0.900000000 then
   return -1
 endi
-
 if $data91 != 1.000000000 then
   return -1
 endi
-
 if $data92 != 81.000000000 then
   return -1
 endi
-
 if $data93 != 0.000000000 then
   return -1
 endi
-
 if $data94 != 18.000000000 then
   return -1
 endi
@@ -237,10 +200,8 @@ sql       select c8+c7, c9+c9+c8+c7/c6 from $tb
 
 # arithmetic expression in join [d.7]==================================================
 
-
 # arithmetic expression in union [d.8]=================================================
 
-
 # arithmetic expression in group by [d.9]==============================================
 # in group by tag, not support for normal table
 sql_error select c5*99 from $tb group by t1
@@ -248,17 +209,14 @@ sql_error select c5*99 from $tb group by t1
 # in group by column
 sql_error select c6-(c6+c3)*12 from $tb group by c3;
 
-
 # limit offset [d.10]==================================================================
 sql select c6 * c1 + 12 from $tb limit 12 offset 99;
 if $rows != 12 then
  return -1
 endi
-
 if $data00 != 93.000000000 then
   return -1
 endi
-
 if $data90 != 76.000000000 then
   return -1
 endi
@@ -267,7 +225,6 @@ sql select c4 / 99.123 from $tb limit 10 offset 9999;
 if $rows != 1 then
   return -1
 endi
-
 if $data00 != 0.090796283 then
   return -1
 endi
@@ -283,27 +240,21 @@ sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb
 if $rows != 10000 then
   return -1
 endi
-
 if $data00 != 0 then
   return -1
 endi
-
 if $data01 != 0.000000000 then
   return -1
 endi
-
 if $data02 != 12.987654568 then
   return -1
 endi
-
 if $data03 != 1 then
   return -1
 endi
-
 if $data04 != 1.100000000 then
   return -1
 endi
-
 if $data10 != 1 then
   return -1
 endi
@@ -313,27 +264,21 @@ sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb where c1<2
 if $rows != 2000 then
   return -1
 endi
-
 if $data00 != 0 then
   return -1
 endi
-
 if $data01 != 0.000000000 then
   return -1
 endi
-
 if $data02 != 12.987654568 then
   return -1
 endi
-
 if $data03 != 1 then
   return -1
 endi
-
 if $data10 != 1 then
   return -1
 endi
-
 if $data20 != 0 then
   return -1
 endi
@@ -377,7 +322,6 @@ sql select first(c1) * ( 2 / 3 ) from $stb order by ts asc;
 if $rows != 1 then
   return -1
 endi
-
 if $data00 != 0.000000000 then
   return -1
 endi
@@ -386,7 +330,6 @@ sql select first(c1) * (2/99) from $stb order by ts desc;
 if $rows != 1 then
   return -1
 endi
-
 if $data00 != 0.000000000 then
   return -1
 endi
@@ -395,15 +338,12 @@ sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb
 if $rows != 1 then
   return -1
 endi
-
 if $data00 != 1.800000000 then
   return -1
 endi
-
 if $data01 != 100000.000000000 then
   return -1
 endi
-
 if $data02 != 200000.000000000 then
   return -1
 endi
@@ -412,16 +352,13 @@ sql select spread( c1 )/44, spread(c1), 0.204545455 * 44 from $stb
 if $rows != 1 then
   return -1
 endi
-
 if $data00 != 0.204545455 then
   print expect 0.204545455, actual: $data00
   return -1
 endi
-
 if $data01 != 9.000000000 then
   return -1
 endi
-
 if $data02 != 9.000000020 then
   return -1
 endi
@@ -431,27 +368,21 @@ sql select min(c1) * max(c2) /4, sum(c1) * apercentile(c2, 20), apercentile(c4,
 if $rows != 1 then
   return -1
 endi
-
 if $data00 != 0.000000000 then
   return -1
 endi
-
 if $data01 != 225000.000000000 then
   return -1
 endi
-
 if $data02 != 8.077777778 then
   return -1
 endi
-
 if $data03 != NULL then
   return -1
 endi
-
 if $data04 != 0.444444444 then
   return -1
 endi
-
 if $data05 != 450000.000000000 then
   return -1
 endi
@@ -489,33 +420,27 @@ sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $stb
 # error case, ts/bool/binary/nchar not support arithmetic expression
 sql select first(c7)*12 from $stb
 sql select last(c8)/55 from $stb
-sql_error select last_row(c9) + last_row(c8) from $stb
+sql select last_row(c9) + last_row(c8) from $stb
 
 # arithmetic expression in join [d.7]===============================================================
 
-
 # arithmetic expression in union [d.8]===============================================================
 
-
 # arithmetic expression in group by [d.9]===============================================================
 # in group by tag
-sql select avg(c4)*99 from $stb group by t1
+sql select avg(c4)*99, t1 from $stb group by t1 order by t1
 if $rows != 10 then
   return -1
 endi
-
 if $data00 != 445.500000000 then
   return -1
 endi
-
 if $data01 != 0 then
   return -1
 endi
-
 if $data90 != 445.500000000 then
   return -1
 endi
-
 if $data91 != 9 then
   return -1
 endi
@@ -550,22 +475,19 @@ endi
 #  return -1
 #endi
 #
-sql_error select first(c6) - last(c6) *12 / count(*) from $stb group by c3;
+sql select first(c6) - last(c6) *12 / count(*) from $stb group by c3;
 
-sql select first(c6) - last(c6) *12 / count(*) from $stb group by c5;
+sql select first(c6) - last(c6) *12 / count(*) from $stb group by c5 order by c5;
-if $rows != 10 then
+if $rows != 11 then
   return -1
 endi
-
+if $data10 != 0.000000000 then
-if $data00 != 0.000000000 then
   return -1
 endi
-
+if $data20 != 0.997600000 then
-if $data10 != 0.997600000 then
   return -1
 endi
-
+if $data90 != 7.980800000 then
-if $data90 != 8.978400000 then
   return -1
 endi
 
@@ -574,7 +496,6 @@ sql select first(c6) - sum(c6) + 12 from $stb limit 12 offset 0;
 if $rows != 1 then
   return -1
 endi
-
 if $data00 != -449988.000000000 then
   return -1
 endi
@@ -604,10 +525,8 @@ sql_error select first(c1) from $stb fill(value, 20);
 
 # constant column. [d.13]===============================================================
 
-
 # column value filter [d.14]===============================================================
 
-
 # tag filter. [d.15]===============================================================
 sql select sum(c2)+99 from $stb where t1=12;
 
@@ -633,7 +552,6 @@ sql select avg(c2)*count(c2), sum(c3)-first(c3), last(c4)+9 from $stb interval(1
 if $rows != 10000 then
   return -1
 endi
-
 if $data00 != @18-09-17 09:00:00.000@ then
   return -1
 endi
@@ -645,11 +563,9 @@ sql_error select first(c7)- last(c1) from $tb interval(2y)
 
 # first/last query  [d.19]===============================================================
 
-
 # multiple retrieve [d.20]===============================================================
 sql select c2-c2 from $tb
 
-
 sql select first(c1)-last(c1), spread(c2), max(c3) - min(c3), avg(c4)*count(c4) from $tb
 
 

tests/script/tsim/parser/columnValue_bigint.sim

@@ -1,4 +1,3 @@
-sleep 100
 sql connect
 sql create database if not exists db
 sql use db

tests/script/tsim/parser/columnValue_bool.sim

@@ -1,4 +1,3 @@
-sleep 100
 sql connect
 sql create database if not exists db
 sql use db

tests/script/tsim/parser/columnValue_double.sim

@@ -1,5 +1,3 @@
-####
-sleep 100
 sql connect
 sql create database if not exists db
 sql use db

tests/script/tsim/parser/columnValue_float.sim

@@ -1,5 +1,3 @@
-####
-sleep 100
 sql connect
 sql create database if not exists db
 sql use db

tests/script/tsim/parser/columnValue_int.sim

@@ -1,4 +1,3 @@
-sleep 100
 sql connect
 sql create database if not exists db
 sql use db

tests/script/tsim/parser/columnValue_smallint.sim

@@ -1,5 +1,3 @@
-sleep 100
-sql connect
 sql create database if not exists db
 sql use db
 

tests/script/tsim/parser/columnValue_tinyint.sim

@@ -1,5 +1,3 @@
-sleep 100
-sql connect
 sql create database if not exists db
 sql use db
 

tests/script/tsim/parser/columnValue_unsign.sim

@@ -1,5 +1,3 @@
-sleep 100
-sql connect
 sql create database if not exists db
 sql use db
 

tests/script/tsim/parser/fill.sim

@@ -960,6 +960,7 @@ endi
 
 sql select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k) from tm0 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 4:2:15'  interval(500a) fill(value, 99,91,90,89,88,87,86,85) ;
 if $rows != 21749 then
+ print expect 21749, actual: $rows
  return -1
 endi
 

tests/script/tsim/parser/first_last_query.sim

@@ -1,4 +1,3 @@
-sleep 100
 sql connect
 
 $dbPrefix = first_db

tests/script/tsim/parser/function.sim

@@ -278,7 +278,6 @@ sql create stable td6086st(ts timestamp, d double) tags(t nchar(50));
 sql create table td6086ct1 using td6086st tags("ct1");
 sql create table td6086ct2 using td6086st tags("ct2");
 
-return
 sql SELECT LAST(d),t FROM td6086st WHERE tbname in ('td6086ct1', 'td6086ct2') and ts>="2019-07-30 00:00:00" and ts<="2021-08-31 00:00:00" partition BY tbname interval(1800s) fill(prev);
 
 print ==================> td-2624

tests/script/tsim/parser/gendata.sh

@@ -3,6 +3,6 @@
 Cur_Dir=$(pwd)
 echo $Cur_Dir
 
-echo "'2020-1-1 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> ~/data.sql
+echo "'2020-1-1 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> /tmp/data.sql
-echo "'2020-1-2 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> ~/data.sql
+echo "'2020-1-2 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> /tmp/data.sql
-echo "'2020-1-3 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> ~/data.sql
+echo "'2020-1-3 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> /tmp/data.sql

tests/script/tsim/parser/groupby-basic.sim

@@ -3,25 +3,6 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/exec.sh -n dnode1 -s start
 sql connect
 
-$loop_cnt = 0
-check_dnode_ready:
-	$loop_cnt = $loop_cnt + 1
-	sleep 200
-	if $loop_cnt == 10 then
-	  print ====> dnode not ready!
-		return -1
-	endi
-sql show dnodes
-print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
-if $data00 != 1 then
-  return -1
-endi
-if $data04 != ready then
-  goto check_dnode_ready
-endi
-
-sql connect
-
 $dbPrefix = group_db
 $tbPrefix = group_tb
 $mtPrefix = group_mt
@@ -80,8 +61,6 @@ while $i < $tbNum
   $tstart = 1640966400000
 endw
 
-sleep 100
-
 $i1 = 1
 $i2 = 0
 
@@ -752,12 +731,7 @@ sql insert into tm1 values('2020-2-1 1:1:1', 2, 10);
 sql insert into tm1 values('2020-2-1 1:1:2', 2, 20);
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 100
 system sh/exec.sh -n dnode1 -s start
-sleep 100
-
-sql connect
-sleep 100
 sql use group_db0;
 
 print =========================>TD-4894

tests/script/tsim/parser/import_file.sim

@@ -7,7 +7,7 @@ sql drop database if exists indb
 sql create database if not exists indb
 sql use indb
 
-$inFileName = '~/data.csv'
+$inFileName = '/tmp/data.csv'
 $numOfRows = 10000
 system tsim/parser/gendata.sh
 
@@ -16,8 +16,8 @@ sql create table stbx (ts TIMESTAMP, collect_area NCHAR(12), device_id BINARY(16
 sql create table tbx (ts TIMESTAMP, collect_area NCHAR(12), device_id BINARY(16), imsi BINARY(16),  imei BINARY(16),  mdn BINARY(10), net_type BINARY(4), mno NCHAR(4), province NCHAR(10), city NCHAR(16), alarm BINARY(2))
 print ====== create tables success, starting insert data
 
-sql insert into tbx file '~/data.sql'
+sql insert into tbx file '/tmp/data.sql'
-sql import into tbx file '~/data.sql'
+sql import into tbx file '/tmp/data.sql'
 
 sql select count(*) from tbx
 if $rows != 1 then
@@ -31,8 +31,8 @@ endi
 
 sql drop table tbx;
 
-sql insert into tbx using stbx tags(1,'abc') file '~/data.sql';
+sql insert into tbx using stbx tags(1,'abc') file '/tmp/data.sql';
-sql insert into tbx using stbx tags(1,'abc') file '~/data.sql';
+sql insert into tbx using stbx tags(1,'abc') file '/tmp/data.sql';
 
 sql select count(*) from tbx
 if $rows != 1 then
@@ -44,7 +44,7 @@ if $data00 != 3 then
 endi
 
 sql drop table tbx;
-sql insert into tbx using stbx(b) tags('abcf') file '~/data.sql';
+sql insert into tbx using stbx(b) tags('abcf') file '/tmp/data.sql';
 
 sql select ts,a,b from tbx;
 if $rows != 3 then
@@ -64,6 +64,6 @@ if $data02 != @abcf@ then
   return -1
 endi
 
-system rm -f ~/data.sql
+system rm -f /tmp/data.sql
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT