Merge branch '3.0' into fix/3_liaohj

docs/en/08-client-libraries/07-python.mdx

@@ -842,12 +842,12 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"})
 
 In addition to native connections, the client library also supports subscriptions via websockets.
 
-The syntax for creating a consumer is "consumer = consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer).
+The syntax for creating a consumer is "consumer = Consumer(conf=configs)". You need to specify that the `td.connect.websocket.scheme` parameter is set to "ws" in the configuration. For more subscription api parameters, please refer to [Data Subscription](../../develop/tmq/#create-a-consumer).
 
 ```python
 import taosws
 
-consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
+consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
 ```
 
 </TabItem>
@@ -887,13 +887,13 @@ The `poll` function is used to consume data in tmq. The parameter of the `poll`
 
 ```python
 while True:
-    res = consumer.poll(1)
+    message = consumer.poll(1)
-    if not res:
+    if not message:
         continue
-    err = res.error()
+    err = message.error()
     if err is not None:
         raise err
-    val = res.value()
+    val = message.value()
 
     for block in val:
         print(block.fetchall())
@@ -902,16 +902,14 @@ while True:
 </TabItem>
 <TabItem value="websocket" label="WebSocket connection">
 
-The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. You have to handle error messages in response data.
+The `poll` function is used to consume data in tmq. The parameter of the `poll` function is a value of type float representing the timeout in seconds. It returns a `Message` before timing out, or `None` on timing out. 
 
 ```python
 while True:
-    res = consumer.poll(timeout=1.0)
+    message = consumer.poll(1)
-    if not res:
+    if not message:
         continue
-    err = res.error()
+    
-    if err is not None:
-        raise err
     for block in message:
         for row in block:
             print(row)

docs/en/12-taos-sql/14-stream.md

@@ -41,16 +41,28 @@ window_clause: {
     SESSION(ts_col, tol_val)
   | STATE_WINDOW(col)
   | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)]
+  | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
+  | COUNT_WINDOW(count_val[, sliding_val])
 }
 ```
 
 `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.
 
+`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns.
+
+`COUNT_WINDOW` is a counting window that is divided by a fixed number of data rows.`count_val`: A constant, which is a positive integer and must be greater than or equal to 2. The maximum value is 2147483648. `count_val` represents the maximum number of data rows contained in each `COUNT_WINDOW`. When the total number of data rows cannot be divided by `count_val`, the number of rows in the last window will be less than `count_val`. `sliding_val`: is a constant that represents the number of window slides, similar to `SLIDING` in `INTERVAL`.
+
 For example, the following SQL statement creates a stream and automatically creates a supertable named `avg_vol`. The stream has a 1 minute time window that slides forward in 30 second intervals to calculate the average voltage of the meters supertable.
 
 ```sql
 CREATE STREAM avg_vol_s INTO avg_vol AS
 SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
+
+CREATE STREAM streams0 INTO streamt0 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9;
+
+CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
 ```
 
 ## Partitions of Stream

docs/zh/08-connector/30-python.mdx

@@ -856,7 +856,7 @@ taosws `Consumer` API 提供了基于 Websocket 订阅 TMQ 数据的 API。创
 ```python
 import taosws
 
-consumer = taosws.(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
+consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
 ```
 
 </TabItem>
@@ -896,13 +896,13 @@ Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 flo
 
 ```python
 while True:
-    res = consumer.poll(1)
+    message = consumer.poll(1)
-    if not res:
+    if not message:
         continue
-    err = res.error()
+    err = message.error()
     if err is not None:
         raise err
-    val = res.value()
+    val = message.value()
 
     for block in val:
         print(block.fetchall())
@@ -911,16 +911,14 @@ while True:
 </TabItem>
 <TabItem value="websocket" label="WebSocket 连接">
 
-Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。消费者必须通过 Message 的 `error()` 方法校验返回数据的 error 信息。
+Consumer API 的 `poll` 方法用于消费数据，`poll` 方法接收一个 float 类型的超时时间，超时时间单位为秒（s），`poll` 方法在超时之前返回一条 Message 类型的数据或超时返回 `None`。
 
 ```python
 while True:
-    res = consumer.poll(timeout=1.0)
+    message = consumer.poll(1)
-    if not res:
+    if not message:
         continue
-    err = res.error()
+    
-    if err is not None:
-        raise err
     for block in message:
         for row in block:
             print(row)

docs/zh/12-taos-sql/14-stream.md

@@ -49,10 +49,14 @@ window_clause: {
     SESSION(ts_col, tol_val)
   | STATE_WINDOW(col)
   | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)]
+  | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition
+  | COUNT_WINDOW(count_val[, sliding_val])
 }
 ```
 
 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。
+EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
+COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。
 
 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)
 
@@ -61,6 +65,12 @@ window_clause: {
 ```sql
 CREATE STREAM avg_vol_s INTO avg_vol AS
 SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
+
+CREATE STREAM streams0 INTO streamt0 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname EVENT_WINDOW START WITH voltage < 0 END WITH voltage > 9;
+
+CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS
+SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10);
 ```
 
 ## 流式计算的 partition

include/common/tmsg.h

@@ -581,8 +581,8 @@ typedef struct {
   };
 } SSubmitRsp;
 
-int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
+// int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
-int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
+// int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
 // void    tFreeSSubmitBlkRsp(void* param);
 void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
@@ -885,8 +885,8 @@ typedef struct {
   int64_t maxStorage;
 } SCreateAcctReq, SAlterAcctReq;
 
-int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
+// int32_t tSerializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
-int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
+// int32_t tDeserializeSCreateAcctReq(void* buf, int32_t bufLen, SCreateAcctReq* pReq);
 
 typedef struct {
   char    user[TSDB_USER_LEN];
@@ -3446,7 +3446,7 @@ int32_t tDeserializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexR
 
 typedef SMDropSmaReq SDropTagIndexReq;
 
-int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
+// int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
 int32_t tDeserializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq);
 
 typedef struct {
@@ -3567,8 +3567,8 @@ typedef struct {
   int8_t igNotExists;
 } SMDropFullTextReq;
 
-int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
+// int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
-int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
+// int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
 
 typedef struct {
   char indexFName[TSDB_INDEX_FNAME_LEN];
@@ -3820,6 +3820,7 @@ typedef struct {
   uint32_t phyLen;
   char*    sql;
   char*    msg;
+  int8_t   source;
 } SVDeleteReq;
 
 int32_t tSerializeSVDeleteReq(void* buf, int32_t bufLen, SVDeleteReq* pReq);
@@ -3841,6 +3842,7 @@ typedef struct SDeleteRes {
   char     tableFName[TSDB_TABLE_NAME_LEN];
   char     tsColName[TSDB_COL_NAME_LEN];
   int64_t  ctimeMs;  // fill by vnode
+  int8_t   source;
 } SDeleteRes;
 
 int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes);

include/libs/scheduler/scheduler.h

@@ -78,6 +78,7 @@ typedef struct SSchedulerReq {
   void*              chkKillParam;
   SExecResult*       pExecRes;
   void**             pFetchRes;
+  int8_t             source;
 } SSchedulerReq;
 
 int32_t schedulerInit(void);

include/libs/stream/tstream.h

@@ -56,7 +56,6 @@ extern "C" {
 #define STREAM_EXEC_T_RESTART_ALL_TASKS (-4)
 #define STREAM_EXEC_T_STOP_ALL_TASKS    (-5)
 #define STREAM_EXEC_T_RESUME_TASK       (-6)
-#define STREAM_EXEC_T_UPDATE_TASK_EPSET (-7)
 
 typedef struct SStreamTask   SStreamTask;
 typedef struct SStreamQueue  SStreamQueue;
@@ -783,7 +782,7 @@ bool    streamTaskIsAllUpstreamClosed(SStreamTask* pTask);
 bool    streamTaskSetSchedStatusWait(SStreamTask* pTask);
 int8_t  streamTaskSetSchedStatusActive(SStreamTask* pTask);
 int8_t  streamTaskSetSchedStatusInactive(SStreamTask* pTask);
-int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt, bool metaLock);
+int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t clearRelHalt);
 
 int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
 

include/util/tdef.h

@@ -187,6 +187,8 @@ typedef enum ELogicConditionType {
   LOGIC_COND_TYPE_NOT,
 } ELogicConditionType;
 
+#define TSDB_INT32_ID_LEN 11
+
 #define TSDB_NAME_DELIMITER_LEN 1
 
 #define TSDB_UNI_LEN  24

source/client/inc/clientInt.h

@@ -284,6 +284,7 @@ typedef struct SRequestObj {
   void*                pWrapper;
   SMetaData            parseMeta;
   char*                effectiveUser;
+  int8_t               source;
 } SRequestObj;
 
 typedef struct SSyncQueryParam {
@@ -306,10 +307,10 @@ void    doFreeReqResultInfo(SReqResultInfo* pResInfo);
 int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq);
 void    syncCatalogFn(SMetaData* pResult, void* param, int32_t code);
 
-TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly);
+TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source);
 TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, int64_t reqid);
 
-void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly);
+void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source);
 void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly,
                                  int64_t reqid);
 void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param);
@@ -354,6 +355,7 @@ SRequestObj* acquireRequest(int64_t rid);
 int32_t      releaseRequest(int64_t rid);
 int32_t      removeRequest(int64_t rid);
 void         doDestroyRequest(void* p);
+int64_t      removeFromMostPrevReq(SRequestObj* pRequest);
 
 char* getDbOfConnection(STscObj* pObj);
 void  setConnectionDB(STscObj* pTscObj, const char* db);

source/client/src/clientEnv.c

@@ -385,6 +385,33 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri
 
 int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); }
 
+/// return the most previous req ref id
+int64_t removeFromMostPrevReq(SRequestObj* pRequest) {
+  int64_t mostPrevReqRefId = pRequest->self;
+  SRequestObj* pTmp = pRequest;
+  while (pTmp->relation.prevRefId) {
+    pTmp = acquireRequest(pTmp->relation.prevRefId);
+    if (pTmp) {
+      mostPrevReqRefId = pTmp->self;
+      releaseRequest(mostPrevReqRefId);
+    } else {
+      break;
+    }
+  }
+  removeRequest(mostPrevReqRefId);
+  return mostPrevReqRefId;
+}
+
+void destroyNextReq(int64_t nextRefId) {
+  if (nextRefId) {
+    SRequestObj* pObj = acquireRequest(nextRefId);
+    if (pObj) {
+      releaseRequest(nextRefId);
+      releaseRequest(nextRefId);
+    }
+  }
+}
+
 void destroySubRequests(SRequestObj *pRequest) {
   int32_t      reqIdx = -1;
   SRequestObj *pReqList[16] = {NULL};
@@ -435,7 +462,7 @@ void doDestroyRequest(void *p) {
   uint64_t reqId = pRequest->requestId;
   tscTrace("begin to destroy request %" PRIx64 " p:%p", reqId, pRequest);
 
-  destroySubRequests(pRequest);
+  int64_t nextReqRefId = pRequest->relation.nextRefId;
 
   taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self));
 
@@ -471,6 +498,7 @@ void doDestroyRequest(void *p) {
   taosMemoryFreeClear(pRequest->sqlstr);
   taosMemoryFree(pRequest);
   tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest);
+  destroyNextReq(nextReqRefId);
 }
 
 void destroyRequest(SRequestObj *pRequest) {
@@ -479,7 +507,7 @@ void destroyRequest(SRequestObj *pRequest) {
   }
 
   taos_stop_query(pRequest);
-  removeRequest(pRequest->self);
+  removeFromMostPrevReq(pRequest);
 }
 
 void taosStopQueryImpl(SRequestObj *pRequest) {

source/client/src/clientImpl.c

@@ -743,6 +743,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
          .chkKillFp = chkRequestKilled,
          .chkKillParam = (void*)pRequest->self,
          .pExecRes = &res,
+         .source = pRequest->source,
   };
 
   int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob);
@@ -1212,6 +1213,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
            .chkKillFp = chkRequestKilled,
            .chkKillParam = (void*)pRequest->self,
            .pExecRes = NULL,
+           .source = pRequest->source,
     };
     code = schedulerExecJob(&req, &pRequest->body.queryJob);
     taosArrayDestroy(pNodeList);
@@ -2475,7 +2477,7 @@ void syncQueryFn(void* param, void* res, int32_t code) {
   tsem_post(&pParam->sem);
 }
 
-void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) {
+void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, int8_t source) {
   if (sql == NULL || NULL == fp) {
     terrno = TSDB_CODE_INVALID_PARA;
     if (fp) {
@@ -2501,6 +2503,7 @@ void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp,
     return;
   }
 
+  pRequest->source = source;
   pRequest->body.queryFp = fp;
   doAsyncQuery(pRequest, false);
 }
@@ -2535,7 +2538,7 @@ void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_
   doAsyncQuery(pRequest, false);
 }
 
-TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
+TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly, int8_t source) {
   if (NULL == taos) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     return NULL;
@@ -2550,7 +2553,7 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
   }
   tsem_init(&param->sem, 0, 0);
 
-  taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
+  taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, source);
   tsem_wait(&param->sem);
 
   SRequestObj* pRequest = NULL;

source/client/src/clientMain.c

@@ -402,7 +402,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
   return pResInfo->userFields;
 }
 
-TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false); }
+TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false, TD_REQ_FROM_APP); }
 TAOS_RES *taos_query_with_reqid(TAOS *taos, const char *sql, int64_t reqid) {
   return taosQueryImplWithReqid(taos, sql, false, reqid);
 }
@@ -828,7 +828,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
 }
 
 int taos_validate_sql(TAOS *taos, const char *sql) {
-  TAOS_RES *pObj = taosQueryImpl(taos, sql, true);
+  TAOS_RES *pObj = taosQueryImpl(taos, sql, true, TD_REQ_FROM_APP);
 
   int code = taos_errno(pObj);
 
@@ -1126,7 +1126,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
 void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
   int64_t connId = *(int64_t *)taos;
   tscDebug("taos_query_a start with sql:%s", sql);
-  taosAsyncQueryImpl(connId, sql, fp, param, false);
+  taosAsyncQueryImpl(connId, sql, fp, param, false, TD_REQ_FROM_APP);
   tscDebug("taos_query_a end with sql:%s", sql);
 }
 
@@ -1254,54 +1254,34 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
 }
 
 void restartAsyncQuery(SRequestObj *pRequest, int32_t code) {
-  int32_t      reqIdx = 0;
+  tscInfo("restart request: %s p: %p", pRequest->sqlstr, pRequest);
-  SRequestObj *pReqList[16] = {NULL};
+  SRequestObj* pUserReq = pRequest;
-  SRequestObj *pUserReq = NULL;
+  acquireRequest(pRequest->self);
-  pReqList[0] = pRequest;
+  while (pUserReq) {
-  uint64_t     tmpRefId = 0;
+    if (pUserReq->self == pUserReq->relation.userRefId || pUserReq->relation.userRefId == 0) {
-  SRequestObj *pTmp = pRequest;
-  while (pTmp->relation.prevRefId) {
-    tmpRefId = pTmp->relation.prevRefId;
-    pTmp = acquireRequest(tmpRefId);
-    if (pTmp) {
-      pReqList[++reqIdx] = pTmp;
-      releaseRequest(tmpRefId);
-    } else {
-      tscError("prev req ref 0x%" PRIx64 " is not there", tmpRefId);
       break;
-    }
-  }
-
-  tmpRefId = pRequest->relation.nextRefId;
-  while (tmpRefId) {
-    pTmp = acquireRequest(tmpRefId);
-    if (pTmp) {
-      tmpRefId = pTmp->relation.nextRefId;
-      removeRequest(pTmp->self);
-      releaseRequest(pTmp->self);
     } else {
-      tscError("next req ref 0x%" PRIx64 " is not there", tmpRefId);
+      int64_t nextRefId = pUserReq->relation.nextRefId;
-      break;
+      releaseRequest(pUserReq->self);
+      if (nextRefId) {
+        pUserReq = acquireRequest(nextRefId);
       }
     }
-
-  for (int32_t i = reqIdx; i >= 0; i--) {
-    destroyCtxInRequest(pReqList[i]);
-    if (pReqList[i]->relation.userRefId == pReqList[i]->self || 0 == pReqList[i]->relation.userRefId) {
-      pUserReq = pReqList[i];
-    } else {
-      removeRequest(pReqList[i]->self);
   }
-  }
+  bool hasSubRequest = pUserReq != pRequest || pRequest->relation.prevRefId != 0;
-
   if (pUserReq) {
+    destroyCtxInRequest(pUserReq);
     pUserReq->prevCode = code;
     memset(&pUserReq->relation, 0, sizeof(pUserReq->relation));
   } else {
-    tscError("user req is missing");
+    tscError("User req is missing");
+    removeFromMostPrevReq(pRequest);
     return;
   }
-
+  if (hasSubRequest)
+    removeFromMostPrevReq(pRequest);
+  else
+    releaseRequest(pUserReq->self);
   doAsyncQuery(pUserReq, true);
 }
 

source/client/src/clientRawBlockWrite.c

@@ -1256,7 +1256,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
   snprintf(sql, sizeof(sql), "delete from `%s` where `%s` >= %" PRId64 " and `%s` <= %" PRId64, req.tableFName,
            req.tsColName, req.skey, req.tsColName, req.ekey);
 
-  TAOS_RES*    res = taos_query(taos, sql);
+  TAOS_RES*    res = taosQueryImpl(taos, sql, false, TD_REQ_FROM_TAOX);
   SRequestObj* pRequest = (SRequestObj*)res;
   code = pRequest->code;
   if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_PAR_GET_META_ERROR) {

source/client/src/clientTmq.c

@@ -1010,19 +1010,8 @@ int32_t tmq_unsubscribe(tmq_t* tmq) {
   }
   taosSsleep(2);  // sleep 2s for hb to send offset and rows to server
 
-  int32_t     rsp;
-  int32_t     retryCnt = 0;
   tmq_list_t* lst = tmq_list_new();
-  while (1) {
+  int32_t rsp = tmq_subscribe(tmq, lst);
-    rsp = tmq_subscribe(tmq, lst);
-    if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) {
-      break;
-    } else {
-      retryCnt++;
-      taosMsleep(500);
-    }
-  }
-
   tmq_list_destroy(lst);
   return rsp;
 }
@@ -1272,10 +1261,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   }
 
   int32_t retryCnt = 0;
-  while (syncAskEp(tmq) != 0) {
+  while ((code = syncAskEp(tmq)) != 0) {
-    if (retryCnt++ > MAX_RETRY_COUNT) {
+    if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) {
       tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes", tmq->consumerId);
-      code = TSDB_CODE_MND_CONSUMER_NOT_READY;
       goto FAIL;
     }
 
@@ -2148,26 +2136,19 @@ int32_t tmq_consumer_close(tmq_t* tmq) {
   if (tmq->status == TMQ_CONSUMER_STATUS__READY) {
     // if auto commit is set, commit before close consumer. Otherwise, do nothing.
     if (tmq->autoCommit) {
-      int32_t rsp = tmq_commit_sync(tmq, NULL);
+      int32_t code = tmq_commit_sync(tmq, NULL);
-      if (rsp != 0) {
+      if (code != 0) {
-        return rsp;
+        return code;
       }
     }
     taosSsleep(2);  // sleep 2s for hb to send offset and rows to server
 
-    int32_t     retryCnt = 0;
     tmq_list_t* lst = tmq_list_new();
-    while (1) {
+    int32_t code = tmq_subscribe(tmq, lst);
-      int32_t rsp = tmq_subscribe(tmq, lst);
-      if (rsp != TSDB_CODE_MND_CONSUMER_NOT_READY || retryCnt > 5) {
-        break;
-      } else {
-        retryCnt++;
-        taosMsleep(500);
-      }
-    }
-
     tmq_list_destroy(lst);
+    if (code != 0) {
+      return code;
+    }
   } else {
     tscInfo("consumer:0x%" PRIx64 " not in ready state, close it directly", tmq->consumerId);
   }

source/common/src/tglobal.c

@@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4;
 int32_t tsNumOfMnodeFetchThreads = 1;
 int32_t tsNumOfMnodeReadThreads = 1;
 int32_t tsNumOfVnodeQueryThreads = 4;
-float   tsRatioOfVnodeStreamThreads = 1.5F;
+float   tsRatioOfVnodeStreamThreads = 0.5F;
 int32_t tsNumOfVnodeFetchThreads = 4;
 int32_t tsNumOfVnodeRsmaThreads = 2;
 int32_t tsNumOfQnodeQueryThreads = 4;
@@ -586,7 +586,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   tsNumOfSupportVnodes = tsNumOfCores * 2;
   tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
-  if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
   if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)

source/common/src/tmsg.c

@@ -1009,19 +1009,19 @@ int32_t tDeserializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexR
   tDecoderClear(&decoder);
   return 0;
 }
-int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
+// int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
-  SEncoder encoder = {0};
+//   SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+//   tEncoderInit(&encoder, buf, bufLen);
-  if (tStartEncode(&encoder) < 0) return -1;
+//   if (tStartEncode(&encoder) < 0) return -1;
-  tEndEncode(&encoder);
+//   tEndEncode(&encoder);
 
-  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
 
-  int32_t tlen = encoder.pos;
+//   int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
+//   tEncoderClear(&encoder);
-  return tlen;
+//   return tlen;
-}
+// }
 int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) {
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
@@ -1035,6 +1035,7 @@ int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *
 
   return 0;
 }
+
 int32_t tSerializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -1059,32 +1060,32 @@ void tFreeSMCreateFullTextReq(SMCreateFullTextReq *pReq) {
   // impl later
   return;
 }
-int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
+// int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
-  SEncoder encoder = {0};
+//   SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+//   tEncoderInit(&encoder, buf, bufLen);
 
-  if (tStartEncode(&encoder) < 0) return -1;
+//   if (tStartEncode(&encoder) < 0) return -1;
 
-  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
 
-  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+//   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
 
-  tEndEncode(&encoder);
+//   tEndEncode(&encoder);
-  int32_t tlen = encoder.pos;
+//   int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
+//   tEncoderClear(&encoder);
-  return tlen;
+//   return tlen;
-}
+// }
-int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
+// int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+//   if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
 
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) {
   SEncoder encoder = {0};
@@ -1474,44 +1475,44 @@ void tFreeSStatisReq(SStatisReq *pReq) {
   taosMemoryFreeClear(pReq->pCont);
 }
 
-int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
+// int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
-  SEncoder encoder = {0};
+//   SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
+//   tEncoderInit(&encoder, buf, bufLen);
 
-  if (tStartEncode(&encoder) < 0) return -1;
+//   if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
+//   if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxUsers) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxDbs) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxTimeSeries) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->maxStreams) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1;
+//   if (tEncodeI32(&encoder, pReq->accessState) < 0) return -1;
-  if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1;
+//   if (tEncodeI64(&encoder, pReq->maxStorage) < 0) return -1;
-  tEndEncode(&encoder);
+//   tEndEncode(&encoder);
 
-  int32_t tlen = encoder.pos;
+//   int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
+//   tEncoderClear(&encoder);
-  return tlen;
+//   return tlen;
-}
+// }
 
-int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
+// int32_t tDeserializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) {
-  SDecoder decoder = {0};
+//   SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
+//   tDecoderInit(&decoder, buf, bufLen);
 
-  if (tStartDecode(&decoder) < 0) return -1;
+//   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
-  if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
+//   if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxUsers) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxDbs) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxTimeSeries) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->maxStreams) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1;
+//   if (tDecodeI32(&decoder, &pReq->accessState) < 0) return -1;
-  if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1;
+//   if (tDecodeI64(&decoder, &pReq->maxStorage) < 0) return -1;
-  tEndDecode(&decoder);
+//   tEndDecode(&decoder);
 
-  tDecoderClear(&decoder);
+//   tDecoderClear(&decoder);
-  return 0;
+//   return 0;
-}
+// }
 
 int32_t tSerializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) {
   SEncoder encoder = {0};
@@ -7191,6 +7192,7 @@ int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
   if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
   if (tEncodeBinary(&encoder, pReq->msg, pReq->phyLen) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -7227,6 +7229,9 @@ int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) {
   if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, &msgLen) < 0) return -1;
   pReq->phyLen = msgLen;
 
+  if (!tDecodeIsEnd(&decoder)) {
+    if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
+  }
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
@@ -7934,64 +7939,64 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl
   return 0;
 }
 
-static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
+// static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
-  if (tStartDecode(pDecoder) < 0) return -1;
+//   if (tStartDecode(pDecoder) < 0) return -1;
 
-  if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
+//   if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
-  if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
+//   if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
-  pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
+//   pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
-  if (NULL == pBlock->tblFName) return -1;
+//   if (NULL == pBlock->tblFName) return -1;
-  if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1;
+//   if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pBlock->numOfRows) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pBlock->affectedRows) < 0) return -1;
-  if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1;
+//   if (tDecodeI64v(pDecoder, &pBlock->sver) < 0) return -1;
 
-  int32_t meta = 0;
+//   int32_t meta = 0;
-  if (tDecodeI32(pDecoder, &meta) < 0) return -1;
+//   if (tDecodeI32(pDecoder, &meta) < 0) return -1;
-  if (meta) {
+//   if (meta) {
-    pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
+//     pBlock->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
-    if (NULL == pBlock->pMeta) return -1;
+//     if (NULL == pBlock->pMeta) return -1;
-    if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1;
+//     if (tDecodeSTableMetaRsp(pDecoder, pBlock->pMeta) < 0) return -1;
-  } else {
+//   } else {
-    pBlock->pMeta = NULL;
+//     pBlock->pMeta = NULL;
-  }
+//   }
 
-  tEndDecode(pDecoder);
+//   tEndDecode(pDecoder);
-  return 0;
+//   return 0;
-}
+// }
 
-int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) {
+// int32_t tEncodeSSubmitRsp(SEncoder *pEncoder, const SSubmitRsp *pRsp) {
-  int32_t nBlocks = taosArrayGetSize(pRsp->pArray);
+//   int32_t nBlocks = taosArrayGetSize(pRsp->pArray);
 
-  if (tStartEncode(pEncoder) < 0) return -1;
+//   if (tStartEncode(pEncoder) < 0) return -1;
 
-  if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1;
+//   if (tEncodeI32v(pEncoder, pRsp->numOfRows) < 0) return -1;
-  if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1;
+//   if (tEncodeI32v(pEncoder, pRsp->affectedRows) < 0) return -1;
-  if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1;
+//   if (tEncodeI32v(pEncoder, nBlocks) < 0) return -1;
-  for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) {
+//   for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) {
-    if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1;
+//     if (tEncodeSSubmitBlkRsp(pEncoder, (SSubmitBlkRsp *)taosArrayGet(pRsp->pArray, iBlock)) < 0) return -1;
-  }
+//   }
 
-  tEndEncode(pEncoder);
+//   tEndEncode(pEncoder);
-  return 0;
+//   return 0;
-}
+// }
 
-int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
+// int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
-  if (tStartDecode(pDecoder) < 0) return -1;
+//   if (tStartDecode(pDecoder) < 0) return -1;
 
-  if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pRsp->numOfRows) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pRsp->affectedRows) < 0) return -1;
-  if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1;
+//   if (tDecodeI32v(pDecoder, &pRsp->nBlocks) < 0) return -1;
-  pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks));
+//   pRsp->pBlocks = taosMemoryCalloc(pRsp->nBlocks, sizeof(*pRsp->pBlocks));
-  if (pRsp->pBlocks == NULL) return -1;
+//   if (pRsp->pBlocks == NULL) return -1;
-  for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) {
+//   for (int32_t iBlock = 0; iBlock < pRsp->nBlocks; iBlock++) {
-    if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1;
+//     if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1;
-  }
+//   }
 
-  tEndDecode(pDecoder);
+//   tEndDecode(pDecoder);
-  tDecoderClear(pDecoder);
+//   tDecoderClear(pDecoder);
-  return 0;
+//   return 0;
-}
+// }
 
 // void tFreeSSubmitBlkRsp(void *param) {
 //   if (NULL == param) {
@@ -8426,6 +8431,7 @@ int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) {
   if (tEncodeCStr(pCoder, pRes->tableFName) < 0) return -1;
   if (tEncodeCStr(pCoder, pRes->tsColName) < 0) return -1;
   if (tEncodeI64(pCoder, pRes->ctimeMs) < 0) return -1;
+  if (tEncodeI8(pCoder, pRes->source) < 0) return -1;
   return 0;
 }
 
@@ -8450,6 +8456,9 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) {
   if (!tDecodeIsEnd(pCoder)) {
     if (tDecodeI64(pCoder, &pRes->ctimeMs) < 0) return -1;
   }
+  if (!tDecodeIsEnd(pCoder)) {
+    if (tDecodeI8(pCoder, &pRes->source) < 0) return -1;
+  }
   return 0;
 }
 

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -345,6 +345,7 @@ int32_t dmInitClient(SDnode *pDnode) {
   rpcInit.parent = pDnode;
   rpcInit.rfp = rpcRfp;
   rpcInit.compressSize = tsCompressMsgSize;
+  rpcInit.dfp = destroyAhandle;
 
   rpcInit.retryMinInterval = tsRedirectPeriod;
   rpcInit.retryStepFactor = tsRedirectFactor;

source/dnode/mnode/impl/inc/mndStream.h

@@ -111,7 +111,7 @@ STrans  *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, const
 int32_t  mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
 SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
 void     killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo);
-int32_t  mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
+int32_t  mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
 
 SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId);
 int32_t     extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId);

source/dnode/mnode/impl/src/mndCompact.c

@@ -454,7 +454,7 @@ int32_t mndProcessKillCompactReq(SRpcMsg *pReq){
 
   code = TSDB_CODE_ACTION_IN_PROGRESS;
 
-  char obj[MND_COMPACT_ID_LEN] = {0};
+  char obj[TSDB_INT32_ID_LEN] = {0};
   sprintf(obj, "%d", pCompact->compactId);
 
   auditRecord(pReq, pMnode->clusterId, "killCompact", pCompact->dbname, obj, killCompactReq.sql, killCompactReq.sqlLen);

source/dnode/mnode/impl/src/mndDnode.c

@@ -1409,24 +1409,6 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
   if (strcasecmp(cfgReq.config, "resetlog") == 0) {
     strcpy(dcfgReq.config, "resetlog");
 #ifdef TD_ENTERPRISE
-  } else if (strncasecmp(cfgReq.config, "supportvnodes", 13) == 0) {
-    int32_t optLen = strlen("supportvnodes");
-    int32_t flag = -1;
-    int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
-    if (code < 0) return code;
-
-    if (flag < 0 || flag > 4096) {
-      mError("dnode:%d, failed to config supportVnodes since value:%d. Valid range: [0, 4096]", cfgReq.dnodeId, flag);
-      terrno = TSDB_CODE_OUT_OF_RANGE;
-      goto _err_out;
-    }
-    if (flag == 0) {
-      flag = tsNumOfCores * 2;
-    }
-    flag = TMAX(flag, 2);
-
-    strcpy(dcfgReq.config, "supportvnodes");
-    snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
   } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) {
     int32_t optLen = strlen("s3blocksize");
     int32_t flag = -1;

source/dnode/mnode/impl/src/mndMain.c

@@ -709,7 +709,8 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
 
   int32_t code = syncProcessMsg(pMgmt->sync, pMsg);
   if (code != 0) {
-    mGError("vgId:1, failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr());
+    mGError("vgId:1, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pMsg, TMSG_INFO(pMsg->msgType),
+            terrstr(), code);
   }
 
   return code;

source/dnode/mnode/impl/src/mndStream.c

@@ -1815,7 +1815,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
     mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid,
            pStream->name, pTrans->id);
 
-    int32_t code = mndStreamSetUpdateEpsetAction(pStream, pChangeInfo, pTrans);
+    int32_t code = mndStreamSetUpdateEpsetAction(pMnode, pStream, pChangeInfo, pTrans);
 
     // todo: not continue, drop all and retry again
     if (code != TSDB_CODE_SUCCESS) {

source/dnode/mnode/impl/src/mndStreamUtil.c

@@ -462,14 +462,22 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) {
+static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask, SVgroupChangeInfo *pInfo) {
   void   *pBuf = NULL;
   int32_t len = 0;
   streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
 
   doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
 
-  int32_t code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0);
+  SEpSet  epset = {0};
+  bool    hasEpset = false;
+  int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
+  if (code != TSDB_CODE_SUCCESS || !hasEpset) {
+    terrno = code;
+    return code;
+  }
+
+  code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID);
   if (code != TSDB_CODE_SUCCESS) {
     taosMemoryFree(pBuf);
   }
@@ -478,14 +486,14 @@ static int32_t doSetUpdateTaskAction(STrans *pTrans, SStreamTask *pTask, SVgroup
 }
 
 // build trans to update the epset
-int32_t mndStreamSetUpdateEpsetAction(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
+int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
   mDebug("stream:0x%" PRIx64 " set tasks epset update action", pStream->uid);
   taosWLockLatch(&pStream->lock);
 
   SStreamTaskIter *pIter = createStreamTaskIter(pStream);
   while (streamTaskIterNextTask(pIter)) {
     SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
-    int32_t      code = doSetUpdateTaskAction(pTrans, pTask, pInfo);
+    int32_t      code = doSetUpdateTaskAction(pMnode, pTrans, pTask, pInfo);
     if (code != TSDB_CODE_SUCCESS) {
       destroyStreamTaskIter(pIter);
       taosWUnLockLatch(&pStream->lock);

source/dnode/vnode/src/tq/tqUtil.c

@@ -263,8 +263,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
           } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) {
             PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq)
           } else if (pHead->msgType == TDMT_VND_DELETE) {
-            fetchVer++;
+            PROCESS_EXCLUDED_MSG(SDeleteRes, tDecodeDeleteRes)
-            continue;
           }
         }
 

source/dnode/vnode/src/tqCommon/tqCommon.c

@@ -142,8 +142,10 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
   if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
     ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id));
     if (ppHTask == NULL || *ppHTask == NULL) {
-      tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already",
+      tqError(
-              vgId, req.taskId);
+          "vgId:%d failed to acquire fill-history task:0x%x when handling update, may have been dropped already, rel "
+          "stream task:0x%x",
+          vgId, (uint32_t)pTask->hTaskInfo.id.taskId, req.taskId);
       CLEAR_RELATED_FILLHISTORY_TASK(pTask);
     } else {
       tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr);
@@ -612,23 +614,35 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
 
 int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) {
   SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
-
   int32_t              vgId = pMeta->vgId;
+  STaskId              hTaskId = {0};
+
   tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId);
 
-  SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
+  streamMetaWLock(pMeta);
-  if (pTask != NULL) {
+
-    // drop the related fill-history task firstly
+  STaskId       id = {.streamId = pReq->streamId, .taskId = pReq->taskId};
+  SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
+  if ((ppTask != NULL) && ((*ppTask) != NULL)) {
+    streamMetaAcquireOneTask(*ppTask);
+    SStreamTask* pTask = *ppTask;
+
     if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
-      STaskId* pHTaskId = &pTask->hTaskInfo.id;
+      hTaskId.streamId = pTask->hTaskInfo.id.streamId;
-      streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId);
+      hTaskId.taskId = pTask->hTaskInfo.id.taskId;
-      tqDebug("s-task:0x%x vgId:%d drop fill-history task:0x%x firstly", pReq->taskId, vgId,
-              (int32_t)pHTaskId->taskId);
     }
+
+    streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt);
     streamMetaReleaseTask(pMeta, pTask);
   }
 
-  streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt, true);
+  streamMetaWUnLock(pMeta);
+
+  // drop the related fill-history task firstly
+  if (hTaskId.taskId != 0 && hTaskId.streamId != 0) {
+    streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
+    tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
+  }
 
   // drop the stream task now
   streamMetaUnregisterTask(pMeta, pReq->streamId, pReq->taskId);

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -372,8 +372,8 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
 
   int32_t code = syncProcessMsg(pVnode->sync, pMsg);
   if (code != 0) {
-    vGError("vgId:%d, failed to process sync msg:%p type:%s since %s", pVnode->config.vgId, pMsg,
+    vGError("vgId:%d, failed to process sync msg:%p type:%s, errno: %s, code:0x%x", pVnode->config.vgId, pMsg,
-            TMSG_INFO(pMsg->msgType), terrstr());
+            TMSG_INFO(pMsg->msgType), terrstr(), code);
   }
 
   return code;

source/libs/executor/src/dynqueryctrloperator.c

@@ -457,6 +457,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS
     }
   }
 
+  bool initParam = pSrcParam0 ? true : false;  
   if (TSDB_CODE_SUCCESS == code) {
     code = buildGroupCacheOperatorParam(&pGcParam0, 0, *leftVg, *leftUid, pPost->leftNeedCache, pSrcParam0);
     pSrcParam0 = NULL;
@@ -466,7 +467,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS
     pSrcParam1 = NULL;
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = buildMergeJoinOperatorParam(ppParam, pSrcParam0 ? true : false, pGcParam0, pGcParam1);
+    code = buildMergeJoinOperatorParam(ppParam, initParam, pGcParam0, pGcParam1);
   }
   if (TSDB_CODE_SUCCESS != code) {
     if (pSrcParam0) {

source/libs/executor/src/streamcountwindowoperator.c

@@ -390,10 +390,12 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
   buf = taosDecodeFixedI32(buf, &mapSize);
   for (int32_t i = 0; i < mapSize; i++) {
     SSessionKey       key = {0};
-    SResultWindowInfo winfo = {0};
+    SCountWindowInfo curWin = {0};
     buf = decodeSSessionKey(buf, &key);
-    buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
+    SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
-    tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
+    setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
+    buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
+    tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
   }
 
   // 2.twAggSup
@@ -694,6 +696,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->recvGetAll = false;
 
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT;
+  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, true,
+                  OP_NOT_OPENED, pInfo, pTaskInfo);
   // for stream
   void*   buff = NULL;
   int32_t len = 0;
@@ -704,8 +708,6 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
     doStreamCountDecodeOpState(buff, len, pOperator, true);
     taosMemoryFree(buff);
   }
-  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, true,
-                  OP_NOT_OPENED, pInfo, pTaskInfo);
   pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamCountAgg, NULL, destroyStreamCountAggOperatorInfo,
                                          optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
   setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState);

source/libs/executor/src/streameventwindowoperator.c

@@ -406,6 +406,7 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
   if (!pInfo) {
     return buf;
   }
+  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
 
   // 4.checksum
   int32_t dataLen = len - sizeof(uint32_t);
@@ -423,6 +424,8 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
     SSessionKey       key = {0};
     SResultWindowInfo winfo = {0};
     buf = decodeSSessionKey(buf, &key);
+    pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap,
+                                                        (void**)&winfo.pStatePos, &pAggSup->resultRowSize);
     buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
     tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
   }
@@ -735,6 +738,8 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->reCkBlock = false;
   pInfo->recvGetAll = false;
 
+  setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED,
+                  pInfo, pTaskInfo);
   // for stream
   void*   buff = NULL;
   int32_t len = 0;
@@ -746,8 +751,6 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
     taosMemoryFree(buff);
   }
 
-  setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED,
-                  pInfo, pTaskInfo);
   pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo,
                                          optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
   setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);

source/libs/executor/src/streamtimewindowoperator.c

@@ -2533,7 +2533,6 @@ int32_t encodeSResultWindowInfo(void** buf, SResultWindowInfo* key, int32_t outL
 
 void* decodeSResultWindowInfo(void* buf, SResultWindowInfo* key, int32_t outLen) {
   buf = taosDecodeFixedBool(buf, &key->isOutput);
-  key->pStatePos->pRowBuff = NULL;
   buf = decodeSSessionKey(buf, &key->sessionWin);
   return buf;
 }
@@ -2591,6 +2590,7 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
   if (!pInfo) {
     return buf;
   }
+  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
 
   // 5.checksum
   if (isParent) {
@@ -2609,6 +2609,8 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
     SSessionKey       key = {0};
     SResultWindowInfo winfo = {0};
     buf = decodeSSessionKey(buf, &key);
+    pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &winfo.sessionWin, pAggSup->gap,
+                                                        (void**)&winfo.pStatePos, &pAggSup->resultRowSize);
     buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
     tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
   }
@@ -2992,6 +2994,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
   pInfo->recvGetAll = false;
 
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
+  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
+                  OP_NOT_OPENED, pInfo, pTaskInfo);
   // for stream
   void*   buff = NULL;
   int32_t len = 0;
@@ -3002,8 +3006,6 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
     doStreamSessionDecodeOpState(buff, len, pOperator, true);
     taosMemoryFree(buff);
   }
-  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
-                  OP_NOT_OPENED, pInfo, pTaskInfo);
   pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo,
                                          optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
   setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
@@ -3538,6 +3540,7 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
   if (!pInfo) {
     return buf;
   }
+  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
 
   // 5.checksum
   if (isParent) {
@@ -3556,6 +3559,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
     SSessionKey       key = {0};
     SResultWindowInfo winfo = {0};
     buf = decodeSSessionKey(buf, &key);
+    pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL,
+                                                      pAggSup->stateKeySize, compareStateKey,
+                                                      (void**)&winfo.pStatePos, &pAggSup->resultRowSize);
     buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
     tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
   }
@@ -3873,6 +3879,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
   pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
   pInfo->recvGetAll = false;
 
+  setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
+                  pInfo, pTaskInfo);
   // for stream
   void*   buff = NULL;
   int32_t len = 0;
@@ -3884,8 +3892,6 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
     taosMemoryFree(buff);
   }
 
-  setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
-                  pInfo, pTaskInfo);
   pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo,
                                          optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
   setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);

source/libs/executor/src/tsort.c

@@ -1199,6 +1199,18 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
   return code;
 }
 
+static void freeSSortSource(SSortSource* source) {
+  if (NULL == source) return;
+  if (source->param && !source->onlyRef) {
+    taosMemoryFree(source->param);
+  }
+  if (!source->onlyRef && source->src.pBlock) {
+    blockDataDestroy(source->src.pBlock);
+    source->src.pBlock = NULL;
+  }
+  taosMemoryFree(source);
+}
+
 static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
   int32_t code = 0;
   size_t  sortBufSize = pHandle->numOfPages * pHandle->pageSize;
@@ -1231,14 +1243,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
 
     code = blockDataMerge(pHandle->pDataBlock, pBlock);
     if (code != TSDB_CODE_SUCCESS) {
-      if (source->param && !source->onlyRef) {
+      freeSSortSource(source);
-        taosMemoryFree(source->param);
-      }
-      if (!source->onlyRef && source->src.pBlock) {
-        blockDataDestroy(source->src.pBlock);
-        source->src.pBlock = NULL;
-      }
-      taosMemoryFree(source);
       return code;
     }
 
@@ -1248,15 +1253,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
       int64_t p = taosGetTimestampUs();
       code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
       if (code != 0) {
-        if (source->param && !source->onlyRef) {
+        freeSSortSource(source);
-          taosMemoryFree(source->param);
-        }
-        if (!source->onlyRef && source->src.pBlock) {
-          blockDataDestroy(source->src.pBlock);
-          source->src.pBlock = NULL;
-        }
-
-        taosMemoryFree(source);
         return code;
       }
 
@@ -1265,16 +1262,13 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
       if (pHandle->pqMaxRows > 0) blockDataKeepFirstNRows(pHandle->pDataBlock, pHandle->pqMaxRows);
       code = doAddToBuf(pHandle->pDataBlock, pHandle);
       if (code != TSDB_CODE_SUCCESS) {
+        freeSSortSource(source);
         return code;
       }
     }
   }
 
-  if (source->param && !source->onlyRef) {
+  freeSSortSource(source);
-    taosMemoryFree(source->param);
-  }
-
-  taosMemoryFree(source);
 
   if (pHandle->pDataBlock != NULL && pHandle->pDataBlock->info.rows > 0) {
     size_t size = blockDataGetSize(pHandle->pDataBlock);

source/libs/parser/src/parTranslater.c

@@ -4009,6 +4009,26 @@ static int32_t translateEventWindow(STranslateContext* pCxt, SSelectStmt* pSelec
 }
 
 static int32_t translateCountWindow(STranslateContext* pCxt, SSelectStmt* pSelect) {
+  SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow;
+  if (pCountWin->windowCount <= 1) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
+                                    "Size of Count window must exceed 1.");
+  }
+
+  if (pCountWin->windowSliding <= 0) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
+                                    "Size of Count window must exceed 0.");
+  }
+
+  if (pCountWin->windowSliding > pCountWin->windowCount) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
+                                    "sliding value no larger than the count value.");
+  }
+
+  if (pCountWin->windowCount > INT32_MAX) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
+                                    "Size of Count window must less than 2147483647(INT32_MAX).");
+  }
   if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) &&
       !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
     return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_QUERY,
@@ -4527,7 +4547,7 @@ static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) {
   if (TSDB_CODE_SUCCESS == code) {
     code = getQueryTimeRange(pCxt, pSelect->pWhere, &pSelect->timeRange);
   }
-  if (pSelect->pWhere != NULL) {
+  if (pSelect->pWhere != NULL && pCxt->pParseCxt->topicQuery == false) {
     setTableVgroupsFromEqualTbnameCond(pCxt, pSelect);
   }
   return code;
@@ -7829,28 +7849,6 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
       return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
                                     "Ignore expired data of Count window must be 1.");
     }
-
-    SCountWindowNode* pCountWin = (SCountWindowNode*)pSelect->pWindow;
-    if (pCountWin->windowCount <= 1) {
-      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
-                                     "Size of Count window must exceed 1.");
-    }
-
-    if (pCountWin->windowSliding <= 0) {
-      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
-                                     "Size of Count window must exceed 0.");
-    }
-
-    if (pCountWin->windowSliding > pCountWin->windowCount) {
-      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
-                                     "sliding value no larger than the count value.");
-    }
-
-    if (pCountWin->windowCount > INT32_MAX) {
-      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
-                                     "Size of Count window must less than 2147483647(INT32_MAX).");
-    }
-
   }
 
   return TSDB_CODE_SUCCESS;
@@ -8137,7 +8135,9 @@ static int32_t adjustTagsForCreateTable(STranslateContext* pCxt, SCreateStreamSt
     SColumnDefNode* pDef = (SColumnDefNode*)pTagDef;
     if (!dataTypeEqual(&pDef->dataType, &((SExprNode*)pTagExpr)->resType)) {
       SNode*  pFunc = NULL;
-      int32_t code = createCastFunc(pCxt, pTagExpr, pDef->dataType, &pFunc);
+      SDataType defType = pDef->dataType;
+      defType.bytes = calcTypeBytes(defType);
+      int32_t code = createCastFunc(pCxt, pTagExpr, defType, &pFunc);
       if (TSDB_CODE_SUCCESS != code) {
         return code;
       }

source/libs/planner/src/planSpliter.c

@@ -274,7 +274,7 @@ static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) {
     }
   }
 
-  if (WINDOW_TYPE_STATE == pWindow->winType) {
+  if (WINDOW_TYPE_STATE == pWindow->winType || WINDOW_TYPE_COUNT == pWindow->winType) {
     if (!streamQuery) {
       return stbSplHasMultiTbScan(streamQuery, pNode);
     } else {

source/libs/qworker/src/qwMsg.c

@@ -715,6 +715,7 @@ int32_t qWorkerProcessDeleteMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, SD
   uint64_t tId = req.taskId;
   int64_t  rId = 0;
   int32_t  eId = -1;
+  pRes->source = req.source;
 
   SQWMsg qwMsg = {.node = node, .msg = req.msg, .msgLen = req.phyLen, .connInfo = pMsg->info};
   QW_SCH_TASK_DLOG("processDelete start, node:%p, handle:%p, sql:%s", node, pMsg->info.handle, req.sql);

source/libs/scheduler/inc/schInt.h

@@ -304,6 +304,7 @@ typedef struct SSchJob {
   SSchResInfo          userRes;
   char                *sql;
   SQueryProfileSummary summary;
+  int8_t               source;
 } SSchJob;
 
 typedef struct SSchTaskCtx {

source/libs/scheduler/src/schJob.c

@@ -746,6 +746,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) {
   pJob->chkKillParam = pReq->chkKillParam;
   pJob->userRes.execFp = pReq->execFp;
   pJob->userRes.cbParam = pReq->cbParam;
+  pJob->source = pReq->source;
 
   if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) {
     qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId);

source/libs/scheduler/src/schRemote.c

@@ -940,6 +940,10 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery
   SCH_ERR_JRET(schGenerateCallBackInfo(pJob, pTask, msg, msgSize, msgType, trans, isHb, &pMsgSendInfo));
   SCH_ERR_JRET(schUpdateSendTargetInfo(pMsgSendInfo, addr, pTask));
 
+  if (isHb && persistHandle && trans->pHandle == 0) {
+    trans->pHandle = rpcAllocHandle();
+  } 
+
   if (pJob && pTask) {
     SCH_TASK_DLOG("start to send %s msg to node[%d,%s,%d], pTrans:%p, pHandle:%p", TMSG_INFO(msgType), addr->nodeId,
            epSet->eps[epSet->inUse].fqdn, epSet->eps[epSet->inUse].port, trans->pTrans, trans->pHandle);
@@ -1086,6 +1090,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
       req.sqlLen = strlen(pJob->sql);
       req.sql = (char *)pJob->sql;
       req.msg = pTask->msg;
+      req.source = pJob->source;
       msgSize = tSerializeSVDeleteReq(NULL, 0, &req);
       msg = taosMemoryCalloc(1, msgSize);
       if (NULL == msg) {

source/libs/stream/inc/streamInt.h

@@ -99,6 +99,7 @@ void    streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration);
 int32_t streamDispatchStreamBlock(SStreamTask* pTask);
 void    destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups);
 int32_t getNumOfDispatchBranch(SStreamTask* pTask);
+void    clearBufferedDispatchMsg(SStreamTask* pTask);
 
 int32_t           streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock);
 SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg);

source/libs/stream/src/streamBackendRocksdb.c

@@ -906,6 +906,7 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) {
   return 0;
 }
 
+#ifdef BUILD_NO_CALL
 static int32_t chkpIdComp(const void* a, const void* b) {
   int64_t x = *(int64_t*)a;
   int64_t y = *(int64_t*)b;
@@ -964,6 +965,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) {
   taosMemoryFree(chkpPath);
   return 0;
 }
+#endif
 
 #ifdef BUILD_NO_CALL
 int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t*** ppHandle, SArray* refs) {

source/libs/stream/src/streamDispatch.c

@@ -315,6 +315,16 @@ int32_t getNumOfDispatchBranch(SStreamTask* pTask) {
              : taosArrayGetSize(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos);
 }
 
+void clearBufferedDispatchMsg(SStreamTask* pTask) {
+  SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo;
+  if (pMsgInfo->pData != NULL) {
+    destroyDispatchMsg(pMsgInfo->pData, getNumOfDispatchBranch(pTask));
+  }
+
+  pMsgInfo->pData = NULL;
+  pMsgInfo->dispatchMsgType = 0;
+}
+
 static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pData) {
   int32_t code = 0;
   int32_t numOfBlocks = taosArrayGetSize(pData->blocks);
@@ -678,8 +688,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) {
     // todo deal with only partially success dispatch case
     atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0);
     if (terrno == TSDB_CODE_APP_IS_STOPPING) {  // in case of this error, do not retry anymore
-      destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask));
+      clearBufferedDispatchMsg(pTask);
-      pTask->msgInfo.pData = NULL;
       return code;
     }
 
@@ -740,6 +749,8 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) {
 
 int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) {
   int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+  ASSERT(dataStrLen > 0);
+
   void*   buf = taosMemoryCalloc(1, dataStrLen);
   if (buf == NULL) return -1;
 
@@ -936,15 +947,12 @@ void streamClearChkptReadyMsg(SStreamTask* pTask) {
 // this message has been sent successfully, let's try next one.
 static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId) {
   stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData);
-  destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask));
-
   bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER);
   if (delayDispatch) {
     pTask->chkInfo.dispatchCheckpointTrigger = true;
   }
 
-  pTask->msgInfo.pData = NULL;
+  clearBufferedDispatchMsg(pTask);
-  pTask->msgInfo.dispatchMsgType = 0;
 
   int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs;
 
@@ -1084,7 +1092,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
     } else {  // this message has been sent successfully, let's try next one.
       pTask->msgInfo.retryCount = 0;
 
-      // transtate msg has been sent to downstream successfully. let's transfer the fill-history task state
+      // trans-state msg has been sent to downstream successfully. let's transfer the fill-history task state
       if (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__TRANS_STATE) {
         stDebug("s-task:%s dispatch transtate msgId:%d to downstream successfully, start to prepare transfer state", id, msgId);
         ASSERT(pTask->info.fillHistory == 1);
@@ -1093,6 +1101,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
         if (code != TSDB_CODE_SUCCESS) {  // todo: do nothing if error happens
         }
 
+        clearBufferedDispatchMsg(pTask);
+
         // now ready for next data output
         atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL);
       } else {

source/libs/stream/src/streamSessionState.c

@@ -156,6 +156,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey,
       (*pVal) = pPos;
       SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
       pPos->beUsed = true;
+      pPos->beFlushed = false;
       *pKey = *pDestWinKey;
       goto _end;
     }
@@ -167,6 +168,7 @@ int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey,
       (*pVal) = pPos;
       SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
       pPos->beUsed = true;
+      pPos->beFlushed = false;
       *pKey = *pDestWinKey;
       goto _end;
     }
@@ -380,6 +382,14 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co
     (*pWins) = pWinStates;
   }
 
+  if (size > 0 && index == -1) {
+    SRowBuffPos* pPos = taosArrayGetP(pWinStates, 0);
+    SSessionKey* pWin = (SSessionKey*)pPos->pKey;
+    if (pWinKey->win.skey == pWin->win.skey) {
+      index = 0;
+    }
+  }
+
   if (index >= 0) {
     pCur = createSessionStateCursor(pFileState);
     pCur->buffIndex = index;
@@ -387,6 +397,7 @@ static SStreamStateCur* seekKeyCurrentPrev_buff(SStreamFileState* pFileState, co
       *pIndex = index;
     }
   }
+
   return pCur;
 }
 
@@ -666,6 +677,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch
       (*pVal) = pPos;
       SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
       pPos->beUsed = true;
+      pPos->beFlushed = false;
       *key = *pDestWinKey;
       goto _end;
     }
@@ -679,6 +691,7 @@ int32_t getStateWinResultBuff(SStreamFileState* pFileState, SSessionKey* key, ch
       (*pVal) = pPos;
       SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
       pPos->beUsed = true;
+      pPos->beFlushed = false;
       *key = *pDestWinKey;
       goto _end;
     }
@@ -771,6 +784,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
       (*pVal) = pPos;
       SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
       pPos->beUsed = true;
+      pPos->beFlushed = false;
       *pWinKey = *pDestWinKey;
       goto _end;
     }
@@ -799,6 +813,7 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C
     (*pVal) = pPos;
     SSessionKey* pDestWinKey = (SSessionKey*)pPos->pKey;
     pPos->beUsed = true;
+    pPos->beFlushed = false;
     *pWinKey = *pDestWinKey;
     goto _end;
   }

source/libs/stream/src/streamTask.c

@@ -39,7 +39,7 @@ static int32_t doUpdateTaskEpset(SStreamTask* pTask, int32_t nodeId, SEpSet* pEp
     stDebug("s-task:0x%x (vgId:%d) self node epset is updated %s", pTask->id.taskId, nodeId, buf);
   }
 
-  // check for the dispath info and the upstream task info
+  // check for the dispatch info and the upstream task info
   int32_t level = pTask->info.taskLevel;
   if (level == TASK_LEVEL__SOURCE) {
     streamTaskUpdateDownstreamInfo(pTask, nodeId, pEpSet);
@@ -412,9 +412,7 @@ void tFreeStreamTask(SStreamTask* pTask) {
   pTask->pReadyMsgList = taosArrayDestroy(pTask->pReadyMsgList);
 
   if (pTask->msgInfo.pData != NULL) {
-    destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask));
+    clearBufferedDispatchMsg(pTask);
-    pTask->msgInfo.pData = NULL;
-    pTask->msgInfo.dispatchMsgType = 0;
   }
 
   if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) {
@@ -627,6 +625,7 @@ void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDo
 void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet) {
   char buf[512] = {0};
   EPSET_TO_STR(pEpSet, buf);
+  int32_t id = pTask->id.taskId;
 
   int8_t type = pTask->outputInfo.type;
   if (type == TASK_OUTPUT__SHUFFLE_DISPATCH) {
@@ -638,8 +637,8 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE
 
       if (pVgInfo->vgId == nodeId) {
         epsetAssign(&pVgInfo->epSet, pEpSet);
-        stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", pTask->id.taskId,
+        stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pVgInfo->taskId, nodeId,
-                pVgInfo->taskId, nodeId, buf);
+                buf);
         break;
       }
     }
@@ -647,8 +646,8 @@ void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SE
     STaskDispatcherFixed* pDispatcher = &pTask->outputInfo.fixedDispatcher;
     if (pDispatcher->nodeId == nodeId) {
       epsetAssign(&pDispatcher->epSet, pEpSet);
-      stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpSet:%s", pTask->id.taskId,
+      stDebug("s-task:0x%x update the dispatch info, task:0x%x(nodeId:%d) newEpset:%s", id, pDispatcher->taskId, nodeId,
-              pDispatcher->taskId, nodeId, buf);
+              buf);
     }
   }
 }
@@ -767,21 +766,13 @@ int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask) {
   return status;
 }
 
-int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool metaLock) {
+int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt) {
-  if (pTask == NULL) {
-    return TSDB_CODE_SUCCESS;
-  }
-
   SStreamMeta* pMeta = pTask->pMeta;
   STaskId      sTaskId = {.streamId = pTask->streamTaskId.streamId, .taskId = pTask->streamTaskId.taskId};
   if (pTask->info.fillHistory == 0) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if (metaLock) {
-    streamMetaWLock(pMeta);
-  }
-
   SStreamTask** ppStreamTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &sTaskId, sizeof(sTaskId));
   if (ppStreamTask != NULL) {
     stDebug("s-task:%s clear the related stream task:0x%x attr to fill-history task", pTask->id.idStr,
@@ -801,10 +792,6 @@ int32_t streamTaskClearHTaskAttr(SStreamTask* pTask, int32_t resetRelHalt, bool
     taosThreadMutexUnlock(&(*ppStreamTask)->lock);
   }
 
-  if (metaLock) {
-    streamMetaWUnLock(pMeta);
-  }
-
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/stream/test/backendTest.cpp

@@ -410,9 +410,19 @@ TEST_F(BackendEnv, checkOpen) {
   SArray * result = taosArrayInit(4, sizeof(void *));
   bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump);
 
+  taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 4);
+
+  taosArrayClear(result);
+  bkdMgtGetDelta(mgt, p->pTdbState->idstr, 4, result, (char *)dump);
   bkdMgtDestroy(mgt);
   streamStateClose((SStreamState *)p, true);
+  // {
+  //   taosRemoveDir("/tmp/backend");
+  //   const char *  path = "/tmp/backend";
+  //   SStreamState *p = stateCreate(path);
+  // }
   taosRemoveDir(path);
+  // streamStateClose((SStreamState *)p, true);
 }
 
 TEST_F(BackendEnv, backendChkp) { const char *path = "/tmp"; }
@@ -430,6 +440,20 @@ TEST_F(BackendEnv, backendUtil) {
     ASSERT_EQ(nextPow2((uint32_t)(kvDict[i].k)), kvDict[i].v);
   }
 }
+TEST_F(BackendEnv, oldBackendInit) {
+  const char *path = "/tmp/backend1";
+  taosMulMkDir(path);
+  {
+    SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(path, 10, 10);
+    streamBackendCleanup((void *)p);
+  }
+  {
+    SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(path, 10, 10);
+    streamBackendCleanup((void *)p);
+  }
+
+  taosRemoveDir(path);
+}
 
 int main(int argc, char **argv) {
   testing::InitGoogleTest(&argc, argv);

source/libs/transport/inc/transComm.h

@@ -256,7 +256,7 @@ void        transAsyncPoolDestroy(SAsyncPool* pool);
 int         transAsyncSend(SAsyncPool* pool, queue* mq);
 bool        transAsyncPoolIsEmpty(SAsyncPool* pool);
 
-#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc) \
+#define TRANS_DESTROY_ASYNC_POOL_MSG(pool, msgType, freeFunc, param) \
   do {                                                               \
     for (int i = 0; i < pool->nAsync; i++) {                         \
       uv_async_t* async = &(pool->asyncs[i]);                        \
@@ -267,7 +267,7 @@ bool        transAsyncPoolIsEmpty(SAsyncPool* pool);
         QUEUE_REMOVE(h);                                             \
         msgType* msg = QUEUE_DATA(h, msgType, q);                    \
         if (msg != NULL) {                                           \
-          freeFunc(msg);                                      \
+          freeFunc(msg, param);                                      \
         }                                                            \
       }                                                              \
     }                                                                \

source/libs/transport/src/thttp.c

@@ -191,6 +191,15 @@ static void httpDestroyMsg(SHttpMsg* msg) {
   taosMemoryFree(msg->cont);
   taosMemoryFree(msg);
 }
+static void httpDestroyMsgWrapper(void* cont, void* param) {
+  httpDestroyMsg((SHttpMsg*)cont);
+  // if (msg == NULL) return;
+
+  // taosMemoryFree(msg->server);
+  // taosMemoryFree(msg->uri);
+  // taosMemoryFree(msg->cont);
+  // taosMemoryFree(msg);
+}
 
 static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) {
   SHttpMsg *msg = NULL, *quitMsg = NULL;
@@ -554,7 +563,7 @@ void transHttpEnvDestroy() {
   httpSendQuit();
   taosThreadJoin(load->thread, NULL);
 
-  TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg);
+  TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsgWrapper, NULL);
   transAsyncPoolDestroy(load->asyncPool);
   uv_loop_close(load->loop);
   taosMemoryFree(load->loop);

source/libs/transport/src/transCli.c

@@ -219,6 +219,8 @@ static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq,
 /// NULL,cliHandleUpdate};
 
 static FORCE_INLINE void destroyCmsg(void* cmsg);
+
+static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param);
 static FORCE_INLINE void destroyCmsgAndAhandle(void* cmsg);
 static FORCE_INLINE int  cliRBChoseIdx(STrans* pTransInst);
 static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx);
@@ -582,8 +584,8 @@ void* destroyConnPool(SCliThrd* pThrd) {
 
 static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) {
   void*      pool = pThrd->pool;
-  SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1);
   STrans*    pTranInst = pThrd->pTransInst;
+  SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key) + 1);
   if (plist == NULL) {
     SConnList list = {0};
     taosHashPut((SHashObj*)pool, key, strlen(key) + 1, (void*)&list, sizeof(list));
@@ -865,17 +867,18 @@ static void cliDestroyConn(SCliConn* conn, bool clear) {
   QUEUE_INIT(&conn->q);
 
   conn->broken = true;
+  if (conn->list == NULL) {
+    conn->list = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr));
+  }
 
-  if (conn->list != NULL) {
+  if (conn->list) {
-    SConnList* connList = conn->list;
+    SConnList* list = conn->list;
-    connList->list->numOfConn--;
+    list->list->numOfConn--;
-    connList->size--;
+    if (conn->status == ConnInPool) {
-  } else {
+      list->size--;
-    if (pThrd->pool) {
-      SConnList* connList = taosHashGet((SHashObj*)pThrd->pool, conn->dstAddr, strlen(conn->dstAddr) + 1);
-      if (connList != NULL) connList->list->numOfConn--;
     }
   }
+
   conn->list = NULL;
   pThrd->newConnCount--;
 
@@ -1963,7 +1966,17 @@ static FORCE_INLINE void destroyCmsg(void* arg) {
   transFreeMsg(pMsg->msg.pCont);
   taosMemoryFree(pMsg);
 }
-
+static FORCE_INLINE void destroyCmsgWrapper(void* arg, void* param) {
+  SCliMsg* pMsg = arg;
+  if (pMsg == NULL) {
+    return;
+  }
+  if (param != NULL) {
+    SCliThrd* pThrd = param;
+    if (pThrd->destroyAhandleFp) (*pThrd->destroyAhandleFp)(pMsg->msg.info.ahandle);
+  }
+  destroyCmsg(pMsg);
+}
 static FORCE_INLINE void destroyCmsgAndAhandle(void* param) {
   if (param == NULL) return;
 
@@ -2057,7 +2070,7 @@ static void destroyThrdObj(SCliThrd* pThrd) {
   taosThreadJoin(pThrd->thread, NULL);
   CLI_RELEASE_UV(pThrd->loop);
   taosThreadMutexDestroy(&pThrd->msgMtx);
-  TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsg);
+  TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsgWrapper, (void*)pThrd);
   transAsyncPoolDestroy(pThrd->asyncPool);
 
   transDQDestroy(pThrd->delayQueue, destroyCmsgAndAhandle);

source/libs/transport/src/transSvr.c

@@ -527,6 +527,10 @@ void uvOnSendCb(uv_write_t* req, int status) {
       if (!transQueueEmpty(&conn->srvMsgs)) {
         msg = (SSvrMsg*)transQueueGet(&conn->srvMsgs, 0);
         if (msg->type == Register && conn->status == ConnAcquire) {
+          if (conn->regArg.init) {
+            transFreeMsg(conn->regArg.msg.pCont);
+            conn->regArg.init = 0;
+          }
           conn->regArg.notifyCount = 0;
           conn->regArg.init = 1;
           conn->regArg.msg = msg->msg;
@@ -671,6 +675,7 @@ static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) {
   transFreeMsg(smsg->msg.pCont);
   taosMemoryFree(smsg);
 }
+static FORCE_INLINE void destroySmsgWrapper(void* smsg, void* param) { destroySmsg((SSvrMsg*)smsg); }
 static void              destroyAllConn(SWorkThrd* pThrd) {
   tTrace("thread %p destroy all conn ", pThrd);
   while (!QUEUE_IS_EMPTY(&pThrd->conn)) {
@@ -1349,6 +1354,11 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) {
       return;
     }
     transQueuePop(&conn->srvMsgs);
+
+    if (conn->regArg.init) {
+      transFreeMsg(conn->regArg.msg.pCont);
+      conn->regArg.init = 0;
+    }
     conn->regArg.notifyCount = 0;
     conn->regArg.init = 1;
     conn->regArg.msg = msg->msg;
@@ -1394,7 +1404,7 @@ void destroyWorkThrd(SWorkThrd* pThrd) {
   }
   taosThreadJoin(pThrd->thread, NULL);
   SRV_RELEASE_UV(pThrd->loop);
-  TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsg);
+  TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsgWrapper, NULL);
   transAsyncPoolDestroy(pThrd->asyncPool);
 
   uvWhiteListDestroy(pThrd->pWhiteList);

tests/parallel_test/cases.task

@@ -115,7 +115,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/ins_topics_test.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxTopic.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py
-,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R
+#,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqParamsTest.py -R
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqClientConsLog.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqMaxGroupIds.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsumeDiscontinuousData.py
@@ -238,7 +238,8 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True
-,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform.py -N 2 -n 1
+,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db-removewal.py -N 2 -n 1
+,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb-removewal.py -N 6 -n 3
 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 2 -n 1
 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-stb.py -N 6 -n 3
 #,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeTransform-db.py -N 6 -n 3

tests/script/sh/deploy.sh

@@ -117,7 +117,7 @@ echo "supportVnodes          1024"                >> $TAOS_CFG
 echo "statusInterval         1"                  >> $TAOS_CFG
 echo "dataDir                $DATA_DIR"          >> $TAOS_CFG
 echo "logDir                 $LOG_DIR"           >> $TAOS_CFG
-echo "debugFlag              0"                  >> $TAOS_CFG
+echo "debugFlag              135"                  >> $TAOS_CFG
 echo "tmrDebugFlag           131"                >> $TAOS_CFG
 echo "uDebugFlag             143"                >> $TAOS_CFG
 echo "rpcDebugFlag           143"                >> $TAOS_CFG

tests/script/tsim/parser/last_both.sim

@@ -6,7 +6,7 @@ sql connect
 print ======================== dnode1 start
 $db = testdb
 sql drop database if exists $db
-sql create database $db cachemodel 'both' minrows 10 stt_trigger 1 
+sql create database $db cachemodel 'none' minrows 10 stt_trigger 1
 sql use $db
 
 sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp) tags (id int)
@@ -69,13 +69,13 @@ sql drop table tbf;
 sql alter table st2 add column c1 int;
 sql alter table st2 drop column c1;
 
-run tsim/parser/last_cache_query.sim
+run tsim/parser/last_both_query.sim
 
 sql flush database $db
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode1 -s start
 
-run tsim/parser/last_cache_query.sim
+run tsim/parser/last_both_query.sim
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode1 -s start
@@ -145,6 +145,6 @@ sql alter database $db cachemodel 'both'
 sql alter database $db cachesize 2
 sleep 11000
 
-run tsim/parser/last_cache_query.sim
+run tsim/parser/last_both_query.sim
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/parser/last_both_query.sim

@@ -0,0 +1,496 @@
+
+sql connect
+
+$db = testdb
+sql use $db
+print "test tb1"
+
+sql select last(ts) from tb1
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+
+sql select last(f1) from tb1
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 6 then
+  print $data00
+  return -1
+endi
+
+sql select last(*) from tb1
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 5.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != 3 then
+  print expect 3, actual: $data03
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+
+sql select last(tb1.*,ts,f4) from tb1
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 5.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != 3 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+if $data05 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data06 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+
+print "test tb2"
+sql select last(ts) from tb2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-11 10:11:15.000@ then
+  print $data00
+  return -1
+endi
+
+sql select last(f1) from tb2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != -6 then
+  print $data00
+  return -1
+endi
+
+sql select last(*) from tb2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-11 10:11:15.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != -6 then
+  return -1
+endi
+if $data02 != -7.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != -8 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:56.999@ then
+  if $data04 != @70-01-01 07:59:57.-01@ then
+    return -1
+  endi
+endi
+
+sql select last(tb2.*,ts,f4) from tb2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-11 10:11:15.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != -6 then
+  return -1
+endi
+if $data02 != -7.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != -8 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:56.999@ then
+  if $data04 != @70-01-01 07:59:57.-01@ then
+    return -1
+  endi
+endi
+if $data05 != @21-05-11 10:11:15.000@ then
+  print $data00
+  return -1
+endi
+if $data06 != @70-01-01 07:59:56.999@ then
+  if $data04 != @70-01-01 07:59:57.-01@ then
+    return -1
+  endi
+endi
+
+print "test tbd"
+sql select last(*) from tbd
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-11 10:12:29.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != NULL then
+  return -1
+endi
+if $data02 != NULL then
+  print $data02
+  return -1
+endi
+if $data03 != NULL then
+  return -1
+endi
+if $data04 != NULL then
+  return -1
+endi
+
+print "test tbe"
+sql select last(*) from tbe
+if $rows != 0 then
+  return -1
+endi
+
+print "test stable"
+sql select last(ts) from st2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+
+sql select last(f1) from st2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != 6 then
+  print $data00
+  return -1
+endi
+
+sql select last(*) from st2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 37.000000000 then
+  print expect 37.000000000 actual: $data02
+  return -1
+endi
+if $data03 != 27 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+
+
+sql select last(st2.*,ts,f4) from st2
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 37.000000000 then
+  print expect 37.000000000, acutal: $data02
+  return -1
+endi
+if $data03 != 27 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+if $data05 != @21-05-12 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data06 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+
+sql select last(*), id from st2 group by id order by id
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49
+
+if $rows != 5 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 5.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != 21 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+if $data05 != 1 then
+  return -1
+endi
+if $data10 != @21-05-11 10:12:23.000@ then
+  return -1
+endi
+if $data11 != 22 then
+  return -1
+endi
+if $data12 != 23.000000000 then
+  print $data02
+  return -1
+endi
+if $data13 != -8 then
+  return -1
+endi
+if $data14 != @70-01-01 07:59:58.-04@ then
+  return -1
+endi
+if $data15 != 2 then
+  return -1
+endi
+if $data20 != @21-05-10 10:12:24.000@ then
+  return -1
+endi
+if $data21 != 24 then
+  return -1
+endi
+if $data22 != 11.000000000 then
+  print expect 11.000000000 actual: $data22
+  return -1
+endi
+if $data23 != 25 then
+  return -1
+endi
+if $data24 != @70-01-01 07:59:57.-04@ then =
+  return -1
+endi
+if $data25 != 3 then
+  return -1
+endi
+if $data30 != @21-05-11 10:12:25.000@ then
+  return -1
+endi
+if $data31 != 26 then
+  return -1
+endi
+if $data32 != 17.000000000 then
+  print $data02
+  return -1
+endi
+if $data33 != 27 then
+  return -1
+endi
+if $data34 != @70-01-01 07:59:56.-04@ then
+  return -1
+endi
+if $data35 != 4 then
+  return -1
+endi
+if $data40 != @21-05-11 10:12:29.000@ then
+  return -1
+endi
+if $data41 != 36 then
+  return -1
+endi
+if $data42 != 37.000000000 then
+  print $data02
+  return -1
+endi
+if $data43 != 35 then
+  return -1
+endi
+if $data44 != @70-01-01 07:59:56.-05@ then
+  return -1
+endi
+if $data45 != 5 then
+  return -1
+endi
+
+sql select last_row(*), id from st2 group by id order by id
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49
+
+if $rows != 5 then
+  return -1
+endi
+if $data00 != @21-05-12 10:10:12.000@ then
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != NULL then
+  print $data02
+  return -1
+endi
+if $data03 != NULL then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+if $data05 != 1 then
+  return -1
+endi
+if $data10 != @21-05-11 10:12:23.000@ then
+  return -1
+endi
+if $data11 != 22 then
+  return -1
+endi
+if $data12 != 23.000000000 then
+  print $data02
+  return -1
+endi
+if $data13 != NULL then
+  return -1
+endi
+if $data14 != @70-01-01 07:59:58.-04@ then
+  return -1
+endi
+if $data15 != 2 then
+  return -1
+endi
+if $data20 != @21-05-10 10:12:24.000@ then
+  return -1
+endi
+if $data21 != 24 then
+  return -1
+endi
+if $data22 != NULL then
+  print expect NULL actual: $data22
+  return -1
+endi
+if $data23 != 25 then
+  return -1
+endi
+if $data24 != @70-01-01 07:59:57.-04@ then =
+  return -1
+endi
+if $data25 != 3 then
+  return -1
+endi
+if $data30 != @21-05-11 10:12:25.000@ then
+  return -1
+endi
+if $data31 != 26 then
+  return -1
+endi
+if $data32 != NULL then
+  print $data02
+  return -1
+endi
+if $data33 != 27 then
+  return -1
+endi
+if $data34 != @70-01-01 07:59:56.-04@ then
+  return -1
+endi
+if $data35 != 4 then
+  return -1
+endi
+if $data40 != @21-05-11 10:12:29.000@ then
+  return -1
+endi
+#if $data41 != NULL then
+#  return -1
+#endi
+#if $data42 != NULL then
+#  print $data02
+#  return -1
+#endi
+if $data43 != NULL then
+  return -1
+endi
+#if $data44 != NULL then
+#  return -1
+#endi
+if $data45 != 5 then
+  return -1
+endi
+
+print "test tbn"
+sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp)
+sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3',  -1000)
+sql insert into tbn values ("2021-05-10 10:10:11", 4, 5.0, NULL, -2000)
+sql insert into tbn values ("2021-05-12 10:10:12", 6,NULL, NULL, -3000)
+sql insert into tbn values ("2021-05-13 10:10:12", NULL,NULL, NULL,NULL)
+
+sql select last(*) from tbn;
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-05-13 10:10:12.000@ then
+  print $data00
+  return -1
+endi
+if $data01 != 6 then
+  return -1
+endi
+if $data02 != 5.000000000 then
+  print $data02
+  return -1
+endi
+if $data03 != 3 then
+  return -1
+endi
+if $data04 != @70-01-01 07:59:57.000@ then
+  return -1
+endi
+
+sql alter table tbn add column c1 int;
+sql alter table tbn drop column c1;

tests/script/tsim/parser/last_cache_query.sim

@@ -357,112 +357,6 @@ if $data45 != 5 then
   return -1
 endi
 
-sql select last_row(*), id from st2 group by id order by id
-print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
-print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
-print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
-print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
-print ===> $data40 $data41 $data42 $data43 $data44 $data45 $data46 $data47 $data48 $data49
-
-if $rows != 5 then
-  return -1
-endi
-if $data00 != @21-05-12 10:10:12.000@ then
-  return -1
-endi
-if $data01 != 6 then
-  return -1
-endi
-if $data02 != NULL then
-  print $data02
-  return -1
-endi
-if $data03 != NULL then
-  return -1
-endi
-if $data04 != @70-01-01 07:59:57.000@ then
-  return -1
-endi
-if $data05 != 1 then
-  return -1
-endi
-if $data10 != @21-05-11 10:12:23.000@ then
-  return -1
-endi
-if $data11 != 22 then
-  return -1
-endi
-if $data12 != 23.000000000 then
-  print $data02
-  return -1
-endi
-if $data13 != NULL then
-  return -1
-endi
-if $data14 != @70-01-01 07:59:58.-04@ then
-  return -1
-endi
-if $data15 != 2 then
-  return -1
-endi
-if $data20 != @21-05-10 10:12:24.000@ then
-  return -1
-endi
-if $data21 != 24 then
-  return -1
-endi
-if $data22 != NULL then
-  print expect NULL actual: $data22
-  return -1
-endi
-if $data23 != 25 then
-  return -1
-endi
-if $data24 != @70-01-01 07:59:57.-04@ then =
-  return -1
-endi
-if $data25 != 3 then
-  return -1
-endi
-if $data30 != @21-05-11 10:12:25.000@ then
-  return -1
-endi
-if $data31 != 26 then
-  return -1
-endi
-if $data32 != NULL then
-  print $data02
-  return -1
-endi
-if $data33 != 27 then
-  return -1
-endi
-if $data34 != @70-01-01 07:59:56.-04@ then
-  return -1
-endi
-if $data35 != 4 then
-  return -1
-endi
-if $data40 != @21-05-11 10:12:29.000@ then
-  return -1
-endi
-if $data41 != 36 then
-  return -1
-endi
-if $data42 != 37.000000000 then
-  print $data02
-  return -1
-endi
-if $data43 != NULL then
-  return -1
-endi
-if $data44 != @70-01-01 07:59:56.-05@ then
-  return -1
-endi
-if $data45 != 5 then
-  return -1
-endi
-
 print "test tbn"
 sql create table if not exists tbn (ts timestamp, f1 int, f2 double, f3 binary(10), f4 timestamp)
 sql insert into tbn values ("2021-05-09 10:10:10", 1, 2.0, '3',  -1000)

tests/script/tsim/query/query_count1.sim

@@ -79,5 +79,27 @@ if $data22 != 4 then
   goto loop3
 endi
 
+
+print step2
+print =============== create database
+sql create database test1 vgroups 1;
+sql use test1;
+
+sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+
+#2~INT32_MAX
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(-1);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(0);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(1);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(2147483648);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(10, 0);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(10, -1);
+sql_error select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(10, 11);
+
+sql select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(2);
+sql select  _wstart as s, count(*) c1,  sum(b), max(c) from t1 count_window(2147483647);
+
 print query_count0 end
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stream/state0.sim

@@ -21,6 +21,8 @@ print create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0
 
 sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0   into streamt1 as select  _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a)  c4, min(c) c5, max(id) c from t1 state_window(a);
 
+sleep 1000
+
 sql insert into t1 values(1648791213000,1,2,3,1.0,1);
 sql insert into t1 values(1648791213000,1,2,3,1.0,2);
 $loop_count = 0
@@ -457,6 +459,8 @@ print create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0
 
 sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0   into streamt1 as select  _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(a)  c4, min(c) c5, max(id) c from t1 state_window(a);
 
+sleep 1000
+
 sql insert into t1 values(1648791212000,2,2,3,1.0,1);
 sql insert into t1 values(1648791213000,1,2,3,1.0,1);
 sql insert into t1 values(1648791213000,1,2,4,1.0,2);
@@ -504,6 +508,9 @@ sql create table t1(ts timestamp, a int, b int , c int, d double, id int);
 print create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0   into streamt3 as select  _wstart, count(*) c1, sum(b) c3 from t1 state_window(a);
 
 sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0   into streamt3 as select  _wstart, count(*) c1, sum(b) c3 from t1 state_window(a);
+
+sleep 1000
+
 sql insert into t1 values(1648791212000,1,2,3,1.0,1);
 sql insert into t1 values(1648791213000,2,2,3,1.0,1);
 sql insert into t1 values(1648791214000,3,2,4,1.0,2);
@@ -557,6 +564,8 @@ print create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0
 
 sql create stream if not exists streams4 trigger window_close IGNORE EXPIRED 0 IGNORE UPDATE 0  into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1);
 
+sleep 1000
+
 sql insert into t1 (ts, c1) values (1668073288209, 11);
 sql insert into t1 (ts, c1) values (1668073288210, 11);
 sql insert into t1 (ts, c1) values (1668073288211, 11);
@@ -745,6 +754,9 @@ sql create table b (c timestamp, d int, e int , f int, g double);
 print create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0  into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a);
 
 sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0  into streamt as select _wstart c1, count(*) c2, max(a) c3 from tb state_window(a);
+
+sleep 1000
+
 sql insert into b values(1648791213000,NULL,NULL,NULL,NULL);
 sql select * from streamt order by c1, c2, c3;
 

tests/system-test/0-others/com_alltypedata.json

@@ -22,7 +22,7 @@
                 "vgroups": 2,
                 "replica": 1,
                 "precision": "ms",
-                "stt_trigger": 8,
+                "stt_trigger": 1,
                 "minRows": 100,
                 "maxRows": 4096
             },

tests/system-test/0-others/compatibility.py

@@ -1,11 +1,13 @@
 from urllib.parse import uses_relative
 import taos
+import taosws
 import sys
 import os
 import time
 import platform
 import inspect
 from taos.tmq import Consumer
+from taos.tmq import *
 
 from pathlib import Path
 from util.log import *
@@ -17,7 +19,7 @@ from util.dnodes import TDDnode
 from util.cluster import *
 import subprocess
 
-BASEVERSION = "3.0.2.3"
+BASEVERSION = "3.2.0.0"
 class TDTestCase:
     def caseDescription(self):
         f'''
@@ -30,7 +32,7 @@ class TDTestCase:
         self.replicaVar = int(replicaVar)
         tdLog.debug(f"start to excute {__file__}")
         tdSql.init(conn.cursor())
-        self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 4; ;use deldata;
+        self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 1; ;use deldata;
                             create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int);
                             create table deldata.ct1 using deldata.stb1 tags ( 1 );
                             insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a );
@@ -104,8 +106,19 @@ class TDTestCase:
             print(f"{packageName} has been exists")
         os.system(f" cd {packagePath} &&  tar xvf  {packageName} && cd {packageTPath} &&  ./install.sh  -e no  " )
         tdDnodes.stop(1)
-        print(f"start taosd: rm -rf {dataPath}/*  && nohup taosd -c {cPath} & ")
+        print(f"start taosd: rm -rf {dataPath}/*  && nohup /usr/bin/taosd -c {cPath} & ")
-        os.system(f"rm -rf {dataPath}/*  && nohup taosd -c {cPath} & " )
+        os.system(f"rm -rf {dataPath}/*  && nohup  /usr/bin/taosd -c {cPath} & " )
+        os.system(f"killall taosadapter" )
+        os.system(f"cp /etc/taos/taosadapter.toml {cPath}/taosadapter.toml  " )
+        taosadapter_cfg = cPath + "/taosadapter.toml"
+        taosadapter_log_path = cPath + "/../log/"
+        print(f"taosadapter_cfg:{taosadapter_cfg},taosadapter_log_path:{taosadapter_log_path} ")
+        self.alter_string_in_file(taosadapter_cfg,"#path = \"/var/log/taos\"",f"path = \"{taosadapter_log_path}\"")
+        self.alter_string_in_file(taosadapter_cfg,"taosConfigDir = \"\"",f"taosConfigDir = \"{cPath}\"")
+        print("/usr/bin/taosadapter --version")
+        os.system(f"  /usr/bin/taosadapter --version" )
+        print(f" LD_LIBRARY_PATH=/usr/lib -c  {taosadapter_cfg} 2>&1 & ")
+        os.system(f" LD_LIBRARY_PATH=/usr/lib  /usr/bin/taosadapter -c  {taosadapter_cfg} 2>&1 & " )
         sleep(5)
 
 
@@ -117,6 +130,23 @@ class TDTestCase:
         sorted_list = sorted(unordered_list)
         return sorted_list == ordered_list
 
+    def alter_string_in_file(self,file,old_str,new_str):
+        """
+        replace str in file
+        :param file
+        :param old_str
+        :param new_str
+        :return:
+        """
+        file_data = ""
+        with open(file, "r", encoding="utf-8") as f:
+            for line in f:
+                if old_str in line:
+                    line = line.replace(old_str,new_str)
+                file_data += line
+        with open(file,"w",encoding="utf-8") as f:
+            f.write(file_data)
+
     def run(self):
         scriptsPath = os.path.dirname(os.path.realpath(__file__))
         distro_id = distro.id()
@@ -131,7 +161,7 @@ class TDTestCase:
         dbname = "test"
         stb = f"{dbname}.meters"
         self.installTaosd(bPath,cPath)
-        os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ")
+        # os.system(f"echo 'debugFlag 143' >> {cPath}/taos.cfg ")
         tableNumbers=100
         recordNumbers1=100
         recordNumbers2=1000
@@ -163,11 +193,46 @@ class TDTestCase:
         # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ")
         # os.system('LD_LIBRARY_PATH=/usr/lib taos -s  "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ')
         # os.system('LD_LIBRARY_PATH=/usr/lib taos -s  "use test;show streams;" ')
-        os.system(f"sed -i 's/\/etc\/taos/{cPath}/' 0-others/tmqBasic.json ")
+        self.alter_string_in_file("0-others/tmqBasic.json", "/etc/taos/", cPath)
         # os.system("LD_LIBRARY_PATH=/usr/lib  taosBenchmark -f 0-others/tmqBasic.json -y ")
         os.system('LD_LIBRARY_PATH=/usr/lib taos -s  "create topic if not exists tmq_test_topic  as select  current,voltage,phase from test.meters where voltage <= 106 and current <= 5;" ')
         os.system('LD_LIBRARY_PATH=/usr/lib taos -s  "use test;show topics;" ')
+        os.system(f"  /usr/bin/taosadapter --version " )        
+        consumer_dict = {
+            "group.id": "g1",
+            "td.connect.user": "root",
+            "td.connect.pass": "taosdata",
+            "auto.offset.reset": "earliest",
+        }
 
+        consumer = taosws.Consumer(conf={"group.id": "local", "td.connect.websocket.scheme": "ws"})
+        try:
+            consumer.subscribe(["tmq_test_topic"])
+        except TmqError:
+            tdLog.exit(f"subscribe error")
+
+        while True:
+            message = consumer.poll(timeout=1.0)
+            if message:
+                print("message")
+                id = message.vgroup()
+                topic = message.topic()
+                database = message.database()
+
+                for block in message:
+                    nrows = block.nrows()
+                    ncols = block.ncols()
+                    for row in block:
+                        print(row)
+                    values = block.fetchall()
+                    print(nrows, ncols)
+
+                consumer.commit(message)
+            else:
+                print("break")
+                break
+
+        consumer.close()
         tdLog.info(" LD_LIBRARY_PATH=/usr/lib  taosBenchmark -f 0-others/compa4096.json -y  ")
         os.system("LD_LIBRARY_PATH=/usr/lib  taosBenchmark -f 0-others/compa4096.json -y")
         os.system("LD_LIBRARY_PATH=/usr/lib  taos -s 'flush database db4096 '")
@@ -184,7 +249,8 @@ class TDTestCase:
                 
         os.system("pkill  taosd")   # make sure all the data are saved in disk.
         self.checkProcessPid("taosd")
-
+        os.system("pkill  taosadapter")   # make sure all the data are saved in disk.
+        self.checkProcessPid("taosadapter")
 
         tdLog.printNoPrefix("==========step2:update new version ")
         self.buildTaosd(bPath)
@@ -193,6 +259,7 @@ class TDTestCase:
         tdsql=tdCom.newTdSql()
         print(tdsql)
         cmd = f" LD_LIBRARY_PATH=/usr/lib  taos -h localhost ;"
+        print(os.system(cmd))
         if os.system(cmd) == 0:
             raise Exception("failed to execute system command. cmd: %s" % cmd)
         

tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py

@@ -123,134 +123,6 @@ class TDTestCase:
         tdSql.query(redistributeSql)
         tdLog.debug("redistributeSql ok")
         
-    def tmqCase1(self):
-        tdLog.printNoPrefix("======== test case 1: ")
-        paraDict = {'dbName':     'dbt',
-                    'dropFlag':   1,
-                    'event':      '',
-                    'vgroups':    1,
-                    'stbName':    'stb',
-                    'colPrefix':  'c',
-                    'tagPrefix':  't',
-                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
-                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
-                    'ctbPrefix':  'ctb',
-                    'ctbStartIdx': 0,
-                    'ctbNum':     10,
-                    'rowsPerTbl': 1000,
-                    'batchNum':   10,
-                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
-                    'pollDelay':  60,
-                    'showMsg':    1,
-                    'showRow':    1,
-                    'snapshot':   0}
-
-        paraDict['vgroups'] = self.vgroups
-        paraDict['ctbNum'] = self.ctbNum
-        paraDict['rowsPerTbl'] = self.rowsPerTbl
-
-        topicNameList = ['topic1']
-        # expectRowsList = []
-        tmqCom.initConsumerTable()
-
-        tdLog.info("create topics from stb with filter")
-        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
-        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
-        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
-        tdLog.info("create topic sql: %s"%sqlString)
-        tdSql.execute(sqlString)
-        # tdSql.query(queryString)
-        # expectRowsList.append(tdSql.getRows())
-
-        # init consume info, and start tmq_sim, then check consume result
-        tdLog.info("insert consume info to consume processor")
-        consumerId   = 0
-        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
-        topicList    = topicNameList[0]
-        ifcheckdata  = 1
-        ifManualCommit = 1
-        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
-        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
-
-        tdLog.info("start consume processor")
-        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
-        tdLog.info("wait the consume result")
-
-        tdLog.info("create ctb1")
-        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
-                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
-        tdLog.info("insert data")
-        pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
-
-        tmqCom.getStartConsumeNotifyFromTmqsim()
-        tmqCom.getStartCommitNotifyFromTmqsim()
-
-        #restart dnode & remove wal
-        self.restartAndRemoveWal()
-
-        # redistribute vgroup
-        self.redistributeVgroups();
-
-        tdLog.info("create ctb2")
-        paraDict['ctbPrefix'] = "ctbn"
-        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
-                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
-        tdLog.info("insert data")
-        pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict)
-        pInsertThread.join()
-        pInsertThread1.join()
-
-        expectRows = 1
-        resultList = tmqCom.selectConsumeResult(expectRows)
-
-        if expectrowcnt / 2 > resultList[0]:
-            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0]))
-            tdLog.exit("%d tmq consume rows error!"%consumerId)
-
-        # tmqCom.checkFileContent(consumerId, queryString)
-
-        time.sleep(10)
-        for i in range(len(topicNameList)):
-            tdSql.query("drop topic %s"%topicNameList[i])
-
-        tdLog.printNoPrefix("======== test case 1 end ...... ")
-
-    def tmqCase2(self):
-        tdLog.printNoPrefix("======== test case 2: ")
-        paraDict = {'dbName':'dbt'}
-
-        ntbName = "ntb"
-
-        topicNameList = ['topic2']
-        tmqCom.initConsumerTable()
-
-        sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName)
-        tdLog.info("create nomal table sql: %s"%sqlString)
-        tdSql.execute(sqlString)
-
-        tdLog.info("create topics from nomal table")
-        queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName)
-        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
-        tdLog.info("create topic sql: %s"%sqlString)
-        tdSql.execute(sqlString)
-        tdSql.query("flush database %s"%(paraDict['dbName']))
-        #restart dnode & remove wal
-        self.restartAndRemoveWal()
-
-        # redistribute vgroup
-        self.redistributeVgroups();
-
-        sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName)
-        tdLog.info("alter table sql: %s"%sqlString)
-        tdSql.error(sqlString)
-        expectRows = 0
-        resultList = tmqCom.selectConsumeResult(expectRows)
-        time.sleep(1)
-        for i in range(len(topicNameList)):
-            tdSql.query("drop topic %s"%topicNameList[i])
-
-        tdLog.printNoPrefix("======== test case 2 end ...... ")
-
     def tmqCase3(self):
         tdLog.printNoPrefix("======== test case 3: ")
         paraDict = {'dbName':     'dbt',
@@ -330,12 +202,90 @@ class TDTestCase:
 
         tdLog.printNoPrefix("======== test case 3 end ...... ")
 
+    def tmqCaseDbname(self):
+        tdLog.printNoPrefix("======== test case  4 subscrib Dbname start: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stbn',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        topicNameList = ['topic4']
+        tmqCom.initConsumerTable()
+
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+                                               ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+                                               startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+
+        tdLog.info("create topics from database ")
+        queryString = "database  %s "%(paraDict['dbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result")
+
+        time.sleep(1)
+        # restart dnode & remove wal
+        self.restartAndRemoveWal()
+
+        # redistribute vgroup
+        self.redistributeVgroups()
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result")
+        expectRows = 2
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        time.sleep(6)
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+ 
+        tdLog.printNoPrefix("======== test case 4 subscrib Dbname end ...... ")
+
     def run(self):
-        self.prepareTestEnv()
-        self.tmqCase1()
-        self.tmqCase2()
         self.prepareTestEnv()
         self.tmqCase3()
+        self.prepareTestEnv()
+        self.tmqCaseDbname()
 
     def stop(self):
         tdSql.close()

tests/system-test/7-tmq/tmqVnodeTransform-stb-removewal.py

@@ -0,0 +1,266 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+import math
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+from util.cluster import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def __init__(self):
+        self.vgroups    = 1
+        self.ctbNum     = 10
+        self.rowsPerTbl = 1000
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), True)
+
+    def getDataPath(self):
+        selfPath = tdCom.getBuildPath()
+
+        return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*';
+
+    def prepareTestEnv(self):
+        tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  60,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        tdCom.drop_all_db()
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar)
+        tdLog.info("create stb")
+        tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"])
+        # tdLog.info("create ctb")
+        # tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+        #                      ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        # tdLog.info("insert data")
+        # tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"],
+        #                                        ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],
+        #                                        startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx'])
+
+        # tdLog.info("restart taosd to ensure that the data falls into the disk")
+        # tdDnodes.stop(1)
+        # tdDnodes.start(1)
+        # tdSql.query("flush database %s"%(paraDict['dbName']))
+        return
+
+    def restartAndRemoveWal(self):
+        tdDnodes = cluster.dnodes
+        tdSql.query("select * from information_schema.ins_vnodes")
+        for result in tdSql.queryResult:
+            if result[2] == 'dbt':
+                tdLog.debug("dnode is %d"%(result[0]))
+                dnodeId = result[0]
+                vnodeId = result[1]
+
+                tdDnodes[dnodeId - 1].stoptaosd()
+                time.sleep(1)
+                dataPath = self.getDataPath()
+                dataPath = dataPath%(dnodeId,vnodeId)
+                os.system('rm -rf ' + dataPath)
+                tdLog.debug("dataPath:%s"%dataPath)
+                tdDnodes[dnodeId - 1].starttaosd()
+                time.sleep(1)
+                break
+        tdLog.debug("restart dnode ok")
+
+    def redistributeVgroups(self):
+        dnodesList = []
+        tdSql.query("show dnodes")
+        for result in tdSql.queryResult:
+            dnodesList.append(result[0])
+        print("dnodeList:",dnodesList)
+        tdSql.query("select * from information_schema.ins_vnodes")
+        vnodeId = 0
+        for result in tdSql.queryResult:
+            if result[2] == 'dbt':
+                tdLog.debug("dnode is %d"%(result[0]))
+                dnodesList.remove(result[0])
+                vnodeId = result[1]
+        print("its all data",dnodesList)
+        # if self.replicaVar == 1:
+        #     redistributeSql = "redistribute vgroup %d dnode %d" %(vnodeId, dnodesList[0])
+        # else:
+        redistributeSql = f"redistribute vgroup {vnodeId} " 
+        for vgdnode in dnodesList:
+            redistributeSql += f"dnode {vgdnode} "
+        print(redistributeSql)
+        
+        tdLog.debug(f"redistributeSql:{redistributeSql}")
+        tdSql.query(redistributeSql)
+        tdLog.debug("redistributeSql ok")
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'dbt',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    1,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  60,
+                    'showMsg':    1,
+                    'showRow':    1,
+                    'snapshot':   0}
+
+        paraDict['vgroups'] = self.vgroups
+        paraDict['ctbNum'] = self.ctbNum
+        paraDict['rowsPerTbl'] = self.rowsPerTbl
+
+        topicNameList = ['topic1']
+        # expectRowsList = []
+        tmqCom.initConsumerTable()
+
+        tdLog.info("create topics from stb with filter")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName'])
+        # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        # tdSql.query(queryString)
+        # expectRowsList.append(tdSql.getRows())
+
+        # init consume info, and start tmq_sim, then check consume result
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
+        topicList    = topicNameList[0]
+        ifcheckdata  = 1
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot'])
+        tdLog.info("wait the consume result")
+
+        tdLog.info("create ctb1")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict)
+
+        tmqCom.getStartConsumeNotifyFromTmqsim()
+        tmqCom.getStartCommitNotifyFromTmqsim()
+
+        #restart dnode & remove wal
+        self.restartAndRemoveWal()
+
+        # redistribute vgroup
+        self.redistributeVgroups();
+
+        tdLog.info("create ctb2")
+        paraDict['ctbPrefix'] = "ctbn"
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'],
+                             ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict)
+        pInsertThread.join()
+        pInsertThread1.join()
+
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+
+        if expectrowcnt / 2 > resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0]))
+            tdLog.exit("%d tmq consume rows error!"%consumerId)
+
+        # tmqCom.checkFileContent(consumerId, queryString)
+
+        time.sleep(10)
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def tmqCase2(self):
+        tdLog.printNoPrefix("======== test case 2: ")
+        paraDict = {'dbName':'dbt'}
+
+        ntbName = "ntb"
+
+        topicNameList = ['topic2']
+        tmqCom.initConsumerTable()
+
+        sqlString = "create table %s.%s(ts timestamp, i nchar(8))" %(paraDict['dbName'], ntbName)
+        tdLog.info("create nomal table sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+
+        tdLog.info("create topics from nomal table")
+        queryString = "select * from %s.%s"%(paraDict['dbName'], ntbName)
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString)
+        tdSql.query("flush database %s"%(paraDict['dbName']))
+        #restart dnode & remove wal
+        self.restartAndRemoveWal()
+
+        # redistribute vgroup
+        self.redistributeVgroups();
+
+        sqlString = "alter table %s.%s modify column i nchar(16)" %(paraDict['dbName'], ntbName)
+        tdLog.info("alter table sql: %s"%sqlString)
+        tdSql.error(sqlString)
+        expectRows = 0
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        time.sleep(1)
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 2 end ...... ")
+
+    def run(self):
+        self.prepareTestEnv()
+        self.tmqCase1()
+        self.tmqCase2()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/tmq_taosx.py

@@ -359,8 +359,34 @@ class TDTestCase:
         finally:
             consumer.close()
 
+    def consume_ts_4544(self):
+        tdSql.execute(f'create database if not exists d1')
+        tdSql.execute(f'use d1')
+        tdSql.execute(f'create table stt(ts timestamp, i int) tags(t int)')
+        tdSql.execute(f'insert into tt1 using stt tags(1) values(now, 1) (now+1s, 2)')
+        tdSql.execute(f'insert into tt2 using stt tags(2) values(now, 1) (now+1s, 2)')
+        tdSql.execute(f'insert into tt3 using stt tags(3) values(now, 1) (now+1s, 2)')
+        tdSql.execute(f'insert into tt1 using stt tags(1) values(now+5s, 11) (now+10s, 12)')
+
+        tdSql.execute(f'create topic topic_in as select * from stt where tbname in ("tt2")')
+        consumer_dict = {
+            "group.id": "g1",
+            "td.connect.user": "root",
+            "td.connect.pass": "taosdata",
+            "auto.offset.reset": "earliest",
+        }
+        consumer = Consumer(consumer_dict)
+
+        try:
+            consumer.subscribe(["topic_in"])
+        except TmqError:
+            tdLog.exit(f"subscribe error")
+
+        consumer.close()
+
     def run(self):
         self.consumeTest()
+        self.consume_ts_4544()
 
         tdSql.prepare()
         self.checkWal1VgroupOnlyMeta()

utils/test/c/tmq_taosx_ci.c

@@ -574,6 +574,7 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "msg.with.table.name", "true");
   tmq_conf_set(conf, "enable.auto.commit", "true");
   tmq_conf_set(conf, "auto.offset.reset", "earliest");
+  tmq_conf_set(conf, "msg.consume.excluded", "1");
 
   if (g_conf.snapShot) {
     tmq_conf_set(conf, "experimental.snapshot.enable", "true");