Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/3.0_mhli

Jenkinsfile2

@@ -175,6 +175,7 @@ def pre_test_build_mac() {
         cd ${WK}/debug
         cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false
         make -j10
+        ctest -j10 || exit 7
     '''
     sh '''
         date

cmake/taosadapter_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taosadapter
 ExternalProject_Add(taosadapter
         GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
-        GIT_TAG a11131c
+        GIT_TAG 8c3d57d
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

docs/en/07-develop/09-udf.md

@@ -218,9 +218,9 @@ After compiling your function into a DLL, you add it to TDengine. For more infor
 
 ## Sample Code
 
-### Sample scalar function: [bit_and](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/bit_and.c)
+### Sample scalar function: [bit_and](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/bit_and.c)
 
-The bit_add function implements bitwise addition for multiple columns. If there is only one column, the column is returned. The bit_add function ignores null values.
+The bit_and function implements bitwise addition for multiple columns. If there is only one column, the column is returned. The bit_and function ignores null values.
 
 <details>
 <summary>bit_and.c</summary>
@@ -231,7 +231,7 @@ The bit_add function implements bitwise addition for multiple columns. If there
 
 </details>
 
-### Sample aggregate function: [l2norm](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/l2norm.c)
+### Sample aggregate function: [l2norm](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/l2norm.c)
 
 The l2norm function finds the second-order norm for all data in the input column. This squares the values, takes a cumulative sum, and finds the square root.
 

docs/en/14-reference/05-taosbenchmark.md

@@ -217,7 +217,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
 The parameters listed in this section apply to all function modes.
 
 - **filetype** : The function to be tested, with optional values `insert`, `query` and `subscribe`. These correspond to the insert, query, and subscribe functions, respectively. Users can specify only one of these in each configuration file.
-**cfgdir**: specify the TDengine cluster configuration file's directory. The default path is /etc/taos.
+**cfgdir**: specify the TDengine client configuration file's directory. The default path is /etc/taos.
 
 - **host**: Specify the FQDN of the TDengine server to connect. The default value is `localhost`.
 

docs/en/14-reference/12-config/index.md

@@ -106,7 +106,7 @@ The parameters described in this document by the effect that they have on the sy
 | Applicable | Server only                                           |
 | Meaning       | The switch for monitoring inside server. The main object of monitoring is to collect information about load on physical nodes, including CPU usage, memory usage, disk usage, and network bandwidth. Monitoring information is sent over HTTP to the taosKeeper service specified by `monitorFqdn` and `monitorProt`.
 | Value Range   | 0: monitoring disabled, 1: monitoring enabled                                                                                                                                                                |
-| Default   | 1                                                                                                                                                                                                  |
+| Default   | 0                                                                                                                                                                                                  |
 
 ### monitorFqdn
 
@@ -735,19 +735,17 @@ To prevent system resource from being exhausted by multiple concurrent streams,
 | 44  | numOfVnodeQueryThreads | No | Yes |
 | 45  | numOfVnodeStreamThreads | No | Yes |
 | 46  | numOfVnodeFetchThreads | No | Yes |
-| 47  | numOfVnodeWriteThreads | No | Yes |
+| 47  | numOfVnodeRsmaThreads | No | Yes | 
-| 48  | numOfVnodeSyncThreads | No | Yes |
+| 48  | numOfQnodeQueryThreads | No | Yes |
-| 49  | numOfVnodeRsmaThreads | No | Yes | 
+| 49  | numOfQnodeFetchThreads | No | Yes |
-| 50  | numOfQnodeQueryThreads | No | Yes |
+| 50  | numOfSnodeSharedThreads | No | Yes |
-| 51  | numOfQnodeFetchThreads | No | Yes |
+| 51  | numOfSnodeUniqueThreads | No | Yes |
-| 52  | numOfSnodeSharedThreads | No | Yes |
+| 52  | rpcQueueMemoryAllowed | No | Yes |
-| 53  | numOfSnodeUniqueThreads | No | Yes |
+| 53  | logDir | Yes | Yes |
-| 54  | rpcQueueMemoryAllowed | No | Yes |
+| 54  | minimalLogDirGB | Yes | Yes |
-| 55  | logDir | Yes | Yes |
+| 55  | numOfLogLines | Yes | Yes |
-| 56  | minimalLogDirGB | Yes | Yes |
+| 56  | asyncLog | Yes | Yes |
-| 57  | numOfLogLines | Yes | Yes |
+| 57  | logKeepDays | Yes | Yes |
-| 58  | asyncLog | Yes | Yes |
-| 59  | logKeepDays | Yes | Yes |
 | 60  | debugFlag | Yes | Yes |
 | 61  | tmrDebugFlag | Yes | Yes |
 | 62  | uDebugFlag | Yes | Yes |

docs/en/28-releases/01-tdengine.md

@@ -6,6 +6,10 @@ description: TDengine release history, Release Notes and download links.
 
 import Release from "/components/ReleaseV3";
 
+## 3.0.1.6
+
+<Release type="tdengine" version="3.0.1.6" />
+
 ## 3.0.1.5
 
 <Release type="tdengine" version="3.0.1.5" />

docs/en/28-releases/02-tools.md

@@ -6,6 +6,10 @@ description: taosTools release history, Release Notes, download links.
 
 import Release from "/components/ReleaseV3";
 
+## 2.2.7
+
+<Release type="tools" version="2.2.7" />
+
 ## 2.2.6
 
 <Release type="tools" version="2.2.6" />

docs/examples/csharp/wsConnect/Program.cs

@@ -9,9 +9,10 @@ namespace Examples
         {
             string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
             IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
+  
             if (wsConn == IntPtr.Zero)
             {
-                throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
+                throw new Exception("get WS connection failed");
             }
             else
             {

docs/examples/csharp/wsInsert/Program.cs

@@ -13,7 +13,7 @@ namespace Examples
             // Assert if connection is validate
             if (wsConn == IntPtr.Zero)
             {
-                throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
+                throw new Exception("get WS connection failed");
             }
             else
             {

docs/examples/csharp/wsQuery/Program.cs

@@ -13,7 +13,7 @@ namespace Examples
             IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
             if (wsConn == IntPtr.Zero)
             {
-                throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
+                throw new Exception("get WS connection failed");
             }
             else
             {

docs/examples/csharp/wsStmt/Program.cs

@@ -21,7 +21,7 @@ namespace Examples
             IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
             if (wsConn == IntPtr.Zero)
             {
-                throw new Exception($"get WS connection failed,reason:{LibTaosWS.WSErrorStr(IntPtr.Zero)} code:{LibTaosWS.WSErrorNo(IntPtr.Zero)}");
+                throw new Exception($"get WS connection failed");
             }
             else
             {

docs/zh/10-deployment/01-deploy.md

@@ -190,3 +190,16 @@ DROP DNODE dnodeId;
 dnodeID 是集群自动分配的，不得人工指定。它在生成时是递增的，不会重复。
 
 :::
+
+## 常见问题
+
+1、建立集群时使用 CREATE DNODE 增加新节点后，新节点始终显示 offline 状态？
+```sql
+  1）首先要检查增加的新节点上的 taosd 服务是否已经正常启动
+  
+  2）如果已经启动，再检查到新节点的网络是否通畅，可以使用 ping fqdn 验证下
+  
+  3）如果前面两步都没有问题，这一步要检查新节点做为独立集群在运行了，可以使用 taos -h fqdn 连接上后，show dnodes; 命令查看.
+    如果显示的列表与你主节点上显示的不一致，说明此节点自己单独成立了一个集群，解决的方法是停止新节点上的服务，然后清空新节点上 
+    taos.cfg 中配置的 dataDir 目录下的所有文件，重新启动新节点服务即可解决。
+```  

docs/zh/14-reference/05-taosbenchmark.md

@@ -217,7 +217,7 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\)
 本节所列参数适用于所有功能模式。
 
 - **filetype** : 要测试的功能，可选值为 `insert`, `query` 和 `subscribe`。分别对应插入、查询和订阅功能。每个配置文件中只能指定其中之一。
-- **cfgdir** : TDengine 集群配置文件所在的目录，默认路径是 /etc/taos 。
+- **cfgdir** : TDengine 客户端配置文件所在的目录，默认路径是 /etc/taos 。
 
 - **host** : 指定要连接的 TDengine 服务端的 FQDN，默认值为 localhost。
 

docs/zh/14-reference/12-config/index.md

@@ -106,7 +106,7 @@ taos --dump-config
 | 适用范围 | 仅服务端适用                                                                                                                                                                                       |
 | 含义     | 服务器内部的系统监控开关。监控主要负责收集物理节点的负载状况，包括 CPU、内存、硬盘、网络带宽的监控记录，监控信息将通过 HTTP 协议发送给由 `monitorFqdn` 和 `monitorProt` 指定的 TaosKeeper 监控服务 |
 | 取值范围 | 0：关闭监控服务， 1：激活监控服务。                                                                                                                                                                |
-| 缺省值   | 1                                                                                                                                                                                                  |
+| 缺省值   | 0                                                                                                                                                                                                  |
 
 ### monitorFqdn
 
@@ -711,19 +711,17 @@ charset 的有效值是 UTF-8。
 | 44  | numOfVnodeQueryThreads  | 否              | 是              |                                                   |
 | 45  | numOfVnodeStreamThreads | 否              | 是              |                                                   |
 | 46  | numOfVnodeFetchThreads  | 否              | 是              |                                                   |
-| 47  | numOfVnodeWriteThreads  | 否              | 是              |                                                   |
+| 47  |  numOfVnodeRsmaThreads  | 否              | 是              |                                                   |
-| 48  |  numOfVnodeSyncThreads  | 否              | 是              |                                                   |
+| 48  | numOfQnodeQueryThreads  | 否              | 是              |                                                   |
-| 49  |  numOfVnodeRsmaThreads  | 否              | 是              |                                                   |
+| 49  | numOfQnodeFetchThreads  | 否              | 是              |                                                   |
-| 50  | numOfQnodeQueryThreads  | 否              | 是              |                                                   |
+| 50  | numOfSnodeSharedThreads | 否              | 是              |                                                   |
-| 51  | numOfQnodeFetchThreads  | 否              | 是              |                                                   |
+| 51  | numOfSnodeUniqueThreads | 否              | 是              |                                                   |
-| 52  | numOfSnodeSharedThreads | 否              | 是              |                                                   |
+| 52  |  rpcQueueMemoryAllowed  | 否              | 是              |                                                   |
-| 53  | numOfSnodeUniqueThreads | 否              | 是              |                                                   |
+| 53  |         logDir          | 是              | 是              |                                                   |
-| 54  |  rpcQueueMemoryAllowed  | 否              | 是              |                                                   |
+| 54  |     minimalLogDirGB     | 是              | 是              |                                                   |
-| 55  |         logDir          | 是              | 是              |                                                   |
+| 55  |      numOfLogLines      | 是              | 是              |                                                   |
-| 56  |     minimalLogDirGB     | 是              | 是              |                                                   |
+| 56  |        asyncLog         | 是              | 是              |                                                   |
-| 57  |      numOfLogLines      | 是              | 是              |                                                   |
+| 57  |       logKeepDays       | 是              | 是              |                                                   |
-| 58  |        asyncLog         | 是              | 是              |                                                   |
-| 59  |       logKeepDays       | 是              | 是              |                                                   |
 | 60  |        debugFlag        | 是              | 是              |                                                   |
 | 61  |      tmrDebugFlag       | 是              | 是              |                                                   |
 | 62  |       uDebugFlag        | 是              | 是              |                                                   |

docs/zh/28-releases/01-tdengine.md

@@ -6,6 +6,10 @@ description: TDengine 发布历史、Release Notes 及下载链接
 
 import Release from "/components/ReleaseV3";
 
+## 3.0.1.6
+
+<Release type="tdengine" version="3.0.1.6" />
+
 ## 3.0.1.5
 
 <Release type="tdengine" version="3.0.1.5" />

docs/zh/28-releases/02-tools.md

@@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接
 
 import Release from "/components/ReleaseV3";
 
+## 2.2.7
+
+<Release type="tools" version="2.2.7" />
+
 ## 2.2.6
 
 <Release type="tools" version="2.2.6" />

include/common/tcommon.h

@@ -225,13 +225,13 @@ typedef struct SVarColAttr {
 // pBlockAgg->numOfNull == info.rows, all data are null
 // pBlockAgg->numOfNull == 0, no data are null.
 typedef struct SColumnInfoData {
-  SColumnInfo info;     // column info
+  char*         pData;    // the corresponding block data in memory
-  bool        hasNull;  // if current column data has null value.
-  char*       pData;    // the corresponding block data in memory
   union {
     char*       nullbitmap;  // bitmap, one bit for each item in the list
     SVarColAttr varmeta;
   };
+  SColumnInfo info;     // column info
+  bool        hasNull;  // if current column data has null value.
 } SColumnInfoData;
 
 typedef struct SQueryTableDataCond {

include/common/tglobal.h

@@ -55,8 +55,6 @@ extern int32_t tsNumOfMnodeReadThreads;
 extern int32_t tsNumOfVnodeQueryThreads;
 extern int32_t tsNumOfVnodeStreamThreads;
 extern int32_t tsNumOfVnodeFetchThreads;
-extern int32_t tsNumOfVnodeWriteThreads;
-extern int32_t tsNumOfVnodeSyncThreads;
 extern int32_t tsNumOfVnodeRsmaThreads;
 extern int32_t tsNumOfQnodeQueryThreads;
 extern int32_t tsNumOfQnodeFetchThreads;

include/common/tmsg.h

@@ -297,7 +297,6 @@ typedef struct {
 
 typedef struct {
   int32_t        code;
-  int8_t         hashMeta;
   int64_t        uid;
   char*          tblFName;
   int32_t        numOfRows;
@@ -1128,6 +1127,7 @@ typedef struct {
   SQnodeLoad  qload;
   SClusterCfg clusterCfg;
   SArray*     pVloads;  // array of SVnodeLoad
+  int32_t     statusSeq;
 } SStatusReq;
 
 int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
@@ -1149,6 +1149,7 @@ typedef struct {
   int64_t   dnodeVer;
   SDnodeCfg dnodeCfg;
   SArray*   pDnodeEps;  // Array of SDnodeEp
+  int32_t   statusSeq;
 } SStatusRsp;
 
 int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp);

include/common/tmsgdef.h

@@ -241,6 +241,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "vnode-stream-finish", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
 
   TD_NEW_MSG_SEG(TDMT_MON_MSG)

include/libs/catalog/catalog.h

@@ -203,13 +203,11 @@ int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg);
 
 int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp* rspMsg);
 
-int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta);
+int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta);
 
-int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
+int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta);
-                            STableMeta** pTableMeta);
 
-int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
+int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName,           SVgroupInfo* pVgroup, bool* exists);
-                                  SVgroupInfo* pVgroup, bool* exists);
 
 /**
  * Force refresh DB's local cached vgroup info.
@@ -309,7 +307,7 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f
 int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
                        bool* pass);
 
-int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
+int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type,
                                        bool* pass, bool* exists);
 
 int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth);
@@ -326,6 +324,10 @@ SMetaData* catalogCloneMetaData(SMetaData* pData);
 
 void catalogFreeMetaData(SMetaData* pData);
 
+int32_t ctgdEnableDebug(char *option, bool enable);
+
+int32_t ctgdHandleDbgCommand(char *command);
+
 /**
  * Destroy catalog and relase all resources
  */

include/libs/qcom/query.h

@@ -248,7 +248,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
    (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID))
 #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \
   ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID)
-#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED)
+#define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER)
 #define NEED_CLIENT_HANDLE_ERROR(_code)                                          \
   (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
    NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))

include/libs/stream/tstream.h

@@ -46,7 +46,7 @@ enum {
   TASK_STATUS__DROPPING,
   TASK_STATUS__FAIL,
   TASK_STATUS__STOP,
-  TASK_STATUS__RECOVER_DOWNSTREAM,
+  TASK_STATUS__WAIT_DOWNSTREAM,
   TASK_STATUS__RECOVER_PREPARE,
   TASK_STATUS__RECOVER1,
   TASK_STATUS__RECOVER2,
@@ -332,7 +332,10 @@ typedef struct SStreamTask {
   SStreamState* pState;
 
   // do not serialize
-  int32_t recoverWaitingChild;
+  int32_t recoverTryingDownstream;
+  int32_t recoverWaitingUpstream;
+  int64_t checkReqId;
+  SArray* checkReqIds;  // shuffle
 
 } SStreamTask;
 
@@ -418,7 +421,10 @@ typedef struct {
 
 typedef struct {
   int64_t streamId;
-  int32_t taskId;
+  int32_t upstreamNodeId;
+  int32_t upstreamTaskId;
+  int32_t downstreamNodeId;
+  int32_t downstreamTaskId;
   int8_t  inputStatus;
 } SStreamDispatchRsp;
 
@@ -440,6 +446,27 @@ typedef struct {
   int32_t rspToTaskId;
 } SStreamRetrieveRsp;
 
+typedef struct {
+  int64_t reqId;
+  int64_t streamId;
+  int32_t upstreamNodeId;
+  int32_t upstreamTaskId;
+  int32_t downstreamNodeId;
+  int32_t downstreamTaskId;
+  int32_t childId;
+} SStreamTaskCheckReq;
+
+typedef struct {
+  int64_t reqId;
+  int64_t streamId;
+  int32_t upstreamNodeId;
+  int32_t upstreamTaskId;
+  int32_t downstreamNodeId;
+  int32_t downstreamTaskId;
+  int32_t childId;
+  int8_t  status;
+} SStreamTaskCheckRsp;
+
 typedef struct {
   SMsgHead msgHead;
   int64_t  streamId;
@@ -455,47 +482,6 @@ typedef struct {
 int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq);
 int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishReq* pReq);
 
-#if 0
-typedef struct {
-  int64_t streamId;
-  int32_t taskId;
-  int32_t upstreamTaskId;
-  int32_t upstreamNodeId;
-} SStreamTaskRecoverReq;
-
-typedef struct {
-  int64_t streamId;
-  int32_t rspTaskId;
-  int32_t reqTaskId;
-  int8_t  inputStatus;
-} SStreamTaskRecoverRsp;
-
-int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq);
-int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq);
-
-int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp);
-int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pRsp);
-
-typedef struct {
-  int64_t streamId;
-  int32_t taskId;
-} SMStreamTaskRecoverReq;
-
-typedef struct {
-  int64_t streamId;
-  int32_t taskId;
-} SMStreamTaskRecoverRsp;
-
-int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq);
-int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq);
-
-int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp);
-int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pRsp);
-
-int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg);
-int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp);
-#endif
-
 typedef struct {
   int64_t streamId;
   int32_t downstreamTaskId;
@@ -509,20 +495,18 @@ typedef struct {
   SArray* checkpointVer;  // SArray<SStreamCheckpointInfo>
 } SStreamRecoverDownstreamRsp;
 
+int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq);
+int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq);
+
+int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp);
+int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp);
+
 int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq);
 int32_t tDecodeSStreamTaskRecoverReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq);
 
 int32_t tEncodeSStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamRecoverDownstreamRsp* pRsp);
 int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamRecoverDownstreamRsp* pRsp);
 
-typedef struct {
-  int64_t streamId;
-  int32_t taskId;
-  int32_t waitingRspCnt;
-  int32_t totReq;
-  SArray* info;  // SArray<SArray<SStreamCheckpointInfo>*>
-} SStreamRecoverStatus;
-
 int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq);
 int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq);
 void    tDeleteStreamRetrieveReq(SStreamRetrieveReq* pReq);
@@ -533,7 +517,7 @@ int32_t streamSetupTrigger(SStreamTask* pTask);
 
 int32_t streamProcessRunReq(SStreamTask* pTask);
 int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg, bool exec);
-int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp);
+int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code);
 
 int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg);
 int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp);
@@ -544,6 +528,10 @@ int32_t streamSchedExec(SStreamTask* pTask);
 int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz);
 
 // recover and fill history
+int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version);
+int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version);
+int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq);
+int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version);
 // common
 int32_t streamSetParamForRecover(SStreamTask* pTask);
 int32_t streamRestoreParam(SStreamTask* pTask);

include/util/taoserror.h

@@ -340,7 +340,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TDB_TABLE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0618)
 #define TSDB_CODE_TDB_STB_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x0619)
 #define TSDB_CODE_TDB_STB_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x061A)
-#define TSDB_CODE_TDB_TABLE_RECREATED           TAOS_DEF_ERROR_CODE(0, 0x061B)
+#define TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER  TAOS_DEF_ERROR_CODE(0, 0x061B)
 #define TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR        TAOS_DEF_ERROR_CODE(0, 0x061C)
 
 // query
@@ -612,9 +612,12 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TMQ_CONSUMER_MISMATCH          TAOS_DEF_ERROR_CODE(0, 0x4001)
 #define TSDB_CODE_TMQ_CONSUMER_CLOSED            TAOS_DEF_ERROR_CODE(0, 0x4002)
 
+// stream
+#define TSDB_CODE_STREAM_TASK_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x4100)
+
 // TDLite
-#define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS         TAOS_DEF_ERROR_CODE(0, 0x4100)
+#define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS         TAOS_DEF_ERROR_CODE(0, 0x5100)
-#define TSDB_CODE_TDLITE_IVLD_OPEN_DIR           TAOS_DEF_ERROR_CODE(0, 0x4101)
+#define TSDB_CODE_TDLITE_IVLD_OPEN_DIR           TAOS_DEF_ERROR_CODE(0, 0x5101)
 
 #ifdef __cplusplus
 }

include/util/tqueue.h

@@ -59,6 +59,47 @@ typedef enum {
 typedef void (*FItem)(SQueueInfo *pInfo, void *pItem);
 typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems);
 
+typedef struct STaosQnode STaosQnode;
+
+typedef struct STaosQnode {
+  STaosQnode *next;
+  STaosQueue *queue;
+  int64_t     timestamp;
+  int32_t     size;
+  int8_t      itype;
+  int8_t      reserved[3];
+  char        item[];
+} STaosQnode;
+
+typedef struct STaosQueue {
+  STaosQnode   *head;
+  STaosQnode   *tail;
+  STaosQueue   *next;     // for queue set
+  STaosQset    *qset;     // for queue set
+  void         *ahandle;  // for queue set
+  FItem         itemFp;
+  FItems        itemsFp;
+  TdThreadMutex mutex;
+  int64_t       memOfItems;
+  int32_t       numOfItems;
+  int64_t       threadId;
+} STaosQueue;
+
+typedef struct STaosQset {
+  STaosQueue   *head;
+  STaosQueue   *current;
+  TdThreadMutex mutex;
+  tsem_t        sem;
+  int32_t       numOfQueues;
+  int32_t       numOfItems;
+} STaosQset;
+
+typedef struct STaosQall {
+  STaosQnode *current;
+  STaosQnode *start;
+  int32_t     numOfItems;
+} STaosQall;
+
 STaosQueue *taosOpenQueue();
 void        taosCloseQueue(STaosQueue *queue);
 void        taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp);

include/util/tworker.h

@@ -26,8 +26,9 @@ typedef struct SQWorkerPool SQWorkerPool;
 typedef struct SWWorkerPool SWWorkerPool;
 
 typedef struct SQWorker {
-  int32_t       id;      // worker ID
+  int32_t       id;      // worker id
-  TdThread      thread;  // thread
+  int64_t       pid;     // thread pid
+  TdThread      thread;  // thread id
   SQWorkerPool *pool;
 } SQWorker;
 
@@ -43,9 +44,10 @@ typedef struct SQWorkerPool {
 
 typedef struct SWWorker {
   int32_t       id;      // worker id
-  TdThread      thread;  // thread
+  int64_t       pid;     // thread pid
+  TdThread      thread;  // thread id
   STaosQall    *qall;
-  STaosQset    *qset;  // queue set
+  STaosQset    *qset;
   SWWorkerPool *pool;
 } SWWorker;
 

packaging/release.bat

@@ -40,7 +40,7 @@ if not exist %work_dir%\debug\ver-%2-x86 (
 )
 cd %work_dir%\debug\ver-%2-x64
 call vcvarsall.bat x64
-cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DBUILD_HTTP=false -DBUILD_TEST=false -DVERNUMBER=%2 -DCPUTYPE=x64
+cmake ../../ -G "NMake Makefiles JOM" -DCMAKE_MAKE_PROGRAM=jom -DBUILD_TOOLS=true -DWEBSOCKET=true -DBUILD_HTTP=false -DBUILD_TEST=false -DVERNUMBER=%2 -DCPUTYPE=x64
 cmake --build .
 rd /s /Q C:\TDengine
 cmake --install .

packaging/testpackage.sh

@@ -67,6 +67,7 @@ fi
 }
 
 
+
 function wgetFile {
 
 file=$1
@@ -75,7 +76,10 @@ if [ ! -f  ${file}  ];then
     echoColor  BD "wget https://www.taosdata.com/assets-download/3.0/${file}"
     wget https://www.taosdata.com/assets-download/3.0/${file}
 else
-    echoColor  YD "${file} already exists "
+    echoColor  YD "${file} already exists and use new file "
+    rm -rf ${file}
+    echoColor  BD "wget https://www.taosdata.com/assets-download/3.0/${file}"
+    wget https://www.taosdata.com/assets-download/3.0/${file}  
 fi
 }
 
@@ -211,7 +215,8 @@ elif [[ ${packgeName} =~ "tar" ]];then
 
     if [ ${diffNumbers} != 0 ];then
         echoColor R "The number and names of files is different from the previous installation package"
-        echoColor Y `cat ${installPath}/diffFile.log`
+        diffLog=`cat ${installPath}/diffFile.log`
+        echoColor Y "${diffLog}"
         exit -1
     else 
         echoColor G "The number and names of files are the same as previous installation packages"

packaging/tools/make_install.bat

@@ -34,7 +34,6 @@ if exist %binary_dir%\\test\\cfg\\taosadapter.toml (
         copy %binary_dir%\\test\\cfg\\taosadapter.toml %tagert_dir%\\cfg\\taosadapter.toml > nul
     )
 )
-
 copy %source_dir%\\include\\client\\taos.h %tagert_dir%\\include > nul
 copy %source_dir%\\include\\util\\taoserror.h %tagert_dir%\\include > nul
 copy %source_dir%\\include\\libs\\function\\taosudf.h %tagert_dir%\\include > nul
@@ -52,6 +51,7 @@ if exist %binary_dir%\\build\\lib\\taosws.dll.lib (
 )
 if exist %binary_dir%\\build\\lib\\taosws.dll (
     copy %binary_dir%\\build\\lib\\taosws.dll %tagert_dir%\\driver  > nul
+    copy %source_dir%\\tools\\taosws-rs\\target\\release\\taosws.h %tagert_dir%\\include > nul
 )
 if exist %binary_dir%\\build\\bin\\taosdump.exe (
     copy %binary_dir%\\build\\bin\\taosdump.exe %tagert_dir% > nul
@@ -67,3 +67,4 @@ if exist C:\\TDengine\\driver\\taosws.dll (
     copy /y C:\\TDengine\\driver\\taosws.dll C:\\Windows\\System32 > nul
 )
 sc query "taosd" >nul || sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND
+sc query "taosadapter" >nul || sc create "taosadapter" binPath= "C:\\TDengine\\taosadapter.exe" start= DEMAND

packaging/tools/makepkg.sh

@@ -285,10 +285,14 @@ if [[ $dbName == "taos" ]]; then
   fi
 
   # Add web files
-  if [ -d "${web_dir}/admin" ]; then
+  if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then
-    mkdir -p ${install_dir}/share/
+    if [ -d "${web_dir}/admin" ] ; then
-    cp ${web_dir}/admin ${install_dir}/share/ -r
+      mkdir -p ${install_dir}/share/
-    cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png
+      cp ${web_dir}/admin ${install_dir}/share/ -r
+      cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png
+    else
+      echo "directory not found for enterprise release: ${web_dir}/admin"
+    fi
   fi
 fi
 

packaging/tools/tdengine.iss

@@ -15,10 +15,10 @@
 #define MyAppExeName "\*.exe"
 #define MyAppTaosExeName "\taos.bat"
 #define MyAppTaosdemoExeName "\taosBenchmark.exe"
-#define MyAppDLLName "\driver\taos.dll"
+#define MyAppDLLName "\driver\*.dll"
 ;#define MyAppVersion "3.0"
 ;#define MyAppInstallName "TDengine"
-
+;#define MyAppInstallName "TDengine"
 [Setup]
 VersionInfoVersion={#MyAppVersion}
 AppId={{A0F7A93C-79C4-485D-B2B8-F0D03DF42FAB}

source/client/inc/clientInt.h

@@ -318,6 +318,7 @@ void*    createTscObj(const char* user, const char* auth, const char* db, int32_
 void     destroyTscObj(void* pObj);
 STscObj* acquireTscObj(int64_t rid);
 int32_t  releaseTscObj(int64_t rid);
+void destroyAppInst(SAppInstInfo *pAppInfo);
 
 uint64_t generateRequestId();
 

source/client/inc/clientStmt.h

@@ -39,6 +39,7 @@ typedef enum {
   STMT_BIND_COL,
   STMT_ADD_BATCH,
   STMT_EXECUTE,
+  STMT_MAX,
 } STMT_STATUS;
 
 typedef struct SStmtTableCache {
@@ -94,12 +95,18 @@ typedef struct STscStmt {
   STscObj  *taos;
   SCatalog *pCatalog;
   int32_t   affectedRows;
+  uint32_t  seqId;
+  uint32_t  seqIds[STMT_MAX];
 
   SStmtSQLInfo  sql;
   SStmtExecInfo exec;
   SStmtBindInfo bInfo;
 } STscStmt;
 
+extern char *gStmtStatusStr[];
+
+#define STMT_LOG_SEQ(n) do { (pStmt)->seqId++; (pStmt)->seqIds[n]++; STMT_DLOG("the %dth:%d %s", (pStmt)->seqIds[n], (pStmt)->seqId, gStmtStatusStr[n]); } while (0)
+
 #define STMT_STATUS_NE(S) (pStmt->sql.status != STMT_##S)
 #define STMT_STATUS_EQ(S) (pStmt->sql.status == STMT_##S)
 
@@ -128,6 +135,12 @@ typedef struct STscStmt {
     }                                \
   } while (0)
 
+#define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__)
+#define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__)
+
+#define STMT_ELOG_E(param) qError("stmt:%p " param, pStmt)
+#define STMT_DLOG_E(param) qDebug("stmt:%p " param, pStmt)
+
 TAOS_STMT  *stmtInit(STscObj *taos);
 int         stmtClose(TAOS_STMT *stmt);
 int         stmtExec(TAOS_STMT *stmt);

source/client/src/clientImpl.c

@@ -133,6 +133,12 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
     taosThreadMutexInit(&p->qnodeMutex, NULL);
     p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores);
     p->pAppHbMgr = appHbMgrInit(p, key);
+    if (NULL == p->pAppHbMgr) {
+      destroyAppInst(p);
+      taosThreadMutexUnlock(&appInfo.mutex);
+      taosMemoryFreeClear(key);
+      return NULL;
+    }
     taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES);
     p->instKey = key;
     key = NULL;
@@ -1266,7 +1272,9 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) {
   pMsgSendInfo->requestObjRefId = pRequest->self;
   pMsgSendInfo->requestId = pRequest->requestId;
   pMsgSendInfo->fp = getMsgRspHandle(pMsgSendInfo->msgType);
-  pMsgSendInfo->param = pRequest;
+  pMsgSendInfo->param = taosMemoryCalloc(1, sizeof(pRequest->self));
+
+  *(int64_t*)pMsgSendInfo->param = pRequest->self;
 
   SConnectReq connectReq = {0};
   STscObj*    pObj = pRequest->pTscObj;

source/client/src/clientMain.c

@@ -286,7 +286,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
     return NULL;
   } else {
     // assert to avoid un-initialization error
-    ASSERT(0);
+    tscError("invalid result passed to taos_fetch_row");
+    return NULL;
   }
   return NULL;
 }

source/client/src/clientMsgHandler.c

@@ -45,7 +45,12 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
 }
 
 int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
-  SRequestObj* pRequest = param;
+  SRequestObj *pRequest = acquireRequest(*(int64_t*)param);
+  if (NULL == pRequest) {
+    setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED);
+    tsem_post(&pRequest->body.rspSem);
+    goto End;
+  }
   
   if (code != TSDB_CODE_SUCCESS) {
     setErrno(pRequest, code);
@@ -55,6 +60,12 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
 
   STscObj* pTscObj = pRequest->pTscObj;
 
+  if (NULL == pTscObj->pAppInfo || NULL == pTscObj->pAppInfo->pAppHbMgr) {
+    setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED);
+    tsem_post(&pRequest->body.rspSem);
+    goto End;
+  }
+  
   SConnectRsp connectRsp = {0};
   if (tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp) != 0) {
     code = TSDB_CODE_TSC_INVALID_VERSION;
@@ -119,6 +130,11 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
   tsem_post(&pRequest->body.rspSem);
 End:
 
+  if (pRequest) {
+    releaseRequest(pRequest->self);
+  }
+  
+  taosMemoryFree(param);
   taosMemoryFree(pMsg->pEpSet);
   taosMemoryFree(pMsg->pData);
   return code;

source/client/src/clientStmt.c

@@ -5,6 +5,8 @@
 
 #include "clientStmt.h"
 
+char *gStmtStatusStr[] = {"unknown", "init", "prepare", "settbname", "settags", "fetchFields", "bind", "bindCol", "addBatch", "exec"};
+
 static int32_t stmtCreateRequest(STscStmt* pStmt) {
   int32_t code = 0;
 
@@ -21,6 +23,10 @@ static int32_t stmtCreateRequest(STscStmt* pStmt) {
 int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
   int32_t code = 0;
 
+  if (newStatus >= STMT_INIT && newStatus < STMT_MAX) {
+    STMT_LOG_SEQ(newStatus);
+  }
+
   switch (newStatus) {
     case STMT_PREPARE:
       break;
@@ -528,13 +534,17 @@ TAOS_STMT* stmtInit(STscObj* taos) {
   pStmt->bInfo.needParse = true;
   pStmt->sql.status = STMT_INIT;
 
+  STMT_LOG_SEQ(STMT_INIT);
+  
+  tscDebug("stmt:%p initialized", pStmt);
+
   return pStmt;
 }
 
 int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  tscDebug("stmt start to prepare");
+  STMT_DLOG_E("start to prepare");
 
   if (pStmt->sql.status >= STMT_PREPARE) {
     STMT_ERR_RET(stmtResetStmt(pStmt));
@@ -555,7 +565,7 @@ int stmtPrepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
 int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  tscDebug("stmt start to set tbName: %s", tbName);
+  STMT_DLOG("start to set tbName: %s", tbName);
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME));
 
@@ -587,7 +597,7 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) {
 int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  tscDebug("stmt start to set tbTags");
+  STMT_DLOG_E("start to set tbTags");
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
 
@@ -649,7 +659,7 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields
 int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  tscDebug("start to bind stmt data, colIdx: %d", colIdx);
+  STMT_DLOG("start to bind stmt data, colIdx: %d", colIdx);
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
 
@@ -743,7 +753,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) {
 int stmtAddBatch(TAOS_STMT* stmt) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
-  tscDebug("stmt start to add batch");
+  STMT_DLOG_E("start to add batch");
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_ADD_BATCH));
 
@@ -756,8 +766,7 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) {
   tscDebug("stmt start to update tbUid, blockNum: %d", pRsp->nBlocks);
 
   if (pRsp->nBlocks <= 0) {
-    tscError("invalid submit resp block number %d", pRsp->nBlocks);
+    return TSDB_CODE_SUCCESS;
-    STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
   }
 
   size_t             keyLen = 0;
@@ -810,7 +819,7 @@ int stmtExec(TAOS_STMT* stmt) {
   SSubmitRsp* pRsp = NULL;
   bool        autoCreateTbl = pStmt->exec.autoCreateTbl;
 
-  tscDebug("stmt start to exec");
+  STMT_DLOG_E("start to exec");
 
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
 
@@ -885,6 +894,8 @@ int stmtAffectedRowsOnce(TAOS_STMT* stmt) { return ((STscStmt*)stmt)->exec.affec
 int stmtIsInsert(TAOS_STMT* stmt, int* insert) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start is insert");
+
   if (pStmt->sql.type) {
     *insert = (STMT_TYPE_INSERT == pStmt->sql.type || STMT_TYPE_MULTI_INSERT == pStmt->sql.type);
   } else {
@@ -897,6 +908,8 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) {
 int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start to get tag fields");
+
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
     STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
   }
@@ -927,6 +940,8 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
 int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start to get col fields");
+
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
     STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
   }
@@ -957,6 +972,8 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) {
 int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start to get param num");
+
   STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
 
   if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 &&
@@ -986,6 +1003,8 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) {
 int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start to get param");
+
   if (STMT_TYPE_QUERY == pStmt->sql.type) {
     STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR);
   }
@@ -1028,6 +1047,8 @@ int stmtGetParam(TAOS_STMT* stmt, int idx, int* type, int* bytes) {
 TAOS_RES* stmtUseResult(TAOS_STMT* stmt) {
   STscStmt* pStmt = (STscStmt*)stmt;
 
+  STMT_DLOG_E("start to use result");
+
   if (STMT_TYPE_QUERY != pStmt->sql.type) {
     tscError("useResult only for query statement");
     return NULL;

source/client/src/clientTmq.c

@@ -1756,7 +1756,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
 
     rspObj = tmqHandleAllRsp(tmq, timeout, false);
     if (rspObj) {
-      tscDebug("consumer:%" PRId64 ", return rsp", tmq->consumerId);
+      tscDebug("consumer:%" PRId64 ", return rsp %p", tmq->consumerId, rspObj);
       return (TAOS_RES*)rspObj;
     } else if (terrno == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) {
       tscDebug("consumer:%" PRId64 ", return null since no committed offset", tmq->consumerId);

source/client/test/clientTests.cpp

@@ -112,7 +112,7 @@ void createNewTable(TAOS* pConn, int32_t index) {
   }
   taos_free_result(pRes);
 
-  for(int32_t i = 0; i < 10000; i += 20) {
+  for(int32_t i = 0; i < 20; i += 20) {
     char sql[1024] = {0};
     sprintf(sql,
             "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)"
@@ -692,6 +692,7 @@ TEST(testCase, insert_test) {
   taos_free_result(pRes);
   taos_close(pConn);
 }
+#endif
 
 TEST(testCase, projection_query_tables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@@ -703,7 +704,7 @@ TEST(testCase, projection_query_tables) {
   //  }
   //  taos_free_result(pRes);
 
-  TAOS_RES* pRes = taos_query(pConn, "use benchmarkcpu");
+  TAOS_RES* pRes = taos_query(pConn, "use abc2");
   taos_free_result(pRes);
 
   pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");
@@ -725,7 +726,7 @@ TEST(testCase, projection_query_tables) {
   }
   taos_free_result(pRes);
 
-  for (int32_t i = 0; i < 2; ++i) {
+  for (int32_t i = 0; i < 200000; ++i) {
     printf("create table :%d\n", i);
     createNewTable(pConn, i);
   }
@@ -750,7 +751,9 @@ TEST(testCase, projection_query_tables) {
   taos_free_result(pRes);
   taos_close(pConn);
 }
-#endif
+
+
+#if 0
 
 TEST(testCase, tsbs_perf_test) {
   TdThread qid[20] = {0};
@@ -761,7 +764,7 @@ TEST(testCase, tsbs_perf_test) {
   getchar();
 }
 
-#if 0
+
 TEST(testCase, projection_query_stables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);

source/common/src/tglobal.c

@@ -50,8 +50,6 @@ int32_t tsNumOfMnodeReadThreads = 1;
 int32_t tsNumOfVnodeQueryThreads = 4;
 int32_t tsNumOfVnodeStreamThreads = 2;
 int32_t tsNumOfVnodeFetchThreads = 4;
-int32_t tsNumOfVnodeWriteThreads = 2;
-int32_t tsNumOfVnodeSyncThreads = 2;
 int32_t tsNumOfVnodeRsmaThreads = 2;
 int32_t tsNumOfQnodeQueryThreads = 4;
 int32_t tsNumOfQnodeFetchThreads = 1;
@@ -374,14 +372,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4);
   if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, 0) != 0) return -1;
 
-  tsNumOfVnodeWriteThreads = tsNumOfCores;
-  tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1);
-  if (cfgAddInt32(pCfg, "numOfVnodeWriteThreads", tsNumOfVnodeWriteThreads, 1, 1024, 0) != 0) return -1;
-
-  tsNumOfVnodeSyncThreads = tsNumOfCores * 2;
-  tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 16);
-  if (cfgAddInt32(pCfg, "numOfVnodeSyncThreads", tsNumOfVnodeSyncThreads, 1, 1024, 0) != 0) return -1;
-
   tsNumOfVnodeRsmaThreads = tsNumOfCores;
   tsNumOfVnodeRsmaThreads = TMAX(tsNumOfVnodeRsmaThreads, 4);
   if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, 0) != 0) return -1;
@@ -506,22 +496,6 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
     pItem->stype = stype;
   }
 
-  pItem = cfgGetItem(tsCfg, "numOfVnodeWriteThreads");
-  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
-    tsNumOfVnodeWriteThreads = numOfCores;
-    tsNumOfVnodeWriteThreads = TMAX(tsNumOfVnodeWriteThreads, 1);
-    pItem->i32 = tsNumOfVnodeWriteThreads;
-    pItem->stype = stype;
-  }
-
-  pItem = cfgGetItem(tsCfg, "numOfVnodeSyncThreads");
-  if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
-    tsNumOfVnodeSyncThreads = numOfCores * 2;
-    tsNumOfVnodeSyncThreads = TMAX(tsNumOfVnodeSyncThreads, 16);
-    pItem->i32 = tsNumOfVnodeSyncThreads;
-    pItem->stype = stype;
-  }
-
   pItem = cfgGetItem(tsCfg, "numOfVnodeRsmaThreads");
   if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
     tsNumOfVnodeRsmaThreads = numOfCores;
@@ -699,8 +673,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
   tsNumOfVnodeStreamThreads = cfgGetItem(pCfg, "numOfVnodeStreamThreads")->i32;
   tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
-  tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
-  tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
   tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
   tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32;
   //  tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32;
@@ -943,10 +915,6 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) {
               } else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) {
                 tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32;
         */
-      } else if (strcasecmp("numOfVnodeWriteThreads", name) == 0) {
-        tsNumOfVnodeWriteThreads = cfgGetItem(pCfg, "numOfVnodeWriteThreads")->i32;
-      } else if (strcasecmp("numOfVnodeSyncThreads", name) == 0) {
-        tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32;
       } else if (strcasecmp("numOfVnodeRsmaThreads", name) == 0) {
         tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32;
       } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) {

source/common/src/tmsg.c

@@ -1020,6 +1020,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
   if (tEncodeI64(&encoder, pReq->qload.timeInQueryQueue) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->qload.timeInFetchQueue) < 0) return -1;
 
+  if (tEncodeI32(&encoder, pReq->statusSeq) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1095,6 +1096,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
   if (tDecodeI64(&decoder, &pReq->qload.timeInQueryQueue) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->qload.timeInFetchQueue) < 0) return -1;
 
+  if (tDecodeI32(&decoder, &pReq->statusSeq) < 0) return -1;
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
   return 0;
@@ -1126,6 +1128,7 @@ int32_t tSerializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
     if (tEncodeU16(&encoder, pDnodeEp->ep.port) < 0) return -1;
   }
 
+  if (tEncodeI32(&encoder, pRsp->statusSeq) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -1167,6 +1170,7 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
     }
   }
 
+  if (tDecodeI32(&decoder, &pRsp->statusSeq) < 0) return -1;
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
   return 0;
@@ -5432,9 +5436,12 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl
   if (tStartEncode(pEncoder) < 0) return -1;
 
   if (tEncodeI32(pEncoder, pBlock->code) < 0) return -1;
-  if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1;
   if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1;
-  if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1;
+  if (pBlock->tblFName) {
+    if (tEncodeCStr(pEncoder, pBlock->tblFName) < 0) return -1;
+  } else {
+    if (tEncodeCStr(pEncoder, "") < 0) return -1;
+  }
   if (tEncodeI32v(pEncoder, pBlock->numOfRows) < 0) return -1;
   if (tEncodeI32v(pEncoder, pBlock->affectedRows) < 0) return -1;
   if (tEncodeI64v(pEncoder, pBlock->sver) < 0) return -1;
@@ -5451,7 +5458,6 @@ static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
   if (tStartDecode(pDecoder) < 0) return -1;
 
   if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1;
   if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
   pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
   if (NULL == pBlock->tblFName) return -1;

source/dnode/mgmt/mgmt_dnode/inc/dmInt.h

@@ -36,6 +36,7 @@ typedef struct SDnodeMgmt {
   GetVnodeLoadsFp     getVnodeLoadsFp;
   GetMnodeLoadsFp     getMnodeLoadsFp;
   GetQnodeLoadsFp     getQnodeLoadsFp;
+  int32_t             statusSeq;
 } SDnodeMgmt;
 
 // dmHandle.c

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -32,9 +32,13 @@ static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
 }
 
 static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
+  const STraceId *trace = &pRsp->info.traceId;
+  dGTrace("status rsp received from mnode, statusSeq:%d code:0x%x", pMgmt->statusSeq, pRsp->code);
+
   if (pRsp->code != 0) {
     if (pRsp->code == TSDB_CODE_MND_DNODE_NOT_EXIST && !pMgmt->pData->dropped && pMgmt->pData->dnodeId > 0) {
-      dInfo("dnode:%d, set to dropped since not exist in mnode", pMgmt->pData->dnodeId);
+      dGInfo("dnode:%d, set to dropped since not exist in mnode, statusSeq:%d", pMgmt->pData->dnodeId,
+             pMgmt->statusSeq);
       pMgmt->pData->dropped = 1;
       dmWriteEps(pMgmt->pData);
     }
@@ -42,9 +46,9 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
     SStatusRsp statusRsp = {0};
     if (pRsp->pCont != NULL && pRsp->contLen > 0 &&
         tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) {
-      dTrace("status msg received from mnode, dnodeVer:%" PRId64 " saved:%" PRId64, statusRsp.dnodeVer,
-             pMgmt->pData->dnodeVer);
       if (pMgmt->pData->dnodeVer != statusRsp.dnodeVer) {
+        dGInfo("status rsp received from mnode, statusSeq:%d:%d dnodeVer:%" PRId64 ":%" PRId64, pMgmt->statusSeq,
+               statusRsp.statusSeq, pMgmt->pData->dnodeVer, statusRsp.dnodeVer);
         pMgmt->pData->dnodeVer = statusRsp.dnodeVer;
         dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg);
         dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps);
@@ -91,6 +95,9 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
 
   (*pMgmt->getQnodeLoadsFp)(&req.qload);
 
+  pMgmt->statusSeq++;
+  req.statusSeq = pMgmt->statusSeq;
+
   int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
   void   *pHead = rpcMallocCont(contLen);
   tSerializeSStatusReq(pHead, contLen, &req);
@@ -99,13 +106,13 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
   SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527};
   SRpcMsg rpcRsp = {0};
 
-  dTrace("send status msg to mnode, dnodeVer:%" PRId64, req.dnodeVer);
+  dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq);
 
   SEpSet epSet = {0};
   dmGetMnodeEpSet(pMgmt->pData, &epSet);
   rpcSendRecv(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp);
   if (rpcRsp.code != 0) {
-    dError("failed to send status msg since %s, numOfEps:%d inUse:%d", tstrerror(rpcRsp.code), epSet.numOfEps,
+    dError("failed to send status req since %s, numOfEps:%d inUse:%d", tstrerror(rpcRsp.code), epSet.numOfEps,
            epSet.inUse);
     for (int32_t i = 0; i < epSet.numOfEps; ++i) {
       dDebug("index:%d, mnode ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port);

source/dnode/mgmt/mgmt_snode/src/smWorker.c

@@ -58,7 +58,11 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   dTrace("msg:%p, get from snode-stream queue", pMsg);
   int32_t code = sndProcessStreamMsg(pMgmt->pSnode, pMsg);
   if (code < 0) {
-    dGError("snd, msg:%p failed to process stream since %s", pMsg, terrstr(code));
+    if (pMsg) {
+      dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code));
+    } else {
+      dGError("snd, msg:%p failed to process stream empty msg since %s", pMsg, terrstr(code));
+    }
     smSendRsp(pMsg, terrno);
   }
 

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -33,10 +33,6 @@ typedef struct SVnodeMgmt {
   SQWorkerPool   queryPool;
   SQWorkerPool   streamPool;
   SWWorkerPool   fetchPool;
-  SWWorkerPool   syncPool;
-  SWWorkerPool   syncCtrlPool;
-  SWWorkerPool   writePool;
-  SWWorkerPool   applyPool;
   SSingleWorker  mgmtWorker;
   SHashObj      *hash;
   TdThreadRwlock lock;
@@ -52,19 +48,19 @@ typedef struct {
 } SWrapperCfg;
 
 typedef struct {
-  int32_t     vgId;
+  int32_t       vgId;
-  int32_t     vgVersion;
+  int32_t       vgVersion;
-  int32_t     refCount;
+  int32_t       refCount;
-  int8_t      dropped;
+  int8_t        dropped;
-  char       *path;
+  char         *path;
-  SVnode     *pImpl;
+  SVnode       *pImpl;
-  STaosQueue *pWriteQ;
+  SMultiWorker  pWriteW;
-  STaosQueue *pSyncQ;
+  SMultiWorker  pSyncW;
-  STaosQueue *pSyncCtrlQ;
+  SMultiWorker  pSyncCtrlW;
-  STaosQueue *pApplyQ;
+  SMultiWorker  pApplyW;
-  STaosQueue *pQueryQ;
+  STaosQueue   *pQueryQ;
-  STaosQueue *pStreamQ;
+  STaosQueue   *pStreamQ;
-  STaosQueue *pFetchQ;
+  STaosQueue   *pFetchQ;
 } SVnodeObj;
 
 typedef struct {

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -403,7 +403,6 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
 
-  if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
@@ -412,6 +411,9 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_STREAM_RECOVER_FINISH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECK_RSP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -79,29 +79,49 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
 void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
   char path[TSDB_FILENAME_LEN] = {0};
 
-  vnodePreClose(pVnode->pImpl);
-
   taosThreadRwlockWrlock(&pMgmt->lock);
   taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t));
   taosThreadRwlockUnlock(&pMgmt->lock);
   vmReleaseVnode(pMgmt, pVnode);
 
-  dTrace("vgId:%d, wait for vnode ref become 0", pVnode->vgId);
+  dInfo("vgId:%d, pre close", pVnode->vgId);
-  while (pVnode->refCount > 0) taosMsleep(10);
+  vnodePreClose(pVnode->pImpl);
-  dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId);
 
-  while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10);
+  dInfo("vgId:%d, wait for vnode ref become 0", pVnode->vgId);
-  while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10);
+  while (pVnode->refCount > 0) taosMsleep(10);
-  while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
+
+  dInfo("vgId:%d, wait for vnode write queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
+        pVnode->pWriteW.queue->threadId);
+  tMultiWorkerCleanup(&pVnode->pWriteW);
+
+  dInfo("vgId:%d, wait for vnode sync queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
+        pVnode->pSyncW.queue->threadId);
+  tMultiWorkerCleanup(&pVnode->pSyncW);
+
+  dInfo("vgId:%d, wait for vnode sync ctrl queue:%p is empty, thread:%08" PRId64, pVnode->vgId,
+        pVnode->pSyncCtrlW.queue, pVnode->pSyncCtrlW.queue->threadId);
+  tMultiWorkerCleanup(&pVnode->pSyncCtrlW);
+
+  dInfo("vgId:%d, wait for vnode apply queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
+        pVnode->pApplyW.queue->threadId);
+  tMultiWorkerCleanup(&pVnode->pApplyW);
+
+  dInfo("vgId:%d, wait for vnode query queue:%p is empty", pVnode->vgId, pVnode->pQueryQ);
   while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
+
+  dInfo("vgId:%d, wait for vnode fetch queue:%p is empty, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ,
+        pVnode->pFetchQ->threadId);
   while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
+
+  dInfo("vgId:%d, wait for vnode stream queue:%p is empty", pVnode->vgId, pVnode->pStreamQ);
   while (!taosQueueEmpty(pVnode->pStreamQ)) taosMsleep(10);
-  dTrace("vgId:%d, vnode queue is empty", pVnode->vgId);
+
+  dInfo("vgId:%d, all vnode queues is empty", pVnode->vgId);
 
   vmFreeQueue(pMgmt, pVnode);
   vnodeClose(pVnode->pImpl);
   pVnode->pImpl = NULL;
-  dDebug("vgId:%d, vnode is closed", pVnode->vgId);
+  dInfo("vgId:%d, vnode is closed", pVnode->vgId);
 
   if (pVnode->dropped) {
     dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped);

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -86,7 +86,12 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
   if (code != 0) {
     if (terrno != 0) code = terrno;
-    dGError("vgId:%d, msg:%p failed to process stream since %s", pVnode->vgId, pMsg, terrstr(code));
+    if (pMsg) {
+      dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType),
+              terrstr(code));
+    } else {
+      dGError("vgId:%d, msg:%p failed to process stream empty msg since %s", pVnode->vgId, pMsg, terrstr(code));
+    }
     vmSendRsp(pMsg, code);
   }
 
@@ -183,30 +188,20 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
         dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code));
       } else {
         dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg);
-        taosWriteQitem(pVnode->pWriteQ, pMsg);
+        taosWriteQitem(pVnode->pWriteW.queue, pMsg);
-#if 0  // tests for batch writes
-        if (pMsg->msgType == TDMT_VND_CREATE_TABLE) {
-          SRpcMsg *pDup = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
-          memcpy(pDup, pMsg, sizeof(SRpcMsg));
-          pDup->pCont = rpcMallocCont(pMsg->contLen);
-          memcpy(pDup->pCont, pMsg->pCont, pMsg->contLen);
-          pDup->info.handle = NULL;
-          taosWriteQitem(pVnode->pWriteQ, pDup);
-        }
-#endif
       }
       break;
     case SYNC_QUEUE:
       dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg);
-      taosWriteQitem(pVnode->pSyncQ, pMsg);
+      taosWriteQitem(pVnode->pSyncW.queue, pMsg);
       break;
     case SYNC_CTRL_QUEUE:
       dGTrace("vgId:%d, msg:%p put into vnode-sync-ctrl queue", pVnode->vgId, pMsg);
-      taosWriteQitem(pVnode->pSyncCtrlQ, pMsg);
+      taosWriteQitem(pVnode->pSyncCtrlW.queue, pMsg);
       break;
     case APPLY_QUEUE:
       dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg);
-      taosWriteQitem(pVnode->pApplyQ, pMsg);
+      taosWriteQitem(pVnode->pApplyW.queue, pMsg);
       break;
     default:
       code = -1;
@@ -218,7 +213,9 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
   return code;
 }
 
-int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_CTRL_QUEUE); }
+int32_t vmPutMsgToSyncCtrlQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
+  return vmPutMsgToQueue(pMgmt, pMsg, SYNC_CTRL_QUEUE);
+}
 
 int32_t vmPutMsgToSyncQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); }
 
@@ -269,13 +266,13 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
   if (pVnode != NULL) {
     switch (qtype) {
       case WRITE_QUEUE:
-        size = taosQueueItemSize(pVnode->pWriteQ);
+        size = taosQueueItemSize(pVnode->pWriteW.queue);
         break;
       case SYNC_QUEUE:
-        size = taosQueueItemSize(pVnode->pSyncQ);
+        size = taosQueueItemSize(pVnode->pSyncW.queue);
         break;
       case APPLY_QUEUE:
-        size = taosQueueItemSize(pVnode->pApplyQ);
+        size = taosQueueItemSize(pVnode->pApplyW.queue);
         break;
       case QUERY_QUEUE:
         size = taosQueueItemSize(pVnode->pQueryQ);
@@ -299,40 +296,44 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
 }
 
 int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
-  pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeProposeWriteMsg);
+  SMultiWorkerCfg wcfg = {.max = 1, .name = "vnode-write", .fp = (FItems)vnodeProposeWriteMsg, .param = pVnode->pImpl};
-  pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue);
+  SMultiWorkerCfg scfg = {.max = 1, .name = "vnode-sync", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
-  pVnode->pSyncCtrlQ = tWWorkerAllocQueue(&pMgmt->syncCtrlPool, pVnode, (FItems)vmProcessSyncQueue);
+  SMultiWorkerCfg sccfg = {.max = 1, .name = "vnode-sync-ctrl", .fp = (FItems)vmProcessSyncQueue, .param = pVnode};
-  pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->applyPool, pVnode->pImpl, (FItems)vnodeApplyWriteMsg);
+  SMultiWorkerCfg acfg = {.max = 1, .name = "vnode-apply", .fp = (FItems)vnodeApplyWriteMsg, .param = pVnode->pImpl};
+  (void)tMultiWorkerInit(&pVnode->pWriteW, &wcfg);
+  (void)tMultiWorkerInit(&pVnode->pSyncW, &scfg);
+  (void)tMultiWorkerInit(&pVnode->pSyncCtrlW, &sccfg);
+  (void)tMultiWorkerInit(&pVnode->pApplyW, &acfg);
+
   pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue);
   pVnode->pStreamQ = tQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue);
   pVnode->pFetchQ = tWWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItems)vmProcessFetchQueue);
 
-  if (pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pApplyQ == NULL || pVnode->pQueryQ == NULL ||
+  if (pVnode->pWriteW.queue == NULL || pVnode->pSyncW.queue == NULL || pVnode->pSyncCtrlW.queue == NULL ||
-      pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) {
+      pVnode->pApplyW.queue == NULL || pVnode->pQueryQ == NULL || pVnode->pStreamQ == NULL || pVnode->pFetchQ == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
 
-  dDebug("vgId:%d, write-queue:%p is alloced", pVnode->vgId, pVnode->pWriteQ);
+  dInfo("vgId:%d, write-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pWriteW.queue,
-  dDebug("vgId:%d, sync-queue:%p is alloced", pVnode->vgId, pVnode->pSyncQ);
+        pVnode->pWriteW.queue->threadId);
-  dDebug("vgId:%d, apply-queue:%p is alloced", pVnode->vgId, pVnode->pApplyQ);
+  dInfo("vgId:%d, sync-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncW.queue,
-  dDebug("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ);
+        pVnode->pSyncW.queue->threadId);
-  dDebug("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ);
+  dInfo("vgId:%d, sync-ctrl-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pSyncCtrlW.queue,
-  dDebug("vgId:%d, fetch-queue:%p is alloced", pVnode->vgId, pVnode->pFetchQ);
+        pVnode->pSyncCtrlW.queue->threadId);
+  dInfo("vgId:%d, apply-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pApplyW.queue,
+        pVnode->pApplyW.queue->threadId);
+  dInfo("vgId:%d, query-queue:%p is alloced", pVnode->vgId, pVnode->pQueryQ);
+  dInfo("vgId:%d, fetch-queue:%p is alloced, thread:%08" PRId64, pVnode->vgId, pVnode->pFetchQ,
+        pVnode->pFetchQ->threadId);
+  dInfo("vgId:%d, stream-queue:%p is alloced", pVnode->vgId, pVnode->pStreamQ);
   return 0;
 }
 
 void vmFreeQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
-  tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ);
-  tWWorkerFreeQueue(&pMgmt->applyPool, pVnode->pApplyQ);
-  tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
-  tWWorkerFreeQueue(&pMgmt->syncCtrlPool, pVnode->pSyncCtrlQ);
   tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ);
   tQWorkerFreeQueue(&pMgmt->streamPool, pVnode->pStreamQ);
   tWWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ);
-  pVnode->pWriteQ = NULL;
-  pVnode->pSyncQ = NULL;
-  pVnode->pApplyQ = NULL;
   pVnode->pQueryQ = NULL;
   pVnode->pStreamQ = NULL;
   pVnode->pFetchQ = NULL;
@@ -357,26 +358,6 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
   pFPool->max = tsNumOfVnodeFetchThreads;
   if (tWWorkerInit(pFPool) != 0) return -1;
 
-  SWWorkerPool *pWPool = &pMgmt->writePool;
-  pWPool->name = "vnode-write";
-  pWPool->max = tsNumOfVnodeWriteThreads;
-  if (tWWorkerInit(pWPool) != 0) return -1;
-
-  SWWorkerPool *pAPool = &pMgmt->applyPool;
-  pAPool->name = "vnode-apply";
-  pAPool->max = tsNumOfVnodeWriteThreads;
-  if (tWWorkerInit(pAPool) != 0) return -1;
-
-  SWWorkerPool *pSPool = &pMgmt->syncPool;
-  pSPool->name = "vnode-sync";
-  pSPool->max = tsNumOfVnodeSyncThreads;
-  if (tWWorkerInit(pSPool) != 0) return -1;
-
-  SWWorkerPool *pSCPool = &pMgmt->syncCtrlPool;
-  pSCPool->name = "vnode-sync-ctrl";
-  pSCPool->max = tsNumOfVnodeSyncThreads;
-  if (tWWorkerInit(pSCPool) != 0) return -1;
-
   SSingleWorkerCfg mgmtCfg = {
       .min = 1,
       .max = 1,
@@ -391,10 +372,6 @@ int32_t vmStartWorker(SVnodeMgmt *pMgmt) {
 }
 
 void vmStopWorker(SVnodeMgmt *pMgmt) {
-  tWWorkerCleanup(&pMgmt->writePool);
-  tWWorkerCleanup(&pMgmt->applyPool);
-  tWWorkerCleanup(&pMgmt->syncPool);
-  tWWorkerCleanup(&pMgmt->syncCtrlPool);
   tQWorkerCleanup(&pMgmt->queryPool);
   tQWorkerCleanup(&pMgmt->streamPool);
   tWWorkerCleanup(&pMgmt->fetchPool);

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -157,7 +157,11 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
 _OVER:
   if (code != 0) {
     if (terrno != 0) code = terrno;
-    dGTrace("msg:%p, failed to process since %s", pMsg, terrstr());
+    if (pMsg) {
+      dGTrace("msg:%p, failed to process %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr());
+    } else {
+      dGTrace("msg:%p, failed to process empty msg since %s", pMsg, terrstr());
+    }
 
     if (IsReq(pRpc)) {
       SRpcMsg rsp = {.code = code, .info = pRpc->info};

source/dnode/mnode/impl/src/mndDnode.c

@@ -345,6 +345,19 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
     }
   }
 
+  int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE);
+  int64_t curMs = taosGetTimestampMs();
+  bool    online = mndIsDnodeOnline(pDnode, curMs);
+  bool    dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer);
+  bool    reboot = (pDnode->rebootTime != statusReq.rebootTime);
+  bool    needCheck = !online || dnodeChanged || reboot;
+
+  pDnode->accessTimes++;
+  pDnode->lastAccessTime = curMs;
+  const STraceId *trace = &pReq->info.traceId;
+  mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id,
+         pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq);
+
   for (int32_t v = 0; v < taosArrayGetSize(statusReq.pVloads); ++v) {
     SVnodeLoad *pVload = taosArrayGet(statusReq.pVloads, v);
 
@@ -396,17 +409,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
     mndReleaseQnode(pMnode, pQnode);
   }
 
-  int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE);
-  int64_t curMs = taosGetTimestampMs();
-  bool    online = mndIsDnodeOnline(pDnode, curMs);
-  bool    dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer);
-  bool    reboot = (pDnode->rebootTime != statusReq.rebootTime);
-  bool    needCheck = !online || dnodeChanged || reboot;
-
-  pDnode->accessTimes++;
-  mTrace("dnode:%d, status received, access times:%d check:%d online:%d reboot:%d changed:%d", pDnode->id,
-         pDnode->accessTimes, needCheck, online, reboot, dnodeChanged);
-
   if (needCheck) {
     if (statusReq.sver != tsVersion) {
       if (pDnode != NULL) {
@@ -454,6 +456,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
     pDnode->memTotal = statusReq.memTotal;
 
     SStatusRsp statusRsp = {0};
+    statusRsp.statusSeq++;
     statusRsp.dnodeVer = dnodeVer;
     statusRsp.dnodeCfg.dnodeId = pDnode->id;
     statusRsp.dnodeCfg.clusterId = pMnode->clusterId;
@@ -474,8 +477,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
     pReq->info.rsp = pHead;
   }
 
-  pDnode->lastAccessTime = curMs;
-  pDnode->accessTimes++;
   code = 0;
 
 _OVER:

source/dnode/mnode/impl/src/mndStream.c

@@ -705,7 +705,8 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
     return -1;
   }
 
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-stream");
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "drop-stream");
+  mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetDb);
   if (pTrans == NULL) {
     mError("stream:%s, failed to drop since %s", dropReq.name, terrstr());
     sdbRelease(pMnode->pSdb, pStream);

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -612,14 +612,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
       mndReleaseSubscribe(pMnode, pSub);
     }
 
-    // TODO replace assert with error check
+    if (mndDoRebalance(pMnode, &rebInput, &rebOutput) < 0) {
-    ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0);
+      mError("mq rebalance internal error");
+    }
 
     // if add more consumer to balanced subscribe,
     // possibly no vg is changed
     /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/
 
-    // TODO replace assert with error check
     if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) {
       mError("mq rebalance persist rebalance output error, possibly vnode splitted or dropped");
     }

source/dnode/snode/src/snode.c

@@ -231,10 +231,10 @@ int32_t sndProcessTaskRetrieveReq(SSnode *pSnode, SRpcMsg *pMsg) {
 
 int32_t sndProcessTaskDispatchRsp(SSnode *pSnode, SRpcMsg *pMsg) {
   SStreamDispatchRsp *pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
-  int32_t             taskId = pRsp->taskId;
+  int32_t             taskId = ntohl(pRsp->upstreamTaskId);
   SStreamTask        *pTask = streamMetaGetTask(pSnode->pMeta, taskId);
   if (pTask) {
-    streamProcessDispatchRsp(pTask, pRsp);
+    streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
     return 0;
   } else {
     return -1;

source/dnode/vnode/inc/vnode.h

@@ -158,7 +158,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL
 void     tsdbReaderClose(STsdbReader *pReader);
 bool     tsdbNextDataBlock(STsdbReader *pReader);
 bool     tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid);
-void     tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow);
+void     tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow);
 int32_t  tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave);
 SArray  *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
 int32_t  tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);
@@ -240,7 +240,7 @@ bool    tqNextDataBlock(STqReader *pReader);
 bool    tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
 int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader);
 
-void vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg);
+int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg);
 
 // sma
 int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);

source/dnode/vnode/src/inc/tsdb.h

@@ -643,6 +643,7 @@ typedef struct SSttBlockLoadInfo {
   STSchema  *pSchema;
   int16_t   *colIds;
   int32_t    numOfCols;
+  bool       sttBlockLoaded;
 } SSttBlockLoadInfo;
 
 typedef struct SMergeTree {

source/dnode/vnode/src/inc/vnd.h

@@ -97,6 +97,7 @@ bool    vnodeShouldRollback(SVnode* pVnode);
 // vnodeSync.c
 int32_t vnodeSyncOpen(SVnode* pVnode, char* path);
 void    vnodeSyncStart(SVnode* pVnode);
+void    vnodeSyncPreClose(SVnode* pVnode);
 void    vnodeSyncClose(SVnode* pVnode);
 void    vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg);
 bool    vnodeIsLeader(SVnode* pVnode);

source/dnode/vnode/src/inc/vnodeInt.h

@@ -150,19 +150,19 @@ typedef struct {
 int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo);
 
 // tsdb
-int         tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback);
+int     tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback);
-int         tsdbClose(STsdb** pTsdb);
+int     tsdbClose(STsdb** pTsdb);
-int32_t     tsdbBegin(STsdb* pTsdb);
+int32_t tsdbBegin(STsdb* pTsdb);
-int32_t     tsdbCommit(STsdb* pTsdb);
+int32_t tsdbCommit(STsdb* pTsdb);
-int32_t     tsdbFinishCommit(STsdb* pTsdb);
+int32_t tsdbFinishCommit(STsdb* pTsdb);
-int32_t     tsdbRollbackCommit(STsdb* pTsdb);
+int32_t tsdbRollbackCommit(STsdb* pTsdb);
-int32_t     tsdbDoRetention(STsdb* pTsdb, int64_t now);
+int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now);
-int         tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
+int     tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
-int         tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
+int     tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
-int32_t     tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
+int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
-                                SSubmitBlkRsp* pRsp);
+                            SSubmitBlkRsp* pRsp);
-int32_t     tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
+int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
-int32_t     tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
+int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
 
 // tq
 int     tqInit();
@@ -183,13 +183,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg);
 // tq-stream
 int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen);
 int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen);
+int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg);
+int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen);
 int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver);
 int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver);
 int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec);
 int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg);
-// int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg);
-// int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg);

source/dnode/vnode/src/tq/tq.c

@@ -896,6 +896,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
   pTask->startVer = ver;
 
   // expand executor
+  if (pTask->fillHistory) {
+    pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
+  }
+
   if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
     pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1);
     if (pTask->pState == NULL) {
@@ -911,9 +915,6 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
     pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
     ASSERT(pTask->exec.executor);
 
-    if (pTask->fillHistory) {
-      pTask->taskStatus = TASK_STATUS__RECOVER_PREPARE;
-    }
   } else if (pTask->taskLevel == TASK_LEVEL__AGG) {
     pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1);
     if (pTask->pState == NULL) {
@@ -947,11 +948,90 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
 
   streamSetupTrigger(pTask);
 
-  tqInfo("expand stream task on vg %d, task id %d, child id %d", TD_VID(pTq->pVnode), pTask->taskId,
+  tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", TD_VID(pTq->pVnode), pTask->taskId,
-         pTask->selfChildId);
+         pTask->selfChildId, pTask->taskLevel);
   return 0;
 }
 
+int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
+  char*               msgStr = pMsg->pCont;
+  char*               msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
+  int32_t             msgLen = pMsg->contLen - sizeof(SMsgHead);
+  SStreamTaskCheckReq req;
+  SDecoder            decoder;
+  tDecoderInit(&decoder, msgBody, msgLen);
+  tDecodeSStreamTaskCheckReq(&decoder, &req);
+  tDecoderClear(&decoder);
+  int32_t             taskId = req.downstreamTaskId;
+  SStreamTaskCheckRsp rsp = {
+      .reqId = req.reqId,
+      .streamId = req.streamId,
+      .childId = req.childId,
+      .downstreamNodeId = req.downstreamNodeId,
+      .downstreamTaskId = req.downstreamTaskId,
+      .upstreamNodeId = req.upstreamNodeId,
+      .upstreamTaskId = req.upstreamTaskId,
+  };
+  SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId);
+  if (pTask && atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL) {
+    rsp.status = 1;
+  } else {
+    rsp.status = 0;
+  }
+
+  tqDebug("tq recv task check req(reqId: %" PRId64 ") %d at node %d check req from task %d at node %d, status %d",
+          rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
+
+  SEncoder encoder;
+  int32_t  code;
+  int32_t  len;
+  tEncodeSize(tEncodeSStreamTaskCheckRsp, &rsp, len, code);
+  if (code < 0) {
+    ASSERT(0);
+  }
+  void* buf = rpcMallocCont(sizeof(SMsgHead) + len);
+  ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId);
+
+  void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+  tEncoderInit(&encoder, (uint8_t*)abuf, len);
+  tEncodeSStreamTaskCheckRsp(&encoder, &rsp);
+  tEncoderClear(&encoder);
+
+  SRpcMsg rspMsg = {
+      .code = 0,
+      .pCont = buf,
+      .contLen = sizeof(SMsgHead) + len,
+      .info = pMsg->info,
+  };
+
+  tmsgSendRsp(&rspMsg);
+  return 0;
+}
+
+int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {
+  int32_t             code;
+  SStreamTaskCheckRsp rsp;
+
+  SDecoder decoder;
+  tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
+  code = tDecodeSStreamTaskCheckRsp(&decoder, &rsp);
+  if (code < 0) {
+    tDecoderClear(&decoder);
+    return -1;
+  }
+  tDecoderClear(&decoder);
+
+  tqDebug("tq recv task check rsp(reqId: %" PRId64 ") %d at node %d check req from task %d at node %d, status %d",
+          rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
+
+  SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, rsp.upstreamTaskId);
+  if (pTask == NULL) {
+    return -1;
+  }
+
+  return streamProcessTaskCheckRsp(pTask, &rsp, version);
+}
+
 int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) {
   int32_t code;
 #if 0
@@ -982,37 +1062,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg
 
   // 3.go through recover steps to fill history
   if (pTask->fillHistory) {
-    if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
+    streamTaskCheckDownstream(pTask, version);
-      streamSetParamForRecover(pTask);
-      streamSourceRecoverPrepareStep1(pTask, version);
-
-      SStreamRecoverStep1Req req;
-      streamBuildSourceRecover1Req(pTask, &req);
-      int32_t len = sizeof(SStreamRecoverStep1Req);
-
-      void* serializedReq = rpcMallocCont(len);
-      if (serializedReq == NULL) {
-        return -1;
-      }
-
-      memcpy(serializedReq, &req, len);
-
-      SRpcMsg rpcMsg = {
-          .contLen = len,
-          .pCont = serializedReq,
-          .msgType = TDMT_VND_STREAM_RECOVER_STEP1,
-      };
-
-      if (tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &rpcMsg) < 0) {
-        /*ASSERT(0);*/
-      }
-
-    } else if (pTask->taskLevel == TASK_LEVEL__AGG) {
-      streamSetParamForRecover(pTask);
-      streamAggRecoverPrepare(pTask);
-    } else if (pTask->taskLevel == TASK_LEVEL__SINK) {
-      // do nothing
-    }
   }
 
   return 0;
@@ -1268,7 +1318,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
     if (pIter == NULL) break;
     SStreamTask* pTask = *(SStreamTask**)pIter;
     if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue;
-    if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE) {
+    if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) {
       tqDebug("skip push task %d, task status %d", pTask->taskId, pTask->taskStatus);
       continue;
     }
@@ -1335,10 +1385,11 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) {
 
 int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
   SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
-  int32_t             taskId = pRsp->taskId;
+  int32_t             taskId = ntohl(pRsp->upstreamTaskId);
   SStreamTask*        pTask = streamMetaGetTask(pTq->pStreamMeta, taskId);
+  tqDebug("recv dispatch rsp, code: %x", pMsg->code);
   if (pTask) {
-    streamProcessDispatchRsp(pTask, pRsp);
+    streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
     return 0;
   } else {
     return -1;
@@ -1379,12 +1430,12 @@ int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) {
   return 0;
 }
 
-void vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) {
+int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) {
-  STQ*    pTq = pVnode->pTq;
+  STQ*      pTq = pVnode->pTq;
-  char*   msgStr = pMsg->pCont;
+  SMsgHead* msgStr = pMsg->pCont;
-  char*   msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
+  char*     msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
-  int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
+  int32_t   msgLen = pMsg->contLen - sizeof(SMsgHead);
-  int32_t code = 0;
+  int32_t   code = 0;
 
   SStreamDispatchReq req;
   SDecoder           decoder;
@@ -1407,16 +1458,45 @@ void vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) {
     streamProcessDispatchReq(pTask, &req, &rsp, false);
     rpcFreeCont(pMsg->pCont);
     taosFreeQitem(pMsg);
-    return;
+    return 0;
   }
 
+  code = TSDB_CODE_STREAM_TASK_NOT_EXIST;
+
 FAIL:
-  if (pMsg->info.handle == NULL) return;
+  if (pMsg->info.handle == NULL) return -1;
+
+  SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp));
+  if (pRspHead == NULL) {
+    SRpcMsg rsp = {
+        .code = TSDB_CODE_OUT_OF_MEMORY,
+        .info = pMsg->info,
+    };
+    tqDebug("send dispatch error rsp, code: %x", code);
+    tmsgSendRsp(&rsp);
+    rpcFreeCont(pMsg->pCont);
+    taosFreeQitem(pMsg);
+    return -1;
+  }
+
+  pRspHead->vgId = htonl(req.upstreamNodeId);
+  SStreamDispatchRsp* pRsp = POINTER_SHIFT(pRspHead, sizeof(SMsgHead));
+  pRsp->streamId = htobe64(req.streamId);
+  pRsp->upstreamTaskId = htonl(req.upstreamTaskId);
+  pRsp->upstreamNodeId = htonl(req.upstreamNodeId);
+  pRsp->downstreamNodeId = htonl(pVnode->config.vgId);
+  pRsp->downstreamTaskId = htonl(req.taskId);
+  pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL;
+
   SRpcMsg rsp = {
       .code = code,
       .info = pMsg->info,
+      .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp),
+      .pCont = pRspHead,
   };
+  tqDebug("send dispatch error rsp, code: %x", code);
   tmsgSendRsp(&rsp);
   rpcFreeCont(pMsg->pCont);
   taosFreeQitem(pMsg);
+  return -1;
 }

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -333,11 +333,12 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
     code = TSDB_CODE_INVALID_PARA;
   }
 
-  _end:
+_end:
   tsdbDataFReaderClose(&pr->pDataFReaderLast);
   tsdbDataFReaderClose(&pr->pDataFReader);
 
   tsdbUntakeReadSnap(pr->pVnode->pTsdb, pr->pReadSnap, "cache-l");
+  resetLastBlockLoadInfo(pr->pLoadInfo);
 
   for (int32_t j = 0; j < pr->numOfCols; ++j) {
     taosMemoryFree(pRes[j]);

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -116,6 +116,13 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
   if (info.suid) {
     metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info);
   }
+  if (pMsgIter->sversion != info.skmVer) {
+    tsdbError("vgId:%d, req sver:%d, skmVer:%d suid:%" PRId64 " uid:%" PRId64,
+             TD_VID(pTsdb->pVnode), pMsgIter->sversion, info.skmVer, suid, uid);
+    code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
+    goto _err;
+  }
+  
   pRsp->sver = info.skmVer;
 
   // create/get STbData to op
@@ -133,6 +140,7 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI
   return code;
 
 _err:
+  terrno = code;
   return code;
 }
 

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -72,6 +72,7 @@ void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
 
     pLoadInfo[i].elapsedTime = 0;
     pLoadInfo[i].loadBlocks = 0;
+    pLoadInfo[i].sttBlockLoaded = false;
   }
 }
 
@@ -278,9 +279,9 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
 
   (*pIter)->pBlockLoadInfo = pBlockLoadInfo;
 
-  size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
+  if (!pBlockLoadInfo->sttBlockLoaded) {
-  if (size == 0) {
     int64_t st = taosGetTimestampUs();
+    pBlockLoadInfo->sttBlockLoaded = true;
 
     code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk);
     if (code) {
@@ -288,7 +289,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
     }
 
     // only apply to the child tables, ordinary tables will not incur this filter procedure.
-    size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
+    size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
 
     if (size >= 1) {
       SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0);
@@ -296,10 +297,10 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
 
       // all identical
       if (pStart->suid == pEnd->suid) {
-        if (pStart->suid == suid) {
+        if (pStart->suid != suid) {
-          // do nothing
-        } else if (pStart->suid != suid) {
           // no qualified stt block existed
+          taosArrayClear(pBlockLoadInfo->aSttBlk);
+
           (*pIter)->iSttBlk = -1;
           double el = (taosGetTimestampUs() - st) / 1000.0;
           tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
@@ -330,7 +331,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
     tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr);
   }
 
-  size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
+  size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
 
   // find the start block
   (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -185,11 +185,11 @@ static int32_t  doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STabl
                                        SRowMerger* pMerger, SVersionRange* pVerRange);
 static int32_t  doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger,
                                  STsdbReader* pReader);
-static int32_t  doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid);
+static int32_t  doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pInfo);
 static int32_t  doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,
                                          int32_t rowIndex);
 static void     setComposedBlockFlag(STsdbReader* pReader, bool composed);
-static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pRange);
+static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pVerRange);
 
 static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList,
                                         STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow);
@@ -208,7 +208,6 @@ static bool          hasDataInLastBlock(SLastBlockReader* pLastBlockReader);
 static int32_t       doBuildDataBlock(STsdbReader* pReader);
 static TSDBKEY       getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader);
 static bool          hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo);
-static bool          hasDataInLastBlock(SLastBlockReader* pLastBlockReader);
 
 static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
 
@@ -534,7 +533,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity)
   }
 
   for (int32_t i = 0; i < pCond->numOfCols; ++i) {
-    SColumnInfoData colInfo = {{0}, 0};
+    SColumnInfoData colInfo = {0, {0}};
     colInfo.info = pCond->colList[i];
     blockDataAppendColInfo(pResBlock, &colInfo);
   }
@@ -1529,8 +1528,8 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB
   // opt version
   // 1. it is not a border point
   // 2. the direct next point is not an duplicated timestamp
-  if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && pReader->order == TSDB_ORDER_ASC) ||
+  bool asc = (pReader->order == TSDB_ORDER_ASC);
-      (pDumpInfo->rowIndex > 0 && pReader->order == TSDB_ORDER_DESC)) {
+  if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) {
     int32_t step = pReader->order == TSDB_ORDER_ASC ? 1 : -1;
 
     int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step];
@@ -1749,7 +1748,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
     return code;
   }
 
-  doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
+  doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
 
   taosMemoryFree(pTSRow);
   tRowMergerClear(&merge);
@@ -1770,6 +1769,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
   // only last block exists
   if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) {
     if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) {
+      pBlockScanInfo->lastKey = tsLastBlock;
       return TSDB_CODE_SUCCESS;
     } else {
       int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema);
@@ -1786,7 +1786,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
         return code;
       }
 
-      doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
+      doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
 
       taosMemoryFree(pTSRow);
       tRowMergerClear(&merge);
@@ -1810,7 +1810,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader,
       return code;
     }
 
-    doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
+    doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
 
     taosMemoryFree(pTSRow);
     tRowMergerClear(&merge);
@@ -1858,7 +1858,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader
           return code;
         }
 
-        doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
+        doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
 
         taosMemoryFree(pTSRow);
         tRowMergerClear(&merge);
@@ -2082,7 +2082,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
     return code;
   }
 
-  doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
+  doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
 
   taosMemoryFree(pTSRow);
   tRowMergerClear(&merge);
@@ -2186,7 +2186,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum
 static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
   // the last block reader has been initialized for this table.
   if (pLBlockReader->uid == pScanInfo->uid) {
-    return true;
+    return hasDataInLastBlock(pLBlockReader);
   }
 
   if (pLBlockReader->uid != 0) {
@@ -2233,6 +2233,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc
                               STsdbReader* pReader) {
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
   if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) {
+    pBlockScanInfo->lastKey = key;
     return TSDB_CODE_SUCCESS;
   } else {
     TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
@@ -2251,7 +2252,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc
       return code;
     }
 
-    doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid);
+    doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo);
 
     taosMemoryFree(pTSRow);
     tRowMergerClear(&merge);
@@ -2299,26 +2300,32 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
   SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
   SLastBlockReader*   pLastBlockReader = pReader->status.fileIter.pLastBlockReader;
 
+  bool    asc = ASCENDING_TRAVERSE(pReader->order);
   int64_t st = taosGetTimestampUs();
+  int32_t step = asc ? 1 : -1;
 
   STableBlockScanInfo* pBlockScanInfo = NULL;
   if (pBlockInfo != NULL) {
-    pBlockScanInfo = *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
+    void* p = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid));
-    if (pBlockScanInfo == NULL) {
+    if (p == NULL) {
       code = TSDB_CODE_INVALID_PARA;
       tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid,
                 taosHashGetSize(pReader->status.pTableMap), pReader->idStr);
       goto _end;
     }
 
+    pBlockScanInfo = *(STableBlockScanInfo**) p;
+
     SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter);
     TSDBKEY   keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader);
 
     // it is a clean block, load it directly
     if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) {
-      if (pReader->order == TSDB_ORDER_ASC ||
+      if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) {
-          (pReader->order == TSDB_ORDER_DESC && (!hasDataInLastBlock(pLastBlockReader)))) {
         copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
+
+        // record the last key value
+        pBlockScanInfo->lastKey = asc? pBlock->maxKey.ts:pBlock->minKey.ts;
         goto _end;
       }
     }
@@ -2328,7 +2335,6 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
 
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
   SBlockData*         pBlockData = &pReader->status.fileBlockData;
-  int32_t             step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1;
 
   while (1) {
     bool hasBlockData = false;
@@ -2710,6 +2716,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
       pInfo->window = (STimeWindow){.skey = pBlock->minKey.ts, .ekey = pBlock->maxKey.ts};
       setComposedBlockFlag(pReader, false);
       setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order);
+
+      // update the last key for the corresponding table
+      pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order)? pInfo->window.ekey:pInfo->window.skey;
     }
   }
 
@@ -3214,7 +3223,6 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc
 
 int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts,
                                SRowMerger* pMerger, SVersionRange* pVerRange) {
-  pScanInfo->lastKey = ts;
   while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) {
     int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader);
     if (next1 == ts) {
@@ -3407,9 +3415,10 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid) {
+int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, STableBlockScanInfo* pScanInfo) {
   int32_t numOfRows = pBlock->info.rows;
   int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
+  int64_t uid = pScanInfo->uid;
 
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
   STSchema*           pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid);
@@ -3448,6 +3457,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
   }
 
   pBlock->info.rows += 1;
+  pScanInfo->lastKey = pTSRow->ts;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -3511,7 +3521,8 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
       break;
     }
 
-    doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo->uid);
+    doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo);
+
     if (freeTSRow) {
       taosMemoryFree(pTSRow);
     }

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -242,12 +242,7 @@ _err:
   return NULL;
 }
 
-void vnodePreClose(SVnode *pVnode) {
+void vnodePreClose(SVnode *pVnode) { vnodeSyncPreClose(pVnode); }
-  if (pVnode) {
-    syncLeaderTransfer(pVnode->sync);
-    syncPreStop(pVnode->sync);
-  }
-}
 
 void vnodeClose(SVnode *pVnode) {
   if (pVnode) {

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -229,8 +229,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
       break;
     /* TQ */
     case TDMT_VND_TMQ_SUBSCRIBE:
-      if (tqProcessSubscribeReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
+      if (tqProcessSubscribeReq(pVnode->pTq, version, pReq, len) < 0) {
-                                pMsg->contLen - sizeof(SMsgHead)) < 0) {
         goto _err;
       }
       break;
@@ -240,26 +239,22 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
       }
       break;
     case TDMT_VND_TMQ_COMMIT_OFFSET:
-      if (tqProcessOffsetCommitReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
+      if (tqProcessOffsetCommitReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) {
-                                   pMsg->contLen - sizeof(SMsgHead)) < 0) {
         goto _err;
       }
       break;
     case TDMT_VND_TMQ_ADD_CHECKINFO:
-      if (tqProcessAddCheckInfoReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
+      if (tqProcessAddCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) {
-                                   pMsg->contLen - sizeof(SMsgHead)) < 0) {
         goto _err;
       }
       break;
     case TDMT_VND_TMQ_DEL_CHECKINFO:
-      if (tqProcessDelCheckInfoReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
+      if (tqProcessDelCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) {
-                                   pMsg->contLen - sizeof(SMsgHead)) < 0) {
         goto _err;
       }
       break;
     case TDMT_STREAM_TASK_DEPLOY: {
-      if (tqProcessTaskDeployReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
+      if (tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) {
-                                 pMsg->contLen - sizeof(SMsgHead)) < 0) {
         goto _err;
       }
     } break;
@@ -273,6 +268,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
         goto _err;
       }
     } break;
+    case TDMT_STREAM_TASK_CHECK_RSP: {
+      if (tqProcessStreamTaskCheckRsp(pVnode->pTq, version, pReq, len) < 0) {
+        goto _err;
+      }
+    } break;
     case TDMT_VND_ALTER_CONFIRM:
       vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp);
       break;
@@ -390,10 +390,12 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
       return tqProcessPollReq(pVnode->pTq, pMsg);
     case TDMT_STREAM_TASK_RUN:
       return tqProcessTaskRunReq(pVnode->pTq, pMsg);
-#if 0
+#if 1
     case TDMT_STREAM_TASK_DISPATCH:
       return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true);
 #endif
+    case TDMT_STREAM_TASK_CHECK:
+      return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg);
     case TDMT_STREAM_TASK_DISPATCH_RSP:
       return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg);
     case TDMT_STREAM_RETRIEVE:
@@ -863,6 +865,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
   SEncoder       encoder = {0};
   SArray        *newTbUids = NULL;
   SVStatis       statis = {0};
+  bool           tbCreated = false;
   terrno = TSDB_CODE_SUCCESS;
 
   pRsp->code = 0;
@@ -896,11 +899,10 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
     if (pBlock == NULL) break;
 
     SSubmitBlkRsp submitBlkRsp = {0};
+    tbCreated = false;
 
     // create table for auto create table mode
     if (msgIter.schemaLen > 0) {
-      submitBlkRsp.hashMeta = 1;
-
       tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen);
       if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) {
         pRsp->code = TSDB_CODE_INVALID_MSG;
@@ -937,11 +939,12 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
         }
 
         taosArrayPush(newTbUids, &createTbReq.uid);
-      }
 
-      submitBlkRsp.uid = createTbReq.uid;
+        submitBlkRsp.uid = createTbReq.uid;
-      submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2);
+        submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2);
-      sprintf(submitBlkRsp.tblFName, "%s.%s", pVnode->config.dbname, createTbReq.name);
+        sprintf(submitBlkRsp.tblFName, "%s.%s", pVnode->config.dbname, createTbReq.name);
+        tbCreated = true;
+      }
       
       msgIter.uid = createTbReq.uid;
       if (createTbReq.type == TSDB_CHILD_TABLE) {
@@ -955,9 +958,6 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
 #endif
       tDecoderClear(&decoder);
       taosArrayDestroy(createTbReq.ctb.tagName);
-    } else {
-      submitBlkRsp.tblFName = taosMemoryMalloc(TSDB_TABLE_FNAME_LEN);
-      sprintf(submitBlkRsp.tblFName, "%s.", pVnode->config.dbname);
     } 
 
     if (tsdbInsertTableData(pVnode->pTsdb, version, &msgIter, pBlock, &submitBlkRsp) < 0) {
@@ -966,7 +966,9 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
 
     submitRsp.numOfRows += submitBlkRsp.numOfRows;
     submitRsp.affectedRows += submitBlkRsp.affectedRows;
-    taosArrayPush(submitRsp.pArray, &submitBlkRsp);
+    if (tbCreated || submitBlkRsp.code) {
+      taosArrayPush(submitRsp.pArray, &submitBlkRsp);
+    }
   }
 
   if (taosArrayGetSize(newTbUids) > 0) {

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -342,52 +342,26 @@ static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, const SRpcMsg *pMsg, cons
          TMSG_INFO(pMsg->msgType));
 }
 
-#define USE_TSDB_SNAPSHOT
-
 static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) {
-#ifdef USE_TSDB_SNAPSHOT
   SVnode         *pVnode = pFsm->data;
   SSnapshotParam *pSnapshotParam = pParam;
   int32_t code = vnodeSnapReaderOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapReader **)ppReader);
   return code;
-#else
-  *ppReader = taosMemoryMalloc(32);
-  return 0;
-#endif
 }
 
 static int32_t vnodeSnapshotStopRead(const SSyncFSM *pFsm, void *pReader) {
-#ifdef USE_TSDB_SNAPSHOT
   SVnode *pVnode = pFsm->data;
   int32_t code = vnodeSnapReaderClose(pReader);
   return code;
-#else
-  taosMemoryFree(pReader);
-  return 0;
-#endif
 }
 
 static int32_t vnodeSnapshotDoRead(const SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) {
-#ifdef USE_TSDB_SNAPSHOT
   SVnode *pVnode = pFsm->data;
   int32_t code = vnodeSnapRead(pReader, (uint8_t **)ppBuf, len);
   return code;
-#else
-  static int32_t times = 0;
-  if (times++ < 5) {
-    *len = 64;
-    *ppBuf = taosMemoryMalloc(*len);
-    snprintf(*ppBuf, *len, "snapshot block %d", times);
-  } else {
-    *len = 0;
-    *ppBuf = NULL;
-  }
-  return 0;
-#endif
 }
 
 static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void **ppWriter) {
-#ifdef USE_TSDB_SNAPSHOT
   SVnode         *pVnode = pFsm->data;
   SSnapshotParam *pSnapshotParam = pParam;
 
@@ -404,14 +378,9 @@ static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void
 
   int32_t code = vnodeSnapWriterOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapWriter **)ppWriter);
   return code;
-#else
-  *ppWriter = taosMemoryMalloc(32);
-  return 0;
-#endif
 }
 
 static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) {
-#ifdef USE_TSDB_SNAPSHOT
   SVnode *pVnode = pFsm->data;
   vInfo("vgId:%d, stop write vnode snapshot, apply:%d, index:%" PRId64 " term:%" PRIu64 " config:%" PRId64,
         pVnode->config.vgId, isApply, pSnapshot->lastApplyIndex, pSnapshot->lastApplyTerm, pSnapshot->lastConfigIndex);
@@ -419,22 +388,14 @@ static int32_t vnodeSnapshotStopWrite(const SSyncFSM *pFsm, void *pWriter, bool
   int32_t code = vnodeSnapWriterClose(pWriter, !isApply, pSnapshot);
   vInfo("vgId:%d, apply vnode snapshot finished, code:0x%x", pVnode->config.vgId, code);
   return code;
-#else
-  taosMemoryFree(pWriter);
-  return 0;
-#endif
 }
 
 static int32_t vnodeSnapshotDoWrite(const SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) {
-#ifdef USE_TSDB_SNAPSHOT
   SVnode *pVnode = pFsm->data;
   vDebug("vgId:%d, continue write vnode snapshot, len:%d", pVnode->config.vgId, len);
   int32_t code = vnodeSnapWrite(pWriter, pBuf, len);
   vDebug("vgId:%d, continue write vnode snapshot finished, len:%d", pVnode->config.vgId, len);
   return code;
-#else
-  return 0;
-#endif
 }
 
 static void vnodeRestoreFinish(const SSyncFSM *pFsm) {
@@ -461,7 +422,6 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) {
   SVnode *pVnode = pFsm->data;
   vDebug("vgId:%d, become follower", pVnode->config.vgId);
 
-  // clear old leader resource
   taosThreadMutexLock(&pVnode->lock);
   if (pVnode->blocked) {
     pVnode->blocked = false;
@@ -474,15 +434,6 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) {
 static void vnodeBecomeLeader(const SSyncFSM *pFsm) {
   SVnode *pVnode = pFsm->data;
   vDebug("vgId:%d, become leader", pVnode->config.vgId);
-
-#if 0
-  taosThreadMutexLock(&pVnode->lock);
-  if (pVnode->blocked) {
-    pVnode->blocked = false;
-    tsem_post(&pVnode->syncSem);
-  }
-  taosThreadMutexUnlock(&pVnode->lock);
-#endif
 }
 
 static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
@@ -543,12 +494,25 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
 }
 
 void vnodeSyncStart(SVnode *pVnode) {
-  vDebug("vgId:%d, start sync", pVnode->config.vgId);
+  vInfo("vgId:%d, start sync", pVnode->config.vgId);
   syncStart(pVnode->sync);
 }
 
+void vnodeSyncPreClose(SVnode *pVnode) {
+  vInfo("vgId:%d, pre close sync", pVnode->config.vgId);
+  syncLeaderTransfer(pVnode->sync);
+  syncPreStop(pVnode->sync);
+  taosThreadMutexLock(&pVnode->lock);
+  if (pVnode->blocked) {
+    vInfo("vgId:%d, post block after close sync", pVnode->config.vgId);
+    pVnode->blocked = false;
+    tsem_post(&pVnode->syncSem);
+  }
+  taosThreadMutexUnlock(&pVnode->lock);
+}
+
 void vnodeSyncClose(SVnode *pVnode) {
-  vDebug("vgId:%d, close sync", pVnode->config.vgId);
+  vInfo("vgId:%d, close sync", pVnode->config.vgId);
   syncStop(pVnode->sync);
 }
 

source/libs/catalog/inc/catalogInt.h

@@ -33,6 +33,7 @@ extern "C" {
 #define CTG_DEFAULT_MAX_RETRY_TIMES      3
 #define CTG_DEFAULT_BATCH_NUM            64
 #define CTG_DEFAULT_FETCH_NUM            8
+#define CTG_MAX_COMMAND_LEN              512
 
 #define CTG_RENT_SLOT_SECOND 1.5
 
@@ -223,6 +224,7 @@ typedef struct SCtgUserAuth {
 
 typedef struct SCatalog {
   uint64_t     clusterId;
+  bool         stopUpdate;
   SHashObj*    userCache;  // key:user, value:SCtgUserAuth
   SHashObj*    dbCache;    // key:dbname, value:SCtgDBCache
   SCtgRentMgmt dbRent;
@@ -671,7 +673,7 @@ void    ctgdShowClusterCache(SCatalog* pCtg);
 int32_t ctgdShowCacheInfo(void);
 
 int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq);
-int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
+int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
 int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx,
                                int32_t* fetchIdx, int32_t baseResIdx, SArray* pList);
 
@@ -786,6 +788,7 @@ void    ctgFreeTbCacheImpl(SCtgTbCache* pCache);
 int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName);
 int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists);
 SName*  ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch);
+int32_t ctgdGetOneHandle(SCatalog **pHandle);
 
 extern SCatalogMgmt gCtgMgmt;
 extern SCtgDebug    gCTGDebug;

source/libs/catalog/src/catalog.c

@@ -202,7 +202,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx
   int32_t           code = 0;
   STableMetaOutput* output = NULL;
 
-  CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, pConn, ctx, pTableMeta));
+  CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, ctx, pTableMeta));
   if (*pTableMeta || (ctx->flag & CTG_FLAG_ONLY_CACHE)) {
     goto _return;
   }
@@ -959,14 +959,14 @@ int32_t catalogGetTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName
   CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
 }
 
-int32_t catalogGetCachedTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta) {
+int32_t catalogGetCachedTableMeta(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta) {
   CTG_API_ENTER();
 
   SCtgTbMetaCtx ctx = {0};
   ctx.pName = (SName*)pTableName;
   ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_ONLY_CACHE;
 
-  CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
+  CTG_API_LEAVE(ctgGetTbMeta(pCtg, NULL, &ctx, pTableMeta));
 }
 
 
@@ -981,15 +981,14 @@ int32_t catalogGetSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SNam
   CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
 }
 
-int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
+int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName,          STableMeta** pTableMeta) {
-                            STableMeta** pTableMeta) {
   CTG_API_ENTER();
 
   SCtgTbMetaCtx ctx = {0};
   ctx.pName = (SName*)pTableName;
   ctx.flag = CTG_FLAG_STB | CTG_FLAG_ONLY_CACHE;
 
-  CTG_API_LEAVE(ctgGetTbMeta(pCtg, pConn, &ctx, pTableMeta));
+  CTG_API_LEAVE(ctgGetTbMeta(pCtg, NULL, &ctx, pTableMeta));
 }
 
 
@@ -1114,11 +1113,10 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const
   CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup, NULL));
 }
 
-int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName,
+int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName,           SVgroupInfo* pVgroup, bool* exists) {
-                                  SVgroupInfo* pVgroup, bool* exists) {
   CTG_API_ENTER();
 
-  CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, pConn, pTableName, pVgroup, exists));
+  CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, NULL, pTableName, pVgroup, exists));
 }
 
 #if 0
@@ -1387,16 +1385,16 @@ _return:
   CTG_API_LEAVE(code);
 }
 
-int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type,
+int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type,
                                         bool* pass, bool* exists) {
   CTG_API_ENTER();
 
-  if (NULL == pCtg || NULL == pConn || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) {
+  if (NULL == pCtg || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) {
     CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
   }
 
   int32_t code = 0;
-  CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass, exists));
+  CTG_ERR_JRET(ctgChkAuth(pCtg, NULL, user, dbFName, type, pass, exists));
 
 _return:
 

source/libs/catalog/src/ctgAsync.c

@@ -1204,11 +1204,15 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu
           stbCtx.flag = flag;
           stbCtx.pName = &stbName;
 
-          taosMemoryFreeClear(pOut->tbMeta);
+          STableMeta *stbMeta = NULL;
-          CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta));
+          ctgReadTbMetaFromCache(pCtg, &stbCtx, &stbMeta);
-          if (pOut->tbMeta) {
+          if (stbMeta && stbMeta->sversion >= pOut->tbMeta->sversion) {
             ctgDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName));
             exist = 1;
+          } else {
+            ctgDebug("need to get/update stb meta, tbName:%s", tNameGetTableName(pName));
+            taosMemoryFreeClear(pOut->tbMeta);
+            taosMemoryFreeClear(stbMeta);
           }
         }
 
@@ -1641,7 +1645,7 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask* pTask) {
     pMsgCtx->pBatchs = pJob->pBatchs;
   }
 
-  CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, pConn, (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res));
+  CTG_ERR_RET(ctgGetTbMetaFromCache(pCtg, (SCtgTbMetaCtx*)pTask->taskCtx, (STableMeta**)&pTask->res));
   if (pTask->res) {
     CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
     return TSDB_CODE_SUCCESS;

source/libs/catalog/src/ctgCache.c

@@ -248,6 +248,8 @@ int32_t ctgAcquireStbMetaFromCache(SCatalog *pCtg, char *dbFName, uint64_t suid,
     goto _return;
   }
 
+  taosHashRelease(dbCache->stbCache, stName);
+
   CTG_LOCK(CTG_READ, &pCache->metaLock);
   if (NULL == pCache->pMeta) {
     ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName);
@@ -1550,7 +1552,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
   char            *dbFName = msg->dbFName;
   SCatalog        *pCtg = msg->pCtg;
 
-  if (NULL == dbInfo->vgHash) {
+  if (pCtg->stopUpdate || NULL == dbInfo->vgHash) {
     goto _return;
   }
 
@@ -1620,6 +1622,10 @@ int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) {
   SCtgDropDBMsg *msg = operation->data;
   SCatalog      *pCtg = msg->pCtg;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   SCtgDBCache *dbCache = NULL;
   ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache);
   if (NULL == dbCache) {
@@ -1646,6 +1652,10 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) {
   SCtgDropDbVgroupMsg *msg = operation->data;
   SCatalog            *pCtg = msg->pCtg;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   SCtgDBCache *dbCache = NULL;
   ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache);
   if (NULL == dbCache) {
@@ -1675,6 +1685,10 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) {
   STableMetaOutput    *pMeta = msg->pMeta;
   SCtgDBCache         *dbCache = NULL;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+  
   if ((!CTG_IS_META_CTABLE(pMeta->metaType)) && NULL == pMeta->tbMeta) {
     ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", pMeta->dbFName, pMeta->tbName);
     CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
@@ -1723,6 +1737,10 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) {
   SCtgDropStbMetaMsg *msg = operation->data;
   SCatalog           *pCtg = msg->pCtg;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   SCtgDBCache *dbCache = NULL;
   ctgGetDBCache(pCtg, msg->dbFName, &dbCache);
   if (NULL == dbCache) {
@@ -1776,6 +1794,10 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) {
   SCtgDropTblMetaMsg *msg = operation->data;
   SCatalog           *pCtg = msg->pCtg;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   SCtgDBCache *dbCache = NULL;
   ctgGetDBCache(pCtg, msg->dbFName, &dbCache);
   if (NULL == dbCache) {
@@ -1819,6 +1841,10 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) {
   SCtgUpdateUserMsg *msg = operation->data;
   SCatalog          *pCtg = msg->pCtg;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user));
   if (NULL == pUser) {
     SCtgUserAuth userAuth = {0};
@@ -1872,8 +1898,12 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) {
   int32_t             code = 0;
   SCtgUpdateEpsetMsg *msg = operation->data;
   SCatalog           *pCtg = msg->pCtg;
-
   SCtgDBCache *dbCache = NULL;
+
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache));
   if (NULL == dbCache) {
     ctgDebug("db %s not exist, ignore epset update", msg->dbFName);
@@ -1920,6 +1950,10 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) {
   STableIndex          *pIndex = msg->pIndex;
   SCtgDBCache          *dbCache = NULL;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache));
 
   CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex));
@@ -1942,6 +1976,10 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation) {
   SCatalog           *pCtg = msg->pCtg;
   SCtgDBCache        *dbCache = NULL;
 
+  if (pCtg->stopUpdate) {
+    goto _return;
+  }
+
   CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache));
   if (NULL == dbCache) {
     return TSDB_CODE_SUCCESS;
@@ -2154,7 +2192,7 @@ int32_t ctgStartUpdateThread() {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t ctgGetTbMetaFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) {
+int32_t ctgGetTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) {
   if (IS_SYS_DBNAME(ctx->pName->dbname)) {
     CTG_FLAG_SET_SYS_DB(ctx->flag);
   }

source/libs/catalog/src/ctgDbg.c

@@ -226,28 +226,45 @@ _return:
   CTG_RET(code);
 }
 
-int32_t ctgdEnableDebug(char *option) {
+int32_t ctgdEnableDebug(char *option, bool enable) {
   if (0 == strcasecmp(option, "lock")) {
-    gCTGDebug.lockEnable = true;
+    gCTGDebug.lockEnable = enable;
-    qDebug("lock debug enabled");
+    qDebug("catalog lock debug set to %d", enable);
     return TSDB_CODE_SUCCESS;
   }
 
   if (0 == strcasecmp(option, "cache")) {
-    gCTGDebug.cacheEnable = true;
+    gCTGDebug.cacheEnable = enable;
-    qDebug("cache debug enabled");
+    qDebug("catalog cache debug set to %d", enable);
     return TSDB_CODE_SUCCESS;
   }
 
   if (0 == strcasecmp(option, "api")) {
-    gCTGDebug.apiEnable = true;
+    gCTGDebug.apiEnable = enable;
-    qDebug("api debug enabled");
+    qDebug("catalog api debug set to %d", enable);
     return TSDB_CODE_SUCCESS;
   }
 
   if (0 == strcasecmp(option, "meta")) {
-    gCTGDebug.metaEnable = true;
+    gCTGDebug.metaEnable = enable;
-    qDebug("api debug enabled");
+    qDebug("catalog meta debug set to %d", enable);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (0 == strcasecmp(option, "stopUpdate")) {
+    SCatalog *pCtg = NULL;
+    
+    void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
+    while (pIter) {
+      pCtg = *(SCatalog **)pIter;
+
+      pCtg->stopUpdate = enable;
+      
+      pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
+    }
+    
+    qDebug("catalog stopUpdate set to %d", enable);
+    
     return TSDB_CODE_SUCCESS;
   }
 
@@ -256,6 +273,77 @@ int32_t ctgdEnableDebug(char *option) {
   return TSDB_CODE_CTG_INTERNAL_ERROR;
 }
 
+int32_t ctgdHandleDbgCommand(char *command) {
+  if (NULL == command) {
+    CTG_RET(TSDB_CODE_INVALID_PARA);
+  }
+
+  if (strlen(command) > CTG_MAX_COMMAND_LEN) {
+    CTG_RET(TSDB_CODE_INVALID_PARA);
+  }
+
+  char *dup = strdup(command);
+  char *option = NULL;
+  char *param = NULL;
+  
+  int32_t i = 0;
+  bool newItem = true;
+  while (*(dup + i)) {
+    if (isspace(*(dup + i))) {
+      *(dup + i) = 0;
+      ++i;
+      newItem = true;
+      continue;
+    }
+
+    if (!newItem) {
+      ++i;
+      continue;
+    }
+    
+    newItem = false;
+    if (NULL == option) {
+      option = dup + i;
+      ++i;
+      continue;
+    }
+
+    if (NULL == param) {
+      param = dup + i;
+      ++i;
+      continue;
+    }
+
+    taosMemoryFree(dup);
+    CTG_RET(TSDB_CODE_INVALID_PARA);
+  }
+
+  bool enable = atoi(param);
+
+  int32_t code = ctgdEnableDebug(option, enable);
+
+  taosMemoryFree(dup);
+
+  CTG_RET(code);
+}
+
+int32_t ctgdGetOneHandle(SCatalog **pHandle) {
+  SCatalog *pCtg = NULL;
+
+  void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL);
+  while (pIter) {
+    pCtg = *(SCatalog **)pIter;
+
+    taosHashCancelIterate(gCtgMgmt.pCluster, pIter);
+    break;
+  }
+
+  *pHandle = pCtg;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 int32_t ctgdGetStatNum(char *option, void *res) {
   if (0 == strcasecmp(option, "runtime.numOfOpDequeue")) {
     *(uint64_t *)res = atomic_load_64(&gCtgMgmt.stat.runtime.numOfOpDequeue);

source/libs/catalog/test/catalogTests.cpp

@@ -41,7 +41,6 @@
 namespace {
 
 extern "C" int32_t ctgdGetClusterCacheNum(struct SCatalog *pCatalog, int32_t type);
-extern "C" int32_t ctgdEnableDebug(char *option);
 extern "C" int32_t ctgdGetStatNum(char *option, void *res);
 
 void ctgTestSetRspTableMeta();
@@ -49,6 +48,8 @@ void ctgTestSetRspCTableMeta();
 void ctgTestSetRspSTableMeta();
 void ctgTestSetRspMultiSTableMeta();
 
+extern int32_t  clientConnRefPool;
+
 enum {
   CTGT_RSP_VGINFO = 1,
   CTGT_RSP_TBMETA,
@@ -151,10 +152,10 @@ void ctgTestInitLogFile() {
   qDebugFlag = 159;
   strcpy(tsLogDir, TD_LOG_DIR_PATH);
 
-  ctgdEnableDebug("api");
+  ctgdEnableDebug("api", true);
-  ctgdEnableDebug("meta");
+  ctgdEnableDebug("meta", true);
-  ctgdEnableDebug("cache");
+  ctgdEnableDebug("cache", true);
-  ctgdEnableDebug("lock");
+  ctgdEnableDebug("lock", true);
 
   if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
     printf("failed to open log file in directory:%s\n", tsLogDir);
@@ -1204,6 +1205,34 @@ void *ctgTestSetCtableMetaThread(void *param) {
 }
 
 
+void ctgTestFetchRows(TAOS_RES *result, int32_t *rows) {
+  TAOS_ROW    row;
+  int         num_fields = taos_num_fields(result);
+  TAOS_FIELD *fields = taos_fetch_fields(result);
+  char        temp[256];
+
+  // fetch the records row by row
+  while ((row = taos_fetch_row(result))) {
+    (*rows)++;
+    memset(temp, 0, sizeof(temp));
+    taos_print_row(temp, row, fields, num_fields);
+    printf("\t[%s]\n", temp);
+  }
+}
+
+void ctgTestExecQuery(TAOS    * taos, char* sql, bool fetch, int32_t *rows) {
+  TAOS_RES *result = taos_query(taos, sql);
+  int code = taos_errno(result);
+  ASSERT_EQ(code, 0);
+
+  if (fetch) {
+    ctgTestFetchRows(result, rows);
+  }
+  
+  taos_free_result(result);
+}
+
+
 TEST(tableMeta, normalTable) {
   struct SCatalog  *pCtg = NULL;
   SVgroupInfo       vgInfo = {0};
@@ -1245,7 +1274,7 @@ TEST(tableMeta, normalTable) {
 
   memset(&vgInfo, 0, sizeof(vgInfo));
   bool exists = false;
-  code = catalogGetCachedTableHashVgroup(pCtg, mockPointer, &n, &vgInfo, &exists);
+  code = catalogGetCachedTableHashVgroup(pCtg, &n, &vgInfo, &exists);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(vgInfo.vgId, 8);
   ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
@@ -1292,7 +1321,7 @@ TEST(tableMeta, normalTable) {
   taosMemoryFree(tableMeta);
 
   tableMeta = NULL;
-  catalogGetCachedTableMeta(pCtg, mockPointer, &n, &tableMeta);
+  catalogGetCachedTableMeta(pCtg, &n, &tableMeta);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(tableMeta->vgId, 8);
   ASSERT_EQ(tableMeta->tableType, TSDB_NORMAL_TABLE);
@@ -1500,7 +1529,7 @@ TEST(tableMeta, superTableCase) {
   }
 
   tableMeta = NULL;
-  code = catalogGetCachedSTableMeta(pCtg, mockPointer, &n, &tableMeta);
+  code = catalogGetCachedSTableMeta(pCtg, &n, &tableMeta);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(tableMeta->vgId, 0);
   ASSERT_EQ(tableMeta->tableType, TSDB_SUPER_TABLE);
@@ -2772,7 +2801,7 @@ TEST(apiTest, catalogChkAuth_test) {
 
   bool pass = false;
   bool exists = false;
-  code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
+  code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(exists, false);
   
@@ -2790,7 +2819,7 @@ TEST(apiTest, catalogChkAuth_test) {
     }
   }
 
-  code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
+  code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(pass, true);
   ASSERT_EQ(exists, true);
@@ -3063,6 +3092,58 @@ TEST(apiTest, catalogGetDnodeList_test) {
   catalogDestroy();
 }
 
+#ifdef INTEGRATION_TEST
+TEST(intTest, autoCreateTableTest) {
+  struct SCatalog  *pCtg = NULL;
+
+  TAOS     *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  ASSERT_TRUE(NULL != taos);   
+
+  ctgdEnableDebug("api", true);
+  ctgdEnableDebug("meta", true);
+  ctgdEnableDebug("cache", true);
+  ctgdEnableDebug("lock", true);
+  
+  ctgTestExecQuery(taos, "drop database if exists db1", false, NULL);
+  ctgTestExecQuery(taos, "create database db1", false, NULL);
+  ctgTestExecQuery(taos, "create stable db1.st1 (ts timestamp, f1 int) tags(tg1 int)", false, NULL);
+  ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 1)", false, NULL);
+
+  ctgdGetOneHandle(&pCtg);
+  
+  while (true) {
+    uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM);
+    if (2 != n) {
+      taosMsleep(50);
+    } else {
+      break;
+    }
+  }
+
+  uint64_t n = 0, m = 0;
+  ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n);
+
+  ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 2)", false, NULL);
+
+  ctgTestExecQuery(taos, "insert into db1.tb1 values(now, 3)", false, NULL);
+
+  taosMsleep(1000);
+  ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&m);
+
+  ASSERT_EQ(n, m);
+
+  ctgdEnableDebug("stopUpdate", true);
+  ctgTestExecQuery(taos, "alter table db1.st1 add column f2 double", false, NULL);
+
+  ctgdEnableDebug("stopUpdate", false);
+  
+  ctgTestExecQuery(taos, "insert into db1.tb1 (ts, f1) values(now, 4)", false, NULL);
+  
+  taos_close(taos);
+}
+
+#endif
+
 
 int main(int argc, char **argv) {
   testing::InitGoogleTest(&argc, argv);

source/libs/command/inc/commandInt.h

@@ -96,6 +96,7 @@ extern "C" {
 #define COMMAND_RESET_LOG "resetLog"
 #define COMMAND_SCHEDULE_POLICY "schedulePolicy"
 #define COMMAND_ENABLE_RESCHEDULE "enableReSchedule"
+#define COMMAND_CATALOG_DEBUG "catalogDebug"
 
 typedef struct SExplainGroup {
   int32_t   nodeNum;

source/libs/command/src/command.c

@@ -571,6 +571,8 @@ static int32_t execAlterCmd(char* cmd, char* value, bool* processed) {
     code = schedulerUpdatePolicy(atoi(value));
   } else if (0 == strcasecmp(cmd, COMMAND_ENABLE_RESCHEDULE)) {
     code = schedulerEnableReSchedule(atoi(value));
+  } else if (0 == strcasecmp(cmd, COMMAND_CATALOG_DEBUG)) {
+    code = ctgdHandleDbgCommand(value);
   } else {
     goto _return;
   }

source/libs/executor/src/executil.c

@@ -30,10 +30,10 @@
 // The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups
 struct STableListInfo {
   bool      oneTableForEachGroup;
-  int32_t   numOfOuputGroups; // the data block will be generated one by one
+  int32_t   numOfOuputGroups;  // the data block will be generated one by one
-  int32_t*  groupOffset;      // keep the offset value for each group in the tableList
+  int32_t*  groupOffset;       // keep the offset value for each group in the tableList
   SArray*   pTableList;
-  SHashObj* map;              // speedup acquire the tableQueryInfo by table uid
+  SHashObj* map;  // speedup acquire the tableQueryInfo by table uid
   uint64_t  suid;
 };
 
@@ -421,7 +421,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
   }
 
   for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) {
-    SColumnInfoData colInfo = {{0}, 0};
+    SColumnInfoData colInfo = {0, {0}};
     colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i);
     blockDataAppendColInfo(pResBlock, &colInfo);
   }
@@ -582,7 +582,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
   }
 
   for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) {
-    SColumnInfoData colInfo = {{0}, 0};
+    SColumnInfoData colInfo = {0, {0}};
     colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i);
     blockDataAppendColInfo(pResBlock, &colInfo);
   }
@@ -925,6 +925,15 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
 
     SArray* pTbList = getTableNameList(pList);
     int32_t numOfTables = taosArrayGetSize(pTbList);
+    SHashObj *uHash = NULL;
+    size_t    listlen = taosArrayGetSize(list);  // len > 0 means there already have uids
+    if (listlen > 0) {
+      uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
+      for (int i = 0; i < listlen; i++) {
+        int64_t *uid = taosArrayGet(list, i);
+        taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i));
+      }
+    }
 
     for (int i = 0; i < numOfTables; i++) {
       char* name = taosArrayGetP(pTbList, i);
@@ -933,9 +942,12 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
       if (metaGetTableUidByName(metaHandle, name, &uid) == 0) {
         ETableType tbType = TSDB_TABLE_MAX;
         if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) {
-          taosArrayPush(list, &uid);
+          if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) {
+            taosArrayPush(list, &uid);
+          }
         } else {
           taosArrayDestroy(pTbList);
+          taosHashCleanup(uHash);
           return -1;
         }
       } else {
@@ -944,6 +956,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
       }
     }
 
+    taosHashCleanup(uHash);
     taosArrayDestroy(pTbList);
     return 0;
   }
@@ -1482,10 +1495,6 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray
   while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) {
     SColumnInfoData* p = taosArrayGet(pCols, i);
     SColMatchItem*   pmInfo = taosArrayGet(pColMatchInfo, j);
-    /*    if (!outputEveryColumn && pmInfo->reserved) {
-          j++;
-          continue;
-        }*/
 
     if (p->info.colId == pmInfo->colId) {
       SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->dstSlotId);
@@ -1678,9 +1687,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) {
   return taosArrayGetSize(pTableList->pTableList);
 }
 
-uint64_t tableListGetSuid(const STableListInfo* pTableList) {
+uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->suid; }
-  return pTableList->suid;
-}
 
 STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) {
   if (taosArrayGetSize(pTableList->pTableList) == 0) {
@@ -1718,7 +1725,7 @@ int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t
 }
 
 int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalGroupIndex, STableKeyInfo** pKeyInfo,
-                         int32_t* size) {
+                              int32_t* size) {
   int32_t total = tableListGetOutputGroups(pTableList);
   if (ordinalGroupIndex < 0 || ordinalGroupIndex >= total) {
     return TSDB_CODE_INVALID_PARA;
@@ -1728,7 +1735,7 @@ int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalG
   // 1. only one group exists, and 2. one table exists for each group.
   if (total == 1) {
     *size = tableListGetSize(pTableList);
-    *pKeyInfo = (*size == 0)? NULL:taosArrayGet(pTableList->pTableList, 0);
+    *pKeyInfo = (*size == 0) ? NULL : taosArrayGet(pTableList->pTableList, 0);
     return TSDB_CODE_SUCCESS;
   } else if (total == tableListGetSize(pTableList)) {
     *size = 1;
@@ -1806,13 +1813,13 @@ void tableListClear(STableListInfo* pTableListInfo) {
 }
 
 static int32_t orderbyGroupIdComparFn(const void* p1, const void* p2) {
-  STableKeyInfo* pInfo1 = (STableKeyInfo*) p1;
+  STableKeyInfo* pInfo1 = (STableKeyInfo*)p1;
-  STableKeyInfo* pInfo2 = (STableKeyInfo*) p2;
+  STableKeyInfo* pInfo2 = (STableKeyInfo*)p2;
 
   if (pInfo1->groupId == pInfo2->groupId) {
     return 0;
   } else {
-    return pInfo1->groupId < pInfo2->groupId? -1:1;
+    return pInfo1->groupId < pInfo2->groupId ? -1 : 1;
   }
 }
 
@@ -1825,12 +1832,12 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
   SArray* pList = taosArrayInit(4, sizeof(int32_t));
 
   STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, 0);
-  uint64_t gid = pInfo->groupId;
+  uint64_t       gid = pInfo->groupId;
 
   int32_t start = 0;
   taosArrayPush(pList, &start);
 
-  for(int32_t i = 1; i < size; ++i) {
+  for (int32_t i = 1; i < size; ++i) {
     pInfo = taosArrayGet(pTableListInfo->pTableList, i);
     if (pInfo->groupId != gid) {
       taosArrayPush(pList, &i);
@@ -1845,16 +1852,17 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) {
   return TDB_CODE_SUCCESS;
 }
 
-int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group, bool groupSort) {
+int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* group,
+                                    bool groupSort) {
   int32_t code = TSDB_CODE_SUCCESS;
   ASSERT(pTableListInfo->map != NULL);
 
-  bool groupByTbname = groupbyTbname(group);
+  bool   groupByTbname = groupbyTbname(group);
   size_t numOfTables = taosArrayGetSize(pTableListInfo->pTableList);
   if (group == NULL || groupByTbname) {
     for (int32_t i = 0; i < numOfTables; i++) {
       STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
-      info->groupId = groupByTbname? info->uid:0;
+      info->groupId = groupByTbname ? info->uid : 0;
     }
 
     pTableListInfo->oneTableForEachGroup = groupByTbname;
@@ -1878,7 +1886,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
 
   // add all table entry in the hash map
   size_t size = taosArrayGetSize(pTableListInfo->pTableList);
-  for(int32_t i = 0; i < size; ++i) {
+  for (int32_t i = 0; i < size; ++i) {
     STableKeyInfo* p = taosArrayGet(pTableListInfo->pTableList, i);
     taosHashPut(pTableListInfo->map, &p->uid, sizeof(uint64_t), &i, sizeof(int32_t));
   }
@@ -1889,7 +1897,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
 int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle,
                                 STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond,
                                 struct SExecTaskInfo* pTaskInfo) {
-  int64_t st = taosGetTimestampUs();
+  int64_t     st = taosGetTimestampUs();
   const char* idStr = GET_TASKID(pTaskInfo);
 
   if (pHandle == NULL) {
@@ -1919,7 +1927,7 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
     return code;
   }
 
-  pTaskInfo->cost.groupIdMapTime = (taosGetTimestampUs() - st1)/1000.0;
+  pTaskInfo->cost.groupIdMapTime = (taosGetTimestampUs() - st1) / 1000.0;
   qDebug("generate group id map completed, elapsed time:%.2f ms %s", pTaskInfo->cost.groupIdMapTime, idStr);
 
   return TSDB_CODE_SUCCESS;

source/libs/executor/src/executor.c

@@ -321,7 +321,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S
 }
 
 int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) {
-  SExecTaskInfo*  pTaskInfo = (SExecTaskInfo*)tinfo;
+  SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
 
   if (isAdd) {
     qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str);
@@ -473,7 +473,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
 
   qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId);
 
-  _error:
+_error:
   // if failed to add ref for all tables in this query, abort current query
   return code;
 }
@@ -1027,10 +1027,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
 
       if (pTableScanInfo->dataReader == NULL) {
         STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
-        int32_t num = tableListGetSize(pTaskInfo->pTableInfoList);
+        int32_t        num = tableListGetSize(pTaskInfo->pTableInfoList);
 
         if (tsdbReaderOpen(pTableScanInfo->readHandle.vnode, &pTableScanInfo->cond, pList, num,
-                           &pTableScanInfo->dataReader, NULL) < 0 || pTableScanInfo->dataReader == NULL) {
+                           &pTableScanInfo->dataReader, NULL) < 0 ||
+            pTableScanInfo->dataReader == NULL) {
           ASSERT(0);
         }
       }
@@ -1071,14 +1072,14 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
     initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo);
     pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts;
 
-    if (pTaskInfo->pTableInfoList == NULL)  {
+    if (pTaskInfo->pTableInfoList == NULL) {
       pTaskInfo->pTableInfoList = tableListCreate();
     }
 
     tableListAddTableInfo(pTaskInfo->pTableInfoList, mtInfo.uid, 0);
 
     STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0);
-    int32_t size = tableListGetSize(pTaskInfo->pTableInfoList);
+    int32_t        size = tableListGetSize(pTaskInfo->pTableInfoList);
     ASSERT(size == 1);
 
     tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->dataReader, NULL);

source/libs/executor/src/executorimpl.c

@@ -2783,8 +2783,10 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul
     *defaultPgsz <<= 1u;
   }
 
+  // The default buffer for each operator in query is 10MB.
   // at least four pages need to be in buffer
-  *defaultBufsz = 4096 * 256;
+  // TODO: make this variable to be configurable.
+  *defaultBufsz = 4096 * 2560;
   if ((*defaultBufsz) <= (*defaultPgsz)) {
     (*defaultBufsz) = (*defaultPgsz) * 4;
   }
@@ -2971,7 +2973,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
 
   return pOperator;
 
-  _error:
+_error:
   if (pInfo != NULL) {
     destroyAggOperatorInfo(pInfo);
   }
@@ -3187,11 +3189,12 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
   code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
 
-  _error:
+_error:
   if (pInfo != NULL) {
     destroyFillOperatorInfo(pInfo);
   }
 
+  pTaskInfo->code = code;
   taosMemoryFreeClear(pOperator);
   return NULL;
 }

source/libs/executor/src/groupoperator.c

@@ -750,6 +750,14 @@ static void destroyPartitionOperatorInfo(void* param) {
   taosArrayDestroy(pInfo->pGroupColVals);
   taosMemoryFree(pInfo->keyBuf);
   taosArrayDestroy(pInfo->sortedGroupArray);
+
+  void* pGroupIter = taosHashIterate(pInfo->pGroupSet, NULL);
+  while (pGroupIter != NULL) {
+    SDataGroupInfo* pGroupInfo = pGroupIter;
+    taosArrayDestroy(pGroupInfo->pPageList);
+    pGroupIter = taosHashIterate(pInfo->pGroupSet, pGroupIter);
+  }
+
   taosHashCleanup(pInfo->pGroupSet);
   taosMemoryFree(pInfo->columnOffset);
 

source/libs/executor/src/scanoperator.c

@@ -1062,15 +1062,18 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
 
     int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond);
     if (code != TSDB_CODE_SUCCESS) {
-      return NULL;
+      goto _error;
     }
 
     STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList;
     size_t          num = tableListGetSize(pTableListInfo);
     void*           pList = tableListGetInfo(pTableListInfo, 0);
 
-    tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str);
+    code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str);
     cleanupQueryTableDataCond(&cond);
+    if (code != 0) {
+      goto _error;
+    }
   }
 
   pInfo->readHandle = *readHandle;
@@ -1164,6 +1167,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
                                      GET_TASKID(pTaskInfo));
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
+    T_LONG_JMP(pTaskInfo->env, code);
     return NULL;
   }
 
@@ -1884,11 +1888,15 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
     STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
     memcpy(&pTSInfo->cond, &pTaskInfo->streamInfo.tableCond, sizeof(SQueryTableDataCond));
     if (pTaskInfo->streamInfo.recoverStep == STREAM_RECOVER_STEP__PREPARE1) {
-      pTSInfo->cond.startVersion = -1;
+      pTSInfo->cond.startVersion = 0;
       pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer1;
+      qDebug("stream recover step 1, from %" PRId64 " to %" PRId64, pTSInfo->cond.startVersion,
+             pTSInfo->cond.endVersion);
     } else {
       pTSInfo->cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1;
       pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2;
+      qDebug("stream recover step 2, from %" PRId64 " to %" PRId64, pTSInfo->cond.startVersion,
+             pTSInfo->cond.endVersion);
     }
 
     /*resetTableScanInfo(pTSInfo, pWin);*/
@@ -1905,11 +1913,15 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
     if (pBlock != NULL) {
       calBlockTbName(&pInfo->tbnameCalSup, pBlock);
       updateInfoFillBlockData(pInfo->pUpdateInfo, pBlock, pInfo->primaryTsIndex);
+      qDebug("stream recover scan get block, rows %d", pBlock->info.rows);
       return pBlock;
     }
     pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE;
     STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
-    pTSInfo->cond.startVersion = 0;
+    tsdbReaderClose(pTSInfo->dataReader);
+    pTSInfo->dataReader = NULL;
+
+    pTSInfo->cond.startVersion = -1;
     pTSInfo->cond.endVersion = -1;
 
     return NULL;
@@ -2418,8 +2430,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
     if (pHandle->initTableReader) {
       pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
       pTSInfo->dataReader = NULL;
-      if (tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, pList, num, &pTSInfo->dataReader, NULL) < 0) {
+      int32_t code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->cond, pList, num, &pTSInfo->dataReader, NULL);
-        terrno = TSDB_CODE_OUT_OF_MEMORY;
+      if (code != 0) {
+        terrno = code;
         destroyTableScanOperatorInfo(pTableScanOp);
         goto _error;
       }
@@ -4284,131 +4297,6 @@ int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle*
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t createMultipleDataReaders2(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle,
-                                   STableListInfo* pTableListInfo, int32_t tableStartIdx, int32_t tableEndIdx,
-                                   STsdbReader** ppReader, const char* idstr) {
-  STsdbReader* pReader = NULL;
-  void*        pStart = tableListGetInfo(pTableListInfo, tableStartIdx);
-  int32_t      num = tableEndIdx - tableStartIdx + 1;
-
-  int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, pStart, num, &pReader, idstr);
-  if (code != 0) {
-    return code;
-  }
-
-  *ppReader = pReader;
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t loadDataBlockFromOneTable2(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
-                                          SSDataBlock* pBlock, uint32_t* status) {
-  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
-  STableMergeScanInfo* pInfo = pOperator->info;
-
-  uint64_t uid = pBlock->info.uid;
-
-  SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
-
-  pCost->totalBlocks += 1;
-  pCost->totalRows += pBlock->info.rows;
-
-  *status = pInfo->dataBlockLoadFlag;
-  if (pTableScanInfo->pFilterNode != NULL ||
-      overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info, pTableScanInfo->cond.order)) {
-    (*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
-  }
-
-  SDataBlockInfo* pBlockInfo = &pBlock->info;
-  taosMemoryFreeClear(pBlock->pBlockAgg);
-
-  if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
-    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
-           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
-    pCost->filterOutBlocks += 1;
-    return TSDB_CODE_SUCCESS;
-  } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
-    qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo),
-           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
-    pCost->skipBlocks += 1;
-
-    // clear all data in pBlock that are set when handing the previous block
-    for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
-      SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i);
-      pcol->pData = NULL;
-    }
-
-    return TSDB_CODE_SUCCESS;
-  } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
-    pCost->loadBlockStatis += 1;
-
-    bool             allColumnsHaveAgg = true;
-    SColumnDataAgg** pColAgg = NULL;
-    STsdbReader*     reader = pTableScanInfo->pReader;
-    tsdbRetrieveDatablockSMA(reader, &pColAgg, &allColumnsHaveAgg);
-
-    if (allColumnsHaveAgg == true) {
-      int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
-
-      // todo create this buffer during creating operator
-      if (pBlock->pBlockAgg == NULL) {
-        pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES);
-      }
-
-      for (int32_t i = 0; i < numOfCols; ++i) {
-        SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i);
-        if (!pColMatchInfo->needOutput) {
-          continue;
-        }
-        pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
-      }
-
-      return TSDB_CODE_SUCCESS;
-    } else {  // failed to load the block sma data, data block statistics does not exist, load data block instead
-      *status = FUNC_DATA_REQUIRED_DATA_LOAD;
-    }
-  }
-
-  ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD);
-
-  pCost->totalCheckedRows += pBlock->info.rows;
-  pCost->loadBlocks += 1;
-
-  STsdbReader* reader = pTableScanInfo->pReader;
-  SArray*      pCols = tsdbRetrieveDataBlock(reader, NULL);
-  if (pCols == NULL) {
-    return terrno;
-  }
-
-  relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
-
-  // currently only the tbname pseudo column
-  if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
-    int32_t code =
-        addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo,
-                               pTableScanInfo->pseudoSup.numOfExprs, pBlock, pBlock->info.rows, GET_TASKID(pTaskInfo));
-    if (code != TSDB_CODE_SUCCESS) {
-      T_LONG_JMP(pTaskInfo->env, code);
-    }
-  }
-
-  if (pTableScanInfo->pFilterNode != NULL) {
-    int64_t st = taosGetTimestampMs();
-    doFilter(pTableScanInfo->pFilterNode, pBlock, &pTableScanInfo->matchInfo, NULL);
-
-    double el = (taosGetTimestampUs() - st) / 1000.0;
-    pTableScanInfo->readRecorder.filterTime += el;
-
-    if (pBlock->info.rows == 0) {
-      pCost->filterOutBlocks += 1;
-      qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms",
-             GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el);
-    } else {
-      qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el);
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
 // todo refactor
 static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo,
                                          int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) {
@@ -4535,7 +4423,7 @@ typedef struct STableMergeScanSortSourceParam {
   SSDataBlock*   inputBlock;
 } STableMergeScanSortSourceParam;
 
-static SSDataBlock* getTableDataBlockTemp(void* param) {
+static SSDataBlock* getTableDataBlockImpl(void* param) {
   STableMergeScanSortSourceParam* source = param;
   SOperatorInfo*                  pOperator = source->pOperator;
   STableMergeScanInfo*            pInfo = pOperator->info;
@@ -4552,7 +4440,11 @@ static SSDataBlock* getTableDataBlockTemp(void* param) {
 
   void*        p = tableListGetInfo(pInfo->tableListInfo, readIdx + pInfo->tableStartIndex);
   SReadHandle* pHandle = &pInfo->readHandle;
-  tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->pReader, GET_TASKID(pTaskInfo));
+
+  int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->pReader, GET_TASKID(pTaskInfo));
+  if (code != 0) {
+    T_LONG_JMP(pOperator->pTaskInfo->env, code);
+  }
 
   STsdbReader* reader = pInfo->pReader;
   while (tsdbNextDataBlock(reader)) {
@@ -4603,55 +4495,6 @@ static SSDataBlock* getTableDataBlockTemp(void* param) {
   pInfo->pReader = NULL;
   return NULL;
 }
-static SSDataBlock* getTableDataBlock2(void* param) {
-  STableMergeScanSortSourceParam* source = param;
-  SOperatorInfo*                  pOperator = source->pOperator;
-  int64_t                         uid = source->uid;
-  SSDataBlock*                    pBlock = source->inputBlock;
-  STableMergeScanInfo*            pTableScanInfo = pOperator->info;
-
-  int64_t st = taosGetTimestampUs();
-
-  blockDataCleanup(pBlock);
-
-  STsdbReader* reader = pTableScanInfo->pReader;
-  while (tsdbTableNextDataBlock(reader, uid)) {
-    if (isTaskKilled(pOperator->pTaskInfo)) {
-      T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED);
-    }
-
-    // process this data block based on the probabilities
-    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
-    if (!processThisBlock) {
-      continue;
-    }
-
-    blockDataCleanup(pBlock);
-
-    int32_t rows = 0;
-    tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.uid, &pBlock->info.window);
-    blockDataEnsureCapacity(pBlock, rows);
-    pBlock->info.rows = rows;
-
-    uint32_t status = 0;
-    int32_t  code = loadDataBlockFromOneTable2(pOperator, pTableScanInfo, pBlock, &status);
-    if (code != TSDB_CODE_SUCCESS) {
-      T_LONG_JMP(pOperator->pTaskInfo->env, code);
-    }
-
-    // current block is filter out according to filter condition, continue load the next block
-    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
-      continue;
-    }
-
-    pBlock->info.groupId = getTableGroupId(pOperator->pTaskInfo->pTableInfoList, pBlock->info.uid);
-    pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows;
-    pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
-
-    return pBlock;
-  }
-  return NULL;
-}
 
 static SSDataBlock* getTableDataBlock(void* param) {
   STableMergeScanSortSourceParam* source = param;
@@ -4761,7 +4604,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage,
                                              pInfo->pSortInputBlock, pTaskInfo->id.str);
 
-  tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockTemp, NULL, NULL);
+  tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlockImpl, NULL, NULL);
 
   // one table has one data block
   int32_t numOfTable = tableEndIdx - tableStartIdx + 1;

source/libs/executor/src/sortoperator.c

@@ -579,21 +579,8 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
   return TSDB_CODE_SUCCESS;
 }
 
-SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
+static void doGetSortedBlockData(SMultiwayMergeOperatorInfo* pInfo, SSortHandle* pHandle, int32_t capacity, SSDataBlock* p) {
-                                        SArray* pColMatchInfo, SOperatorInfo* pOperator) {
-  SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
-  SExecTaskInfo*              pTaskInfo = pOperator->pTaskInfo;
 
-  blockDataCleanup(pDataBlock);
-
-  SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
-  if (p == NULL) {
-    return NULL;
-  }
-
-  blockDataEnsureCapacity(p, capacity);
-
-_retry:
   while (1) {
     STupleHandle* pTupleHandle = NULL;
     if (pInfo->groupSort) {
@@ -638,22 +625,48 @@ _retry:
     pInfo->hasGroupId = false;
   }
 
-  if (p->info.rows > 0) {  // todo extract method
+}
-    applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo, pOperator);
+
+SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
+                                        SArray* pColMatchInfo, SOperatorInfo* pOperator) {
+  SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
+
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  blockDataCleanup(pDataBlock);
+
+  SSDataBlock* p = tsortGetSortedDataBlock(pHandle);
+  if (p == NULL) {
+    return NULL;
+  }
+
+  blockDataEnsureCapacity(p, capacity);
+
+  while (1) {
+    doGetSortedBlockData(pInfo, pHandle, capacity, p);
     if (p->info.rows == 0) {
-      goto _retry;
+      break;
     }
 
+    if (p->info.rows > 0) {
+      applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo, pOperator);
+      if (p->info.rows > 0) {
+        break;
+      }
+    }
+  }
+
+  if (p->info.rows > 0) {
     blockDataEnsureCapacity(pDataBlock, p->info.rows);
+
     int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
     for (int32_t i = 0; i < numOfCols; ++i) {
       SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i);
-      //      ASSERT(pColMatchInfo-> == COL_MATCH_FROM_SLOT_ID);
 
       SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
       SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId);
       colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
     }
+
     pInfo->limitInfo.numOfOutputRows += p->info.rows;
     pDataBlock->info.rows = p->info.rows;
     pDataBlock->info.groupId = pInfo->groupId;

source/libs/function/src/tudf.c

@@ -122,14 +122,14 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) {
   taosGetCpuCores(&numCpuCores);
   snprintf(thrdPoolSizeEnvItem, 32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores * 2);
 
-  char pathTaosdLdLib[512] = {0};
+  char   pathTaosdLdLib[512] = {0};
   size_t taosdLdLibPathLen = sizeof(pathTaosdLdLib);
   int ret = uv_os_getenv("LD_LIBRARY_PATH", pathTaosdLdLib, &taosdLdLibPathLen);
   if (ret != UV_ENOBUFS) {
     taosdLdLibPathLen = strlen(pathTaosdLdLib);
   }
 
-  char udfdPathLdLib[1024] = {0};
+  char   udfdPathLdLib[1024] = {0};
   size_t udfdLdLibPathLen = strlen(tsUdfdLdLibPath);
   strncpy(udfdPathLdLib, tsUdfdLdLibPath, udfdLdLibPathLen);
   udfdPathLdLib[udfdLdLibPathLen] = ':';
@@ -362,7 +362,7 @@ typedef struct SUdfcProxy {
   SArray    *udfStubs;  // SUdfcFuncStub
 
   uv_mutex_t udfcUvMutex;
-  int8_t initialized;
+  int8_t     initialized;
 } SUdfcProxy;
 
 SUdfcProxy gUdfcProxy = {0};

source/libs/qworker/src/qworker.c

@@ -644,11 +644,13 @@ _return:
   input.msgType = qwMsg->msgType;
   code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL);
 
-  if (ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
+  if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) {
     void         *rsp = NULL;
     int32_t       dataLen = 0;
     SOutputData sOutput = {0};
-    QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput));
+    if (qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)) {
+      return TSDB_CODE_SUCCESS;
+    }
 
     if (rsp) {
       bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd);

source/libs/scheduler/src/schRemote.c

@@ -280,7 +280,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
         }
 
         atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows);
-        SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows);
+        SCH_TASK_DLOG("submit succeed, affectedRows:%d, blocks:%d", rsp->affectedRows, rsp->nBlocks);
 
         SCH_LOCK(SCH_WRITE, &pJob->resLock);
         if (pJob->execRes.res) {

source/libs/stream/inc/streamInc.h

@@ -43,6 +43,8 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock)
 
 int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq);
 
+int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet);
+
 int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId,
                                           SEpSet* pEpSet);
 

source/libs/stream/src/stream.c

@@ -135,8 +135,11 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR
   ((SMsgHead*)buf)->vgId = htonl(pReq->upstreamNodeId);
   SStreamDispatchRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead));
   pCont->inputStatus = status;
-  pCont->streamId = pReq->streamId;
+  pCont->streamId = htobe64(pReq->streamId);
-  pCont->taskId = pReq->upstreamTaskId;
+  pCont->upstreamNodeId = htonl(pReq->upstreamNodeId);
+  pCont->upstreamTaskId = htonl(pReq->upstreamTaskId);
+  pCont->downstreamNodeId = htonl(pTask->nodeId);
+  pCont->downstreamTaskId = htonl(pTask->taskId);
   pRsp->pCont = buf;
   pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp);
   tmsgSendRsp(pRsp);
@@ -203,10 +206,10 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S
   return 0;
 }
 
-int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp) {
+int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) {
   ASSERT(pRsp->inputStatus == TASK_OUTPUT_STATUS__NORMAL || pRsp->inputStatus == TASK_OUTPUT_STATUS__BLOCKED);
 
-  qDebug("task %d receive dispatch rsp", pTask->taskId);
+  qDebug("task %d receive dispatch rsp, code: %x", pTask->taskId, code);
 
   if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
     int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1);

source/libs/stream/src/streamDispatch.c

@@ -210,6 +210,46 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis
   return 0;
 }
 
+int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) {
+  void*   buf = NULL;
+  int32_t code = -1;
+  SRpcMsg msg = {0};
+
+  int32_t tlen;
+  tEncodeSize(tEncodeSStreamTaskCheckReq, pReq, tlen, code);
+  if (code < 0) {
+    return -1;
+  }
+
+  buf = rpcMallocCont(sizeof(SMsgHead) + tlen);
+  if (buf == NULL) {
+    return -1;
+  }
+
+  ((SMsgHead*)buf)->vgId = htonl(nodeId);
+  void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+
+  SEncoder encoder;
+  tEncoderInit(&encoder, abuf, tlen);
+  if ((code = tEncodeSStreamTaskCheckReq(&encoder, pReq)) < 0) {
+    goto FAIL;
+  }
+  tEncoderClear(&encoder);
+
+  msg.contLen = tlen + sizeof(SMsgHead);
+  msg.pCont = buf;
+  msg.msgType = TDMT_STREAM_TASK_CHECK;
+
+  qDebug("dispatch from task %d to task %d node %d: check msg", pTask->taskId, pReq->downstreamTaskId, nodeId);
+
+  tmsgSendReq(pEpSet, &msg);
+
+  return 0;
+FAIL:
+  if (buf) rpcFreeCont(buf);
+  return code;
+}
+
 int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId,
                                           SEpSet* pEpSet) {
   void*   buf = NULL;
@@ -243,7 +283,8 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov
 
   tmsgSendReq(pEpSet, &msg);
 
-  code = 0;
+  qDebug("dispatch from task %d to task %d node %d: recover finish msg", pTask->taskId, pReq->taskId, vgId);
+
   return 0;
 FAIL:
   if (buf) rpcFreeCont(buf);
@@ -279,7 +320,7 @@ int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* p
   msg.pCont = buf;
   msg.msgType = pTask->dispatchMsgType;
 
-  qDebug("dispatch from task %d to task %d node %d", pTask->taskId, pReq->taskId, vgId);
+  qDebug("dispatch from task %d to task %d node %d: data msg", pTask->taskId, pReq->taskId, vgId);
 
   tmsgSendReq(pEpSet, &msg);
 

source/libs/stream/src/streamExec.c

@@ -202,83 +202,83 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch)
 int32_t streamExecForAll(SStreamTask* pTask) {
   while (1) {
     int32_t batchCnt = 1;
-    void*   data = NULL;
+    void*   input = NULL;
     while (1) {
       SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue);
       if (qItem == NULL) {
         qDebug("stream task exec over, queue empty, task: %d", pTask->taskId);
         break;
       }
-      if (data == NULL) {
+      if (input == NULL) {
-        data = qItem;
+        input = qItem;
         streamQueueProcessSuccess(pTask->inputQueue);
         if (pTask->taskLevel == TASK_LEVEL__SINK) {
           break;
         }
       } else {
         void* newRet;
-        if ((newRet = streamMergeQueueItem(data, qItem)) == NULL) {
+        if ((newRet = streamMergeQueueItem(input, qItem)) == NULL) {
           streamQueueProcessFail(pTask->inputQueue);
           break;
         } else {
           batchCnt++;
-          data = newRet;
+          input = newRet;
           streamQueueProcessSuccess(pTask->inputQueue);
         }
       }
     }
 
     if (pTask->taskStatus == TASK_STATUS__DROPPING) {
-      if (data) streamFreeQitem(data);
+      if (input) streamFreeQitem(input);
       return 0;
     }
 
-    if (data == NULL) {
+    if (input == NULL) {
       break;
     }
 
     if (pTask->taskLevel == TASK_LEVEL__SINK) {
-      ASSERT(((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_BLOCK);
+      ASSERT(((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_BLOCK);
-      streamTaskOutput(pTask, data);
+      streamTaskOutput(pTask, input);
       continue;
     }
 
     SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
 
     qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, batchCnt);
-    streamTaskExecImpl(pTask, data, pRes);
+    streamTaskExecImpl(pTask, input, pRes);
     qDebug("stream task %d exec end", pTask->taskId);
 
     if (taosArrayGetSize(pRes) != 0) {
       SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
       if (qRes == NULL) {
         taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
-        streamFreeQitem(data);
+        streamFreeQitem(input);
         return -1;
       }
       qRes->type = STREAM_INPUT__DATA_BLOCK;
       qRes->blocks = pRes;
 
-      if (((SStreamQueueItem*)data)->type == STREAM_INPUT__DATA_SUBMIT) {
+      if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) {
-        SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
+        SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input;
         qRes->childId = pTask->selfChildId;
         qRes->sourceVer = pSubmit->ver;
-      } else if (((SStreamQueueItem*)data)->type == STREAM_INPUT__MERGED_SUBMIT) {
+      } else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) {
-        SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)data;
+        SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)input;
         qRes->childId = pTask->selfChildId;
         qRes->sourceVer = pMerged->ver;
       }
 
       if (streamTaskOutput(pTask, qRes) < 0) {
         taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
-        streamFreeQitem(data);
+        streamFreeQitem(input);
         taosFreeQitem(qRes);
         return -1;
       }
     } else {
       taosArrayDestroy(pRes);
     }
-    streamFreeQitem(data);
+    streamFreeQitem(input);
   }
   return 0;
 }

source/libs/stream/src/streamRecover.c

@@ -15,6 +15,148 @@
 
 #include "streamInc.h"
 
+int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) {
+  qDebug("task %d at node %d launch recover", pTask->taskId, pTask->nodeId);
+  if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
+    atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_PREPARE);
+    streamSetParamForRecover(pTask);
+    streamSourceRecoverPrepareStep1(pTask, version);
+
+    SStreamRecoverStep1Req req;
+    streamBuildSourceRecover1Req(pTask, &req);
+    int32_t len = sizeof(SStreamRecoverStep1Req);
+
+    void* serializedReq = rpcMallocCont(len);
+    if (serializedReq == NULL) {
+      return -1;
+    }
+
+    memcpy(serializedReq, &req, len);
+
+    SRpcMsg rpcMsg = {
+        .contLen = len,
+        .pCont = serializedReq,
+        .msgType = TDMT_VND_STREAM_RECOVER_STEP1,
+    };
+
+    if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) {
+      /*ASSERT(0);*/
+    }
+
+  } else if (pTask->taskLevel == TASK_LEVEL__AGG) {
+    atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
+    streamSetParamForRecover(pTask);
+    streamAggRecoverPrepare(pTask);
+  } else if (pTask->taskLevel == TASK_LEVEL__SINK) {
+    atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
+  }
+  return 0;
+}
+
+// checkstatus
+int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) {
+  SStreamTaskCheckReq req = {
+      .streamId = pTask->streamId,
+      .upstreamTaskId = pTask->taskId,
+      .upstreamNodeId = pTask->nodeId,
+      .childId = pTask->selfChildId,
+  };
+  // serialize
+  if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
+    req.reqId = tGenIdPI64();
+    req.downstreamNodeId = pTask->fixedEpDispatcher.nodeId;
+    req.downstreamTaskId = pTask->fixedEpDispatcher.taskId;
+    pTask->checkReqId = req.reqId;
+
+    qDebug("task %d at node %d check downstream task %d at node %d", pTask->taskId, pTask->nodeId, req.downstreamTaskId,
+           req.downstreamNodeId);
+    streamDispatchOneCheckReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet);
+  } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
+    SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
+    int32_t vgSz = taosArrayGetSize(vgInfo);
+    pTask->recoverTryingDownstream = vgSz;
+    pTask->checkReqIds = taosArrayInit(vgSz, sizeof(int64_t));
+
+    for (int32_t i = 0; i < vgSz; i++) {
+      SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
+      req.reqId = tGenIdPI64();
+      taosArrayPush(pTask->checkReqIds, &req.reqId);
+      req.downstreamNodeId = pVgInfo->vgId;
+      req.downstreamTaskId = pVgInfo->taskId;
+      qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->taskId, pTask->nodeId,
+             req.downstreamTaskId, req.downstreamNodeId);
+      streamDispatchOneCheckReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet);
+    }
+  } else {
+    qDebug("task %d at node %d direct launch recover since no downstream", pTask->taskId, pTask->nodeId);
+    streamTaskLaunchRecover(pTask, version);
+  }
+  return 0;
+}
+
+int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) {
+  SStreamTaskCheckReq req = {
+      .reqId = pRsp->reqId,
+      .streamId = pRsp->streamId,
+      .upstreamTaskId = pRsp->upstreamTaskId,
+      .upstreamNodeId = pRsp->upstreamNodeId,
+      .downstreamTaskId = pRsp->downstreamTaskId,
+      .downstreamNodeId = pRsp->downstreamNodeId,
+      .childId = pRsp->childId,
+  };
+  qDebug("task %d at node %d check downstream task %d at node %d (recheck)", pTask->taskId, pTask->nodeId,
+         req.downstreamTaskId, req.downstreamNodeId);
+  if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
+    streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet);
+  } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
+    SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
+    int32_t vgSz = taosArrayGetSize(vgInfo);
+    for (int32_t i = 0; i < vgSz; i++) {
+      SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
+      if (pVgInfo->taskId == req.downstreamTaskId) {
+        streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pVgInfo->epSet);
+      }
+    }
+  }
+  return 0;
+}
+
+int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq) {
+  return atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL;
+}
+
+int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) {
+  qDebug("task %d at node %d recv check rsp from task %d at node %d: status %d", pRsp->upstreamTaskId,
+         pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status);
+  if (pRsp->status == 1) {
+    if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
+      bool found = false;
+      for (int32_t i = 0; i < taosArrayGetSize(pTask->checkReqIds); i++) {
+        int64_t reqId = *(int64_t*)taosArrayGet(pTask->checkReqIds, i);
+        if (reqId == pRsp->reqId) {
+          found = true;
+          break;
+        }
+      }
+      if (!found) return -1;
+      int32_t left = atomic_sub_fetch_32(&pTask->recoverTryingDownstream, 1);
+      ASSERT(left >= 0);
+      if (left == 0) {
+        taosArrayDestroy(pTask->checkReqIds);
+        streamTaskLaunchRecover(pTask, version);
+      }
+    } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
+      if (pRsp->reqId != pTask->checkReqId) return -1;
+      streamTaskLaunchRecover(pTask, version);
+    } else {
+      ASSERT(0);
+    }
+  } else {
+    streamRecheckOneDownstream(pTask, pRsp);
+  }
+  return 0;
+}
+
 // common
 int32_t streamSetParamForRecover(SStreamTask* pTask) {
   void* exec = pTask->exec.executor;
@@ -86,10 +228,7 @@ int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) {
 // agg
 int32_t streamAggRecoverPrepare(SStreamTask* pTask) {
   void* exec = pTask->exec.executor;
-  /*if (qStreamSetParamForRecover(exec) < 0) {*/
+  pTask->recoverWaitingUpstream = taosArrayGetSize(pTask->childEpInfo);
-  /*return -1;*/
-  /*}*/
-  pTask->recoverWaitingChild = taosArrayGetSize(pTask->childEpInfo);
   return 0;
 }
 
@@ -107,7 +246,7 @@ int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) {
 
 int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) {
   if (pTask->taskLevel == TASK_LEVEL__AGG) {
-    int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingChild, 1);
+    int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingUpstream, 1);
     ASSERT(left >= 0);
     if (left == 0) {
       streamAggChildrenRecoverFinish(pTask);
@@ -116,6 +255,60 @@ int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) {
   return 0;
 }
 
+int32_t tEncodeSStreamTaskCheckReq(SEncoder* pEncoder, const SStreamTaskCheckReq* pReq) {
+  if (tStartEncode(pEncoder) < 0) return -1;
+  if (tEncodeI64(pEncoder, pReq->reqId) < 0) return -1;
+  if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->downstreamNodeId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->childId) < 0) return -1;
+  tEndEncode(pEncoder);
+  return pEncoder->pos;
+}
+
+int32_t tDecodeSStreamTaskCheckReq(SDecoder* pDecoder, SStreamTaskCheckReq* pReq) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pReq->reqId) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->downstreamNodeId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->childId) < 0) return -1;
+  tEndDecode(pDecoder);
+  return 0;
+}
+
+int32_t tEncodeSStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* pRsp) {
+  if (tStartEncode(pEncoder) < 0) return -1;
+  if (tEncodeI64(pEncoder, pRsp->reqId) < 0) return -1;
+  if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->upstreamNodeId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->upstreamTaskId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->downstreamNodeId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pRsp->childId) < 0) return -1;
+  if (tEncodeI8(pEncoder, pRsp->status) < 0) return -1;
+  tEndEncode(pEncoder);
+  return pEncoder->pos;
+}
+
+int32_t tDecodeSStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pRsp->reqId) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->upstreamNodeId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->upstreamTaskId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->downstreamNodeId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pRsp->childId) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pRsp->status) < 0) return -1;
+  tEndDecode(pDecoder);
+  return 0;
+}
+
 int32_t tEncodeSStreamRecoverFinishReq(SEncoder* pEncoder, const SStreamRecoverFinishReq* pReq) {
   if (tStartEncode(pEncoder) < 0) return -1;
   if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
@@ -132,79 +325,6 @@ int32_t tDecodeSStreamRecoverFinishReq(SDecoder* pDecoder, SStreamRecoverFinishR
   tEndDecode(pDecoder);
   return 0;
 }
-#if 0
-int32_t tEncodeStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamTaskRecoverReq* pReq) {
-  if (tStartEncode(pEncoder) < 0) return -1;
-  if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1;
-  tEndEncode(pEncoder);
-  return pEncoder->pos;
-}
-
-int32_t tDecodeStreamTaskRecoverReq(SDecoder* pDecoder, SStreamTaskRecoverReq* pReq) {
-  if (tStartDecode(pDecoder) < 0) return -1;
-  if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1;
-  tEndDecode(pDecoder);
-  return 0;
-}
-
-int32_t tEncodeStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamTaskRecoverRsp* pRsp) {
-  if (tStartEncode(pEncoder) < 0) return -1;
-  if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pRsp->reqTaskId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pRsp->rspTaskId) < 0) return -1;
-  if (tEncodeI8(pEncoder, pRsp->inputStatus) < 0) return -1;
-  tEndEncode(pEncoder);
-  return pEncoder->pos;
-}
-
-int32_t tDecodeStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamTaskRecoverRsp* pReq) {
-  if (tStartDecode(pDecoder) < 0) return -1;
-  if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->reqTaskId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->rspTaskId) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pReq->inputStatus) < 0) return -1;
-  tEndDecode(pDecoder);
-  return 0;
-}
-
-int32_t tEncodeSMStreamTaskRecoverReq(SEncoder* pEncoder, const SMStreamTaskRecoverReq* pReq) {
-  if (tStartEncode(pEncoder) < 0) return -1;
-  if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
-  tEndEncode(pEncoder);
-  return pEncoder->pos;
-}
-
-int32_t tDecodeSMStreamTaskRecoverReq(SDecoder* pDecoder, SMStreamTaskRecoverReq* pReq) {
-  if (tStartDecode(pDecoder) < 0) return -1;
-  if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
-  tEndDecode(pDecoder);
-  return 0;
-}
-
-int32_t tEncodeSMStreamTaskRecoverRsp(SEncoder* pEncoder, const SMStreamTaskRecoverRsp* pRsp) {
-  if (tStartEncode(pEncoder) < 0) return -1;
-  if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1;
-  tEndEncode(pEncoder);
-  return pEncoder->pos;
-}
-
-int32_t tDecodeSMStreamTaskRecoverRsp(SDecoder* pDecoder, SMStreamTaskRecoverRsp* pReq) {
-  if (tStartDecode(pDecoder) < 0) return -1;
-  if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
-  tEndDecode(pDecoder);
-  return 0;
-}
-#endif
 
 int32_t tEncodeSStreamCheckpointInfo(SEncoder* pEncoder, const SStreamCheckpointInfo* pCheckpoint) {
   if (tEncodeI32(pEncoder, pCheckpoint->srcNodeId) < 0) return -1;
@@ -248,308 +368,3 @@ int32_t tDecodeSStreamMultiVgCheckpointInfo(SDecoder* pDecoder, SStreamMultiVgCh
   }
   return 0;
 }
-
-#if 0
-int32_t tEncodeSStreamTaskRecoverReq(SEncoder* pEncoder, const SStreamRecoverDownstreamReq* pReq) {
-  if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->downstreamTaskId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
-  return 0;
-}
-
-int32_t tDecodeSStreamTaskRecoverReq(SDecoder* pDecoder, SStreamRecoverDownstreamReq* pReq) {
-  if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->downstreamTaskId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
-  return 0;
-}
-
-int32_t tEncodeSStreamTaskRecoverRsp(SEncoder* pEncoder, const SStreamRecoverDownstreamRsp* pRsp) {
-  if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pRsp->taskId) < 0) return -1;
-  int32_t sz = taosArrayGetSize(pRsp->checkpointVer);
-  if (tEncodeI32(pEncoder, sz) < 0) return -1;
-  for (int32_t i = 0; i < sz; i++) {
-    SStreamCheckpointInfo* pInfo = taosArrayGet(pRsp->checkpointVer, i);
-    if (tEncodeSStreamCheckpointInfo(pEncoder, pInfo) < 0) return -1;
-  }
-  return 0;
-}
-
-int32_t tDecodeSStreamTaskRecoverRsp(SDecoder* pDecoder, SStreamRecoverDownstreamRsp* pRsp) {
-  if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pRsp->taskId) < 0) return -1;
-  int32_t sz;
-  if (tDecodeI32(pDecoder, &sz) < 0) return -1;
-  pRsp->checkpointVer = taosArrayInit(sz, sizeof(SStreamCheckpointInfo));
-  if (pRsp->checkpointVer == NULL) return -1;
-  for (int32_t i = 0; i < sz; i++) {
-    SStreamCheckpointInfo info;
-    if (tDecodeSStreamCheckpointInfo(pDecoder, &info) < 0) return -1;
-    taosArrayPush(pRsp->checkpointVer, &info);
-  }
-  return 0;
-}
-#endif
-
-int32_t streamSaveStateInfo(SStreamMeta* pMeta, SStreamTask* pTask) {
-#if 0
-  void* buf = NULL;
-
-  ASSERT(pTask->taskLevel == TASK_LEVEL__SINK);
-
-  SStreamMultiVgCheckpointInfo checkpoint;
-  checkpoint.checkpointId = atomic_fetch_add_32(&pTask->nextCheckId, 1);
-  checkpoint.checkTs = taosGetTimestampMs();
-  checkpoint.streamId = pTask->streamId;
-  checkpoint.taskId = pTask->taskId;
-  checkpoint.checkpointVer = pTask->checkpointInfo;
-
-  int32_t len;
-  int32_t code;
-  tEncodeSize(tEncodeSStreamMultiVgCheckpointInfo, &checkpoint, len, code);
-  if (code < 0) {
-    return -1;
-  }
-
-  buf = taosMemoryCalloc(1, len);
-  if (buf == NULL) {
-    return -1;
-  }
-  SEncoder encoder;
-  tEncoderInit(&encoder, buf, len);
-  tEncodeSStreamMultiVgCheckpointInfo(&encoder, &checkpoint);
-  tEncoderClear(&encoder);
-
-  SStreamCheckpointKey key = {
-      .taskId = pTask->taskId,
-      .checkpointId = checkpoint.checkpointId,
-  };
-
-  if (tdbTbUpsert(pMeta->pStateDb, &key, sizeof(SStreamCheckpointKey), buf, len, &pMeta->txn) < 0) {
-    ASSERT(0);
-    goto FAIL;
-  }
-
-  int32_t sz = taosArrayGetSize(pTask->checkpointInfo);
-  for (int32_t i = 0; i < sz; i++) {
-    SStreamCheckpointInfo* pCheck = taosArrayGet(pTask->checkpointInfo, i);
-    pCheck->stateSaveVer = pCheck->stateProcessedVer;
-  }
-
-  taosMemoryFree(buf);
-  return 0;
-FAIL:
-  if (buf) taosMemoryFree(buf);
-  return -1;
-#endif
-  return 0;
-}
-
-int32_t streamLoadStateInfo(SStreamMeta* pMeta, SStreamTask* pTask) {
-#if 0
-  void*   pVal = NULL;
-  int32_t vLen = 0;
-  if (tdbTbGet(pMeta->pStateDb, &pTask->taskId, sizeof(void*), &pVal, &vLen) < 0) {
-    return -1;
-  }
-  SDecoder decoder;
-  tDecoderInit(&decoder, pVal, vLen);
-  SStreamMultiVgCheckpointInfo aggCheckpoint;
-  tDecodeSStreamMultiVgCheckpointInfo(&decoder, &aggCheckpoint);
-  tDecoderClear(&decoder);
-
-  pTask->nextCheckId = aggCheckpoint.checkpointId + 1;
-  pTask->checkpointInfo = aggCheckpoint.checkpointVer;
-#endif
-  return 0;
-}
-
-int32_t streamSaveSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
-  ASSERT(pTask->taskLevel == TASK_LEVEL__SINK);
-  return streamSaveStateInfo(pMeta, pTask);
-}
-
-int32_t streamRecoverSinkLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
-  ASSERT(pTask->taskLevel == TASK_LEVEL__SINK);
-  return streamLoadStateInfo(pMeta, pTask);
-}
-
-int32_t streamSaveAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
-  ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
-  // TODO save and copy state
-
-  // save state info
-  if (streamSaveStateInfo(pMeta, pTask) < 0) {
-    return -1;
-  }
-  return 0;
-}
-
-#if 0
-int32_t streamFetchRecoverStatus(SStreamTask* pTask, const SVgroupInfo* pVgInfo) {
-  int32_t                     taskId = pVgInfo->taskId;
-  int32_t                     nodeId = pVgInfo->vgId;
-  SStreamRecoverDownstreamReq req = {
-      .streamId = pTask->taskId,
-      .downstreamTaskId = taskId,
-      .taskId = pTask->taskId,
-  };
-  int32_t tlen;
-  int32_t code;
-  tEncodeSize(tEncodeSStreamTaskRecoverReq, &req, tlen, code);
-  if (code < 0) {
-    return -1;
-  }
-  void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + tlen);
-  if (buf == NULL) {
-    return -1;
-  }
-  void*    abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-  SEncoder encoder;
-  tEncoderInit(&encoder, abuf, tlen);
-  if (tEncodeSStreamTaskRecoverReq(&encoder, &req) < 0) {
-    tEncoderClear(&encoder);
-    taosMemoryFree(buf);
-    return -1;
-  }
-  tEncoderClear(&encoder);
-
-  ((SMsgHead*)buf)->vgId = htonl(nodeId);
-  SRpcMsg msg = {
-      .pCont = buf, .contLen = sizeof(SMsgHead) + tlen,
-      /*.msgType = */
-  };
-  tmsgSendReq(&pVgInfo->epSet, &msg);
-
-  return 0;
-}
-
-int32_t streamFetchDownstreamStatus(SStreamMeta* pMeta, SStreamTask* pTask) {
-  // set self status to recover_phase1
-  SStreamRecoverStatus* pRecover;
-  atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_DOWNSTREAM);
-  pRecover = taosHashGet(pMeta->pRecoverStatus, &pTask->taskId, sizeof(int32_t));
-  if (pRecover == NULL) {
-    pRecover = taosMemoryCalloc(1, sizeof(SStreamRecoverStatus));
-    if (pRecover == NULL) {
-      return -1;
-    }
-    pRecover->info = taosArrayInit(0, sizeof(void*));
-    if (pRecover->info == NULL) {
-      taosMemoryFree(pRecover);
-      return -1;
-    }
-    taosHashPut(pMeta->pRecoverStatus, &pTask->taskId, sizeof(int32_t), &pRecover, sizeof(void*));
-  }
-
-  if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) {
-    pRecover->totReq = 1;
-  } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
-    int32_t numOfDownstream = taosArrayGetSize(pTask->shuffleDispatcher.dbInfo.pVgroupInfos);
-    pRecover->totReq = numOfDownstream;
-    for (int32_t i = 0; i < numOfDownstream; i++) {
-      SVgroupInfo* pVgInfo = taosArrayGet(pTask->shuffleDispatcher.dbInfo.pVgroupInfos, i);
-      streamFetchRecoverStatus(pTask, pVgInfo);
-    }
-  } else {
-    ASSERT(0);
-  }
-  return 0;
-}
-#endif
-
-#if 0
-int32_t streamProcessFetchStatusRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStreamRecoverDownstreamRsp* pRsp) {
-  // if failed, set timer and retry
-  // if successful
-  int32_t               taskId = pTask->taskId;
-  SStreamRecoverStatus* pRecover = taosHashGet(pMeta->pRecoverStatus, &taskId, sizeof(int32_t));
-  if (pRecover == NULL) {
-    return -1;
-  }
-
-  taosArrayPush(pRecover->info, &pRsp->checkpointVer);
-
-  int32_t leftRsp = atomic_sub_fetch_32(&pRecover->waitingRspCnt, 1);
-  ASSERT(leftRsp >= 0);
-
-  if (leftRsp == 0) {
-    ASSERT(taosArrayGetSize(pRecover->info) == pRecover->totReq);
-
-    // srcNodeId -> SStreamCheckpointInfo*
-    SHashObj* pFinalChecks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
-    if (pFinalChecks == NULL) return -1;
-
-    for (int32_t i = 0; i < pRecover->totReq; i++) {
-      SArray* pChecks = taosArrayGetP(pRecover->info, i);
-      int32_t sz = taosArrayGetSize(pChecks);
-      for (int32_t j = 0; j < sz; j++) {
-        SStreamCheckpointInfo* pOneCheck = taosArrayGet(pChecks, j);
-        SStreamCheckpointInfo* pCheck = taosHashGet(pFinalChecks, &pOneCheck->srcNodeId, sizeof(int32_t));
-        if (pCheck == NULL) {
-          pCheck = taosMemoryCalloc(1, sizeof(SStreamCheckpointInfo));
-          pCheck->srcNodeId = pOneCheck->srcNodeId;
-          pCheck->srcChildId = pOneCheck->srcChildId;
-          pCheck->stateProcessedVer = pOneCheck->stateProcessedVer;
-          taosHashPut(pFinalChecks, &pCheck->srcNodeId, sizeof(int32_t), &pCheck, sizeof(void*));
-        } else {
-          pCheck->stateProcessedVer = TMIN(pCheck->stateProcessedVer, pOneCheck->stateProcessedVer);
-        }
-      }
-    }
-    // load local state
-    //
-    // recover
-    //
-    if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
-      qStreamPrepareRecover(pTask->exec.executor, pTask->startVer, pTask->recoverSnapVer);
-      if (streamPipelineExec(pTask, 10000, true) < 0) {
-        return -1;
-      }
-    }
-    taosHashCleanup(pFinalChecks);
-    taosHashRemove(pMeta->pRecoverStatus, &taskId, sizeof(int32_t));
-    atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL);
-  }
-  return 0;
-}
-
-int32_t streamRecoverAggLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
-  ASSERT(pTask->taskLevel == TASK_LEVEL__AGG);
-  // recover sink level
-  // after all sink level recovered
-  // choose suitable state to recover
-  return 0;
-}
-
-int32_t streamSaveSourceLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
-  ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE);
-  // TODO: save and copy state
-  return 0;
-}
-
-int32_t streamRecoverSourceLevel(SStreamMeta* pMeta, SStreamTask* pTask) {
-  ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE);
-  // if totLevel == 3
-  // fetch agg state
-  // recover from local state to agg state, not send msg
-  // recover from agg state to most recent log v1
-  // enable input queue, set status recover_phase2
-  // recover from v1 to queue msg v2, set status normal
-
-  // if totLevel == 2
-  // fetch sink state
-  // recover from local state to sink state v1, send msg
-  // enable input queue, set status recover_phase2
-  // recover from v1 to queue msg v2, set status normal
-  return 0;
-}
-
-int32_t streamRecoverTask(SStreamTask* pTask) {
-  //
-  return 0;
-}
-#endif

source/libs/transport/src/transCli.c

@@ -427,7 +427,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
 
     if (pCtx == NULL || pCtx->pSem == NULL) {
       if (transMsg.info.ahandle == NULL) {
-        if (REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg);
+        if (pMsg == NULL || REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg);
         once = true;
         continue;
       }

source/libs/transport/src/transSvr.c

@@ -127,7 +127,7 @@ static void uvFreeCb(uv_handle_t* handle);
 
 static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg);
 
-static void uvPrepareSendData(SSvrMsg* msg, uv_buf_t* wb);
+static int  uvPrepareSendData(SSvrMsg* msg, uv_buf_t* wb);
 static void uvStartSendResp(SSvrMsg* msg);
 
 static void uvNotifyLinkBrokenToApp(SSvrConn* conn);
@@ -384,7 +384,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) {
   taosMemoryFree(req);
 }
 
-static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
+static int uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
   SSvrConn*  pConn = smsg->pConn;
   STransMsg* pMsg = &smsg->msg;
   if (pMsg->pCont == 0) {
@@ -397,6 +397,13 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
   pHead->hasEpSet = pMsg->info.hasEpSet;
   pHead->magicNum = htonl(TRANS_MAGIC_NUM);
 
+  // handle invalid drop_task resp, TD-20098
+  if (pMsg->msgType == TDMT_SCH_DROP_TASK && pMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) {
+    transQueuePop(&pConn->srvMsgs);
+    destroySmsg(smsg);
+    return -1;
+  }
+
   if (pConn->status == ConnNormal) {
     pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType);
     if (smsg->type == Release) pHead->msgType = 0;
@@ -431,6 +438,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
 
   wb->base = (char*)pHead;
   wb->len = len;
+  return 0;
 }
 
 static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) {
@@ -440,7 +448,9 @@ static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) {
   }
 
   uv_buf_t wb;
-  uvPrepareSendData(smsg, &wb);
+  if (uvPrepareSendData(smsg, &wb) < 0) {
+    return;
+  }
 
   transRefSrvHandle(pConn);
   uv_write_t* req = transReqQueuePush(&pConn->wreqQueue);
@@ -451,8 +461,9 @@ static void uvStartSendResp(SSvrMsg* smsg) {
   SSvrConn* pConn = smsg->pConn;
   if (pConn->broken == true) {
     // persist by
-    transFreeMsg(smsg->msg.pCont);
+    destroySmsg(smsg);
-    taosMemoryFree(smsg);
+    // transFreeMsg(smsg->msg.pCont);
+    // taosMemoryFree(smsg);
     transUnrefSrvHandle(pConn);
     return;
   }
@@ -748,10 +759,11 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
       return;
     }
     transSockInfo2Str(&sockname, pConn->src);
-    struct sockaddr_in addr = *(struct sockaddr_in*)&sockname;
 
+    struct sockaddr_in addr = *(struct sockaddr_in*)&peername;
     pConn->clientIp = addr.sin_addr.s_addr;
     pConn->port = ntohs(addr.sin_port);
+
     uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocRecvBufferCb, uvOnRecvCb);
 
   } else {

source/util/src/terror.c

@@ -340,7 +340,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW,        "TSDB no cache last ro
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_NOT_EXIST,          "Table not exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST,        "Stable already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST,            "Stable not exists")
-TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECREATED,          "Table re-created")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER, "Table schema is old")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TDB_ENV_OPEN_ERROR,       "TDB env open error")
 
 // query
@@ -609,6 +609,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG,                "Invalid message")
 TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH,          "Consumer mismatch")
 TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED,            "Consumer closed")
 
+// stream
+TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST,          "Stream task not exist")
+
 // TDLite
 TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS,         "Invalid TDLite open flags")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_DIR,           "Invalid TDLite open directory")

source/util/src/tqueue.c

@@ -21,46 +21,6 @@
 int64_t tsRpcQueueMemoryAllowed = 0;
 int64_t tsRpcQueueMemoryUsed = 0;
 
-typedef struct STaosQnode STaosQnode;
-
-typedef struct STaosQnode {
-  STaosQnode *next;
-  STaosQueue *queue;
-  int64_t     timestamp;
-  int32_t     size;
-  int8_t      itype;
-  int8_t      reserved[3];
-  char        item[];
-} STaosQnode;
-
-typedef struct STaosQueue {
-  STaosQnode   *head;
-  STaosQnode   *tail;
-  STaosQueue   *next;     // for queue set
-  STaosQset    *qset;     // for queue set
-  void         *ahandle;  // for queue set
-  FItem         itemFp;
-  FItems        itemsFp;
-  TdThreadMutex mutex;
-  int64_t       memOfItems;
-  int32_t       numOfItems;
-} STaosQueue;
-
-typedef struct STaosQset {
-  STaosQueue   *head;
-  STaosQueue   *current;
-  TdThreadMutex mutex;
-  tsem_t        sem;
-  int32_t       numOfQueues;
-  int32_t       numOfItems;
-} STaosQset;
-
-typedef struct STaosQall {
-  STaosQnode *current;
-  STaosQnode *start;
-  int32_t     numOfItems;
-} STaosQall;
-
 STaosQueue *taosOpenQueue() {
   STaosQueue *queue = taosMemoryCalloc(1, sizeof(STaosQueue));
   if (queue == NULL) {

source/util/src/tworker.c

@@ -36,14 +36,13 @@ int32_t tQWorkerInit(SQWorkerPool *pool) {
     worker->pool = pool;
   }
 
-  uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max);
+  uDebug("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max);
   return 0;
 }
 
 void tQWorkerCleanup(SQWorkerPool *pool) {
   for (int32_t i = 0; i < pool->max; ++i) {
     SQWorker *worker = pool->workers + i;
-    // if (worker == NULL) continue;
     if (taosCheckPthreadValid(worker->thread)) {
       taosQsetThreadResume(pool->qset);
     }
@@ -51,7 +50,6 @@ void tQWorkerCleanup(SQWorkerPool *pool) {
 
   for (int32_t i = 0; i < pool->max; ++i) {
     SQWorker *worker = pool->workers + i;
-    // if (worker == NULL) continue;
     if (taosCheckPthreadValid(worker->thread)) {
       taosThreadJoin(worker->thread, NULL);
       taosThreadClear(&worker->thread);
@@ -73,11 +71,13 @@ static void *tQWorkerThreadFp(SQWorker *worker) {
 
   taosBlockSIGPIPE();
   setThreadName(pool->name);
-  uDebug("worker:%s:%d is running", pool->name, worker->id);
+  worker->pid = taosGetSelfPthreadId();
+  uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, worker->pid);
 
   while (1) {
     if (taosReadQitemFromQset(pool->qset, (void **)&msg, &qinfo) == 0) {
-      uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset);
+      uInfo("worker:%s:%d qset:%p, got no message and exiting, thread:%08" PRId64, pool->name, worker->id, pool->qset,
+            worker->pid);
       break;
     }
 
@@ -124,7 +124,7 @@ STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) {
   }
 
   taosThreadMutexUnlock(&pool->mutex);
-  uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle);
+  uInfo("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle);
 
   return queue;
 }
@@ -191,12 +191,14 @@ static void *tWWorkerThreadFp(SWWorker *worker) {
 
   taosBlockSIGPIPE();
   setThreadName(pool->name);
-  uDebug("worker:%s:%d is running", pool->name, worker->id);
+  worker->pid = taosGetSelfPthreadId();
+  uInfo("worker:%s:%d is running, thread:%08" PRId64, pool->name, worker->id, worker->pid);
 
   while (1) {
     numOfMsgs = taosReadAllQitemsFromQset(worker->qset, worker->qall, &qinfo);
     if (numOfMsgs == 0) {
-      uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, worker->qset);
+      uInfo("worker:%s:%d qset:%p, got no message and exiting, thread:%08" PRId64, pool->name, worker->id, worker->qset,
+            worker->pid);
       break;
     }
 
@@ -244,7 +246,9 @@ STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) {
     pool->nextId = (pool->nextId + 1) % pool->max;
   }
 
-  uDebug("worker:%s, queue:%p is allocated, ahandle:%p", pool->name, queue, ahandle);
+  while (worker->pid <= 0) taosMsleep(10);
+  queue->threadId = worker->pid;
+  uInfo("worker:%s, queue:%p is allocated, ahandle:%p thread:%08" PRId64, pool->name, queue, ahandle, queue->threadId);
   code = 0;
 
 _OVER:

tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py

@@ -25,7 +25,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor())
 

tests/develop-test/5-taos-tools/taosbenchmark/commandline.py

@@ -27,7 +27,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/custom_col_tag.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/default_json.py

@@ -23,7 +23,8 @@ class TDTestCase:
         [TD-11510] taosBenchmark test cases
         """
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/demo.py

@@ -27,7 +27,8 @@ class TDTestCase:
         """
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
 

tests/develop-test/5-taos-tools/taosbenchmark/json_tag.py

@@ -24,7 +24,8 @@ class TDTestCase:
         '''
         return
 
-    def init(self, conn, logSql, replicaVarl=1):
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)