[TD-14388]<feature>: Support value type parameter processing in scalar function

.clang-format

@@ -5,7 +5,6 @@ AccessModifierOffset: -1
 AlignAfterOpenBracket: Align
 AlignConsecutiveAssignments: false
 AlignConsecutiveDeclarations: true
-AlignConsecutiveMacros: true
 AlignEscapedNewlinesLeft: true
 AlignOperands:   true
 AlignTrailingComments: true

example/src/tmq.c

@@ -80,8 +80,9 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  const char* sql = "select * from tu1";
+  /*const char* sql = "select * from tu1";*/
-  pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
+  /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
+  pRes = taos_query(pConn, "create topic test_stb_topic_1 as select * from tu1");
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));
     return -1;

example/src/tstream.c

@@ -20,7 +20,7 @@
 #include "taos.h"
 
 int32_t init_env() {
-  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
   if (pConn == NULL) {
     return -1;
   }
@@ -65,7 +65,7 @@ int32_t init_env() {
 int32_t create_stream() {
   printf("create stream\n");
   TAOS_RES* pRes;
-  TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
+  TAOS*     pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
   if (pConn == NULL) {
     return -1;
   }

include/client/consumer/consumer.h

@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_CONSUMER_H_
-#define _TD_CONSUMER_H_
-
-#include "tlist.h"
-#include "tarray.h"
-#include "hash.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-  //consumer handle
-  struct tmq_consumer_t;
-  typedef struct tmq_consumer_t tmq_consumer_t;
-
-  //consumer config
-  struct tmq_consumer_config_t;
-  typedef struct tmq_consumer_config_t tmq_consumer_config_t;
-
-  //response err
-  struct tmq_resp_err_t;
-  typedef struct tmq_resp_err_t tmq_resp_err_t;
-
-  struct tmq_message_t;
-  typedef struct tmq_message_t tmq_message_t;
-
-  struct tmq_col_batch_t;
-  typedef struct tmq_col_batch_t tmq_col_batch_t;
-
-  //get content of message
-  tmq_col_batch_t* tmq_get_msg_col_by_idx(tmq_message_t*, int32_t col_id);
-  tmq_col_batch_t* tmq_get_msg_col_by_name(tmq_message_t*, const char*);
-
-  //consumer config
-  int32_t tmq_conf_set(tmq_consumer_config_t* , const char* config_key, const char* config_value, char* errstr, int32_t errstr_cap);
-
-  //consumer initialization
-  //resouces are supposed to be free by users by calling tmq_consumer_destroy
-  tmq_consumer_t* tmq_consumer_new(tmq_consumer_config_t* , char* errstr, int32_t errstr_cap);
-
-  //subscribe
-  tmq_resp_err_t tmq_subscribe(tmq_consumer_t*, const SList*);
-  tmq_resp_err_t tmq_unsubscribe(tmq_consumer_t*);
-
-  //consume
-  //resouces are supposed to be free by users by calling tmq_message_destroy
-  tmq_message_t* tmq_consume_poll(tmq_consumer_t*, int64_t blocking_time);
-
-  //destroy message and free memory
-  void tmq_message_destroy(tmq_message_t*);
-
-  //close consumer
-  int32_t tmq_consumer_close(tmq_consumer_t*);
-
-  //destroy consumer
-  void tmq_consumer_destroy(tmq_message_t*);
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_CONSUMER_H_*/

include/client/stream/stream.h

@@ -1,27 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TD_STREAM_H_
-#define _TD_STREAM_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TD_STREAM_H_*/

include/client/taos.h

@@ -26,9 +26,11 @@ extern "C" {
 typedef void   TAOS;
 typedef void   TAOS_STMT;
 typedef void   TAOS_RES;
-typedef void   TAOS_STREAM;
-typedef void   TAOS_SUB;
 typedef void **TAOS_ROW;
+#if 0
+typedef void   TAOS_STREAM;
+#endif
+typedef void   TAOS_SUB;
 
 // Data type definition
 #define TSDB_DATA_TYPE_NULL       0   // 1 bytes
@@ -149,6 +151,7 @@ DLL_EXPORT int       taos_stmt_execute(TAOS_STMT *stmt);
 DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
 DLL_EXPORT int       taos_stmt_close(TAOS_STMT *stmt);
 DLL_EXPORT char     *taos_stmt_errstr(TAOS_STMT *stmt);
+DLL_EXPORT int       taos_stmt_affected_rows(TAOS_STMT *stmt);
 
 DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
 DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
@@ -179,15 +182,20 @@ DLL_EXPORT int         taos_errno(TAOS_RES *tres);
 DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param);
 DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param);
 
+// Shuduo: temporary enable for app build
+#if 1
 typedef void (*__taos_sub_fn_t)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
 DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, __taos_sub_fn_t fp,
                                     void *param, int interval);
 DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
 DLL_EXPORT void      taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
+#endif
 
+#if 0
 DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
                                          int64_t stime, void *param, void (*callback)(void *));
 DLL_EXPORT void         taos_close_stream(TAOS_STREAM *tstr);
+#endif
 
 DLL_EXPORT int       taos_load_table_info(TAOS *taos, const char *tableNameList);
 DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
@@ -215,8 +223,12 @@ DLL_EXPORT tmq_list_t *tmq_list_new();
 DLL_EXPORT int32_t     tmq_list_append(tmq_list_t *, const char *);
 DLL_EXPORT void        tmq_list_destroy(tmq_list_t *);
 
+// will be removed in 3.0
 DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
+
+// will replace last one
 DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
+
 DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
 
 /* ------------------------TMQ CONSUMER INTERFACE------------------------ */
@@ -257,18 +269,24 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
 
 DLL_EXPORT TAOS_ROW    tmq_get_row(tmq_message_t *message);
 DLL_EXPORT char       *tmq_get_topic_name(tmq_message_t *message);
-DLL_EXPORT void    *tmq_get_topic_schema(tmq_t *tmq, const char *topic);
+DLL_EXPORT int32_t     tmq_get_vgroup_id(tmq_message_t *message);
+DLL_EXPORT int64_t     tmq_get_request_offset(tmq_message_t *message);
+DLL_EXPORT int64_t     tmq_get_response_offset(tmq_message_t *message);
+DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic);
+DLL_EXPORT int32_t     tmq_field_count(tmq_t *tmq, const char *topic);
 DLL_EXPORT void        tmq_message_destroy(tmq_message_t *tmq_message);
 
 /* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
+#if 0
 DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
+#endif
 
 DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
 
-/* -------------------------------- OTHER -------------------------------- */
+/* ------------------------------ TMQ END -------------------------------- */
+#if 1 // Shuduo: temporary enable for app build
 typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
-
+#endif
-DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);
 
 #ifdef __cplusplus
 }

include/common/tmsg.h

@@ -2215,23 +2215,6 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
   return buf;
 }
 
-typedef struct {
-  int64_t uid;
-  int32_t numOfRows;
-  char*   colData;
-} SMqTbData;
-
-typedef struct {
-  char       topicName[TSDB_TOPIC_FNAME_LEN];
-  int64_t    committedOffset;
-  int64_t    reqOffset;
-  int64_t    rspOffset;
-  int32_t    skipLogNum;
-  int32_t    bodyLen;
-  int32_t    numOfTb;
-  SMqTbData* tbData;
-} SMqTopicData;
-
 typedef struct {
   int8_t  mqMsgType;
   int32_t code;
@@ -2260,7 +2243,10 @@ typedef struct {
 
 typedef struct {
   char        topic[TSDB_TOPIC_FNAME_LEN];
+  int8_t      isSchemaAdaptive;
   SArray*     vgs;  // SArray<SMqSubVgEp>
+  int32_t     numOfFields;
+  TAOS_FIELD* fields;
 } SMqSubTopicEp;
 
 typedef struct {
@@ -2281,32 +2267,6 @@ typedef struct {
   SArray*    topics;  // SArray<SMqSubTopicEp>
 } SMqCMGetSubEpRsp;
 
-typedef struct {
-  int32_t curBlock;
-  int32_t curRow;
-  void**  uData;
-} SMqRowIter;
-
-struct tmq_message_t {
-  SMqPollRsp msg;
-  void*      vg;
-  SMqRowIter iter;
-};
-
-#if 0
-struct tmq_message_t {
-  SMqRspHead head;
-  union {
-    SMqPollRsp       consumeRsp;
-    SMqCMGetSubEpRsp getEpRsp;
-  };
-  void*   extra;
-  int32_t curBlock;
-  int32_t curRow;
-  void**  uData;
-};
-#endif
-
 static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); }
 
 static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
@@ -2331,17 +2291,21 @@ static FORCE_INLINE void tDeleteSMqCMGetSubEpRsp(SMqCMGetSubEpRsp* pRsp) {
 static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) {
   int32_t tlen = 0;
   tlen += taosEncodeString(buf, pTopicEp->topic);
+  tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive);
   int32_t sz = taosArrayGetSize(pTopicEp->vgs);
   tlen += taosEncodeFixedI32(buf, sz);
   for (int32_t i = 0; i < sz; i++) {
     SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i);
     tlen += tEncodeSMqSubVgEp(buf, pVgEp);
   }
+  tlen += taosEncodeFixedI32(buf, pTopicEp->numOfFields);
+  // tlen += taosEncodeBinary(buf, pTopicEp->fields, pTopicEp->numOfFields * sizeof(TAOS_FIELD));
   return tlen;
 }
 
 static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) {
   buf = taosDecodeStringTo(buf, pTopicEp->topic);
+  buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive);
   int32_t sz;
   buf = taosDecodeFixedI32(buf, &sz);
   pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp));
@@ -2353,6 +2317,8 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE
     buf = tDecodeSMqSubVgEp(buf, &vgEp);
     taosArrayPush(pTopicEp->vgs, &vgEp);
   }
+  buf = taosDecodeFixedI32(buf, &pTopicEp->numOfFields);
+  // buf = taosDecodeBinary(buf, (void**)&pTopicEp->fields, pTopicEp->numOfFields * sizeof(TAOS_FIELD));
   return buf;
 }
 

include/common/tmsgcb.h

@@ -25,6 +25,7 @@ extern "C" {
 typedef struct SRpcMsg      SRpcMsg;
 typedef struct SEpSet       SEpSet;
 typedef struct SMgmtWrapper SMgmtWrapper;
+
 typedef enum {
   QUERY_QUEUE,
   FETCH_QUEUE,
@@ -38,9 +39,11 @@ typedef enum {
 
 typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
 typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueType qtype);
-typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, SEpSet* epSet, SRpcMsg* pReq);
+typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, const SEpSet* epSet, SRpcMsg* pReq);
 typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
-typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, SRpcMsg* pRsp);
+typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp);
+typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg);
+typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type);
 
 typedef struct {
   SMgmtWrapper*           pWrapper;
@@ -49,13 +52,17 @@ typedef struct {
   SendReqFp               sendReqFp;
   SendMnodeReqFp          sendMnodeReqFp;
   SendRspFp               sendRspFp;
+  RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
+  ReleaseHandleFp         releaseHandleFp;
 } SMsgCb;
 
 int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq);
 int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype);
-int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq);
+int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq);
 int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq);
-void    tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp);
+void    tmsgSendRsp(const SMsgCb* pMsgCb, const SRpcMsg* pRsp);
+void    tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg);
+void    tmsgReleaseHandle(const SMsgCb* pMsgCb, void* handle, int8_t type);
 
 #ifdef __cplusplus
 }

include/common/ttokendef.h

@@ -139,18 +139,18 @@
 #define TK_TOPIC                          121
 #define TK_AS                             122
 #define TK_NK_BOOL                        123
-#define TK_NK_VARIABLE                    124
+#define TK_NULL                           124
-#define TK_NK_UNDERLINE                   125
+#define TK_NK_VARIABLE                    125
-#define TK_ROWTS                          126
+#define TK_NK_UNDERLINE                   126
-#define TK_TBNAME                         127
+#define TK_ROWTS                          127
-#define TK_QSTARTTS                       128
+#define TK_TBNAME                         128
-#define TK_QENDTS                         129
+#define TK_QSTARTTS                       129
-#define TK_WSTARTTS                       130
+#define TK_QENDTS                         130
-#define TK_WENDTS                         131
+#define TK_WSTARTTS                       131
-#define TK_WDURATION                      132
+#define TK_WENDTS                         132
-#define TK_BETWEEN                        133
+#define TK_WDURATION                      133
-#define TK_IS                             134
+#define TK_BETWEEN                        134
-#define TK_NULL                           135
+#define TK_IS                             135
 #define TK_NK_LT                          136
 #define TK_NK_GT                          137
 #define TK_NK_LE                          138

include/libs/executor/executor.h

@@ -20,8 +20,8 @@
 extern "C" {
 #endif
 
-#include "tcommon.h"
 #include "query.h"
+#include "tcommon.h"
 
 typedef void* qTaskInfo_t;
 typedef void* DataSinkHandle;
@@ -62,7 +62,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type);
  * @param type
  * @return
  */
-int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, void** pBlocks, size_t numOfBlocks, int32_t type);
+int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type);
 
 /**
  * Update the table id list, add or remove.
@@ -167,7 +167,6 @@ int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type);
  */
 void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle);
 
-
 void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
 
 #ifdef __cplusplus

include/libs/nodes/nodes.h

@@ -121,6 +121,7 @@ typedef enum ENodeType {
   QUERY_NODE_LOGIC_PLAN_EXCHANGE,
   QUERY_NODE_LOGIC_PLAN_WINDOW,
   QUERY_NODE_LOGIC_PLAN_SORT,
+  QUERY_NODE_LOGIC_PLAN_PARTITION,
   QUERY_NODE_LOGIC_SUBPLAN,
   QUERY_NODE_LOGIC_PLAN,
 

include/libs/nodes/plannodes.h

@@ -67,6 +67,10 @@ typedef struct SProjectLogicNode {
   SLogicNode node;
   SNodeList* pProjections;
   char stmtName[TSDB_TABLE_NAME_LEN];
+  int64_t limit;
+  int64_t offset;
+  int64_t slimit;
+  int64_t soffset;
 } SProjectLogicNode;
 
 typedef struct SVnodeModifLogicNode {
@@ -106,6 +110,11 @@ typedef struct SSortLogicNode {
   SNodeList* pSortKeys;
 } SSortLogicNode;
 
+typedef struct SPartitionLogicNode {
+  SLogicNode node;
+  SNodeList* pPartitionKeys;
+} SPartitionLogicNode;
+
 typedef enum ESubplanType {
   SUBPLAN_TYPE_MERGE = 1,
   SUBPLAN_TYPE_PARTIAL,
@@ -150,7 +159,8 @@ typedef struct SDataBlockDescNode {
   ENodeType type;
   int16_t dataBlockId;
   SNodeList* pSlots;
-  int32_t resultRowSize;
+  int32_t totalRowSize;
+  int32_t outputRowSize;
   int16_t precision;
 } SDataBlockDescNode;
 
@@ -195,6 +205,10 @@ typedef STableScanPhysiNode STableSeqScanPhysiNode;
 typedef struct SProjectPhysiNode {
   SPhysiNode node;
   SNodeList* pProjections;
+  int64_t limit;
+  int64_t offset;
+  int64_t slimit;
+  int64_t soffset;
 } SProjectPhysiNode;
 
 typedef struct SJoinPhysiNode {
@@ -283,10 +297,17 @@ typedef struct SSubplan {
   SDataSinkNode* pDataSink;    // data of the subplan flow into the datasink
 } SSubplan;
 
+typedef enum EQueryMode {
+  QUERY_MODE_NORMAL = 1,
+  QUERY_MODE_EXPLAIN,
+  QUERY_MODE_EXPLAIN_AN
+} EQueryMode;
+
 typedef struct SQueryPlan {
   ENodeType type;
   uint64_t queryId;
   int32_t numOfSubplans;
+
   SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
 } SQueryPlan;
 

include/libs/nodes/querynodes.h

@@ -180,8 +180,8 @@ typedef struct SOrderByExprNode {
 
 typedef struct SLimitNode {
   ENodeType type; // QUERY_NODE_LIMIT
-  uint64_t limit;
+  int64_t limit;
-  uint64_t offset;
+  int64_t offset;
 } SLimitNode;
 
 typedef struct SStateWindowNode {

include/libs/stream/tstream.h

@@ -30,6 +30,11 @@ enum {
   STREAM_TASK_STATUS__STOP,
 };
 
+enum {
+  STREAM_CREATED_BY__USER = 1,
+  STREAM_CREATED_BY__SMA,
+};
+
 #if 0
 // pipe  -> fetch/pipe queue
 // merge -> merge      queue

include/libs/transport/trpc.h

@@ -50,6 +50,8 @@ typedef struct SRpcMsg {
 
 typedef struct {
   char     user[TSDB_USER_LEN];
+  uint32_t clientIp;
+  uint16_t clientPort;
   SRpcMsg  rpcMsg;
   int32_t  rspLen;
   void    *pRsp;
@@ -105,20 +107,20 @@ void    rpcClose(void *);
 void *  rpcMallocCont(int contLen);
 void    rpcFreeCont(void *pCont);
 void *  rpcReallocCont(void *ptr, int contLen);
-void    rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
-void    rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
 
+// Because taosd supports multi-process mode
+// These functions should not be used on the server side
+// Please use tmsg<xx> functions, which are defined in tmsgcb.h
+void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
 void rpcSendResponse(const SRpcMsg *pMsg);
+void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
+void rpcReleaseHandle(void *handle, int8_t type);  // just release client conn to rpc instance, no close sock
+
+// These functions will not be called in the child process
 void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);
+void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
 int  rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
 void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
-int  rpcReportProgress(void *pConn, char *pCont, int contLen);
-void rpcCancelRequest(int64_t rid);
-void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
-// just release client conn to rpc instance, no close sock
-void rpcReleaseHandle(void *handle, int8_t type);  //
-void rpcRefHandle(void *handle, int8_t type);
-void rpcUnrefHandle(void *handle, int8_t type);
 
 #ifdef __cplusplus
 }

include/util/tprocess.h

@@ -22,11 +22,14 @@
 extern "C" {
 #endif
 
+typedef enum { PROC_REQ, PROC_RSP, PROC_REGISTER } ProcFuncType;
+
 typedef struct SProcQueue SProcQueue;
 typedef struct SProcObj   SProcObj;
 typedef void *(*ProcMallocFp)(int32_t contLen);
 typedef void *(*ProcFreeFp)(void *pCont);
-typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
+typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen,
+                               ProcFuncType ftype);
 
 typedef struct {
   int32_t       childQueueSize;
@@ -52,9 +55,10 @@ int32_t   taosProcRun(SProcObj *pProc);
 void      taosProcStop(SProcObj *pProc);
 bool      taosProcIsChild(SProcObj *pProc);
 int32_t   taosProcChildId(SProcObj *pProc);
-
+int32_t   taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
+                              ProcFuncType ftype);
-int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
+int32_t   taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
+                               ProcFuncType ftype);
 
 #ifdef __cplusplus
 }

source/client/src/clientMain.c

@@ -1,12 +1,27 @@
-#include "os.h"
+/*
-#include "tref.h"
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
-#include "trpc.h"
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "catalog.h"
 #include "clientInt.h"
 #include "clientLog.h"
+#include "os.h"
 #include "query.h"
-#include "tmsg.h"
 #include "tglobal.h"
-#include "catalog.h"
+#include "tmsg.h"
+#include "tref.h"
+#include "trpc.h"
 #include "version.h"
 
 #define TSC_VAR_NOT_RELEASE 1
@@ -117,9 +132,7 @@ int  taos_field_count(TAOS_RES *res) {
   return pResInfo->numOfCols;
 }
 
-int  taos_num_fields(TAOS_RES *res) {
+int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); }
-  return taos_field_count(res);
-}
 
 TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
   if (taos_num_fields(res) == 0) {
@@ -144,10 +157,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
   }
 
   SRequestObj *pRequest = (SRequestObj *)res;
-  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
+  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
-      pRequest->type == TSDB_SQL_INSERT ||
+      pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
-      pRequest->code != TSDB_CODE_SUCCESS ||
-      taos_num_fields(res) == 0) {
     return NULL;
   }
 
@@ -249,20 +260,33 @@ int* taos_fetch_lengths(TAOS_RES *res) {
 // todo intergrate with tDataTypes
 const char *taos_data_type(int type) {
   switch (type) {
-    case TSDB_DATA_TYPE_NULL:            return "TSDB_DATA_TYPE_NULL";
+    case TSDB_DATA_TYPE_NULL:
-    case TSDB_DATA_TYPE_BOOL:            return "TSDB_DATA_TYPE_BOOL";
+      return "TSDB_DATA_TYPE_NULL";
-    case TSDB_DATA_TYPE_TINYINT:         return "TSDB_DATA_TYPE_TINYINT";
+    case TSDB_DATA_TYPE_BOOL:
-    case TSDB_DATA_TYPE_SMALLINT:        return "TSDB_DATA_TYPE_SMALLINT";
+      return "TSDB_DATA_TYPE_BOOL";
-    case TSDB_DATA_TYPE_INT:             return "TSDB_DATA_TYPE_INT";
+    case TSDB_DATA_TYPE_TINYINT:
-    case TSDB_DATA_TYPE_BIGINT:          return "TSDB_DATA_TYPE_BIGINT";
+      return "TSDB_DATA_TYPE_TINYINT";
-    case TSDB_DATA_TYPE_FLOAT:           return "TSDB_DATA_TYPE_FLOAT";
+    case TSDB_DATA_TYPE_SMALLINT:
-    case TSDB_DATA_TYPE_DOUBLE:          return "TSDB_DATA_TYPE_DOUBLE";
+      return "TSDB_DATA_TYPE_SMALLINT";
-    case TSDB_DATA_TYPE_VARCHAR:         return "TSDB_DATA_TYPE_VARCHAR";
+    case TSDB_DATA_TYPE_INT:
+      return "TSDB_DATA_TYPE_INT";
+    case TSDB_DATA_TYPE_BIGINT:
+      return "TSDB_DATA_TYPE_BIGINT";
+    case TSDB_DATA_TYPE_FLOAT:
+      return "TSDB_DATA_TYPE_FLOAT";
+    case TSDB_DATA_TYPE_DOUBLE:
+      return "TSDB_DATA_TYPE_DOUBLE";
+    case TSDB_DATA_TYPE_VARCHAR:
+      return "TSDB_DATA_TYPE_VARCHAR";
       //    case TSDB_DATA_TYPE_BINARY:          return "TSDB_DATA_TYPE_VARCHAR";
-    case TSDB_DATA_TYPE_TIMESTAMP:       return "TSDB_DATA_TYPE_TIMESTAMP";
+    case TSDB_DATA_TYPE_TIMESTAMP:
-    case TSDB_DATA_TYPE_NCHAR:           return "TSDB_DATA_TYPE_NCHAR";
+      return "TSDB_DATA_TYPE_TIMESTAMP";
-    case TSDB_DATA_TYPE_JSON:            return "TSDB_DATA_TYPE_JSON";
+    case TSDB_DATA_TYPE_NCHAR:
-    default: return "UNKNOWN";
+      return "TSDB_DATA_TYPE_NCHAR";
+    case TSDB_DATA_TYPE_JSON:
+      return "TSDB_DATA_TYPE_JSON";
+    default:
+      return "UNKNOWN";
   }
 }
 
@@ -335,10 +359,8 @@ int  taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
   }
 
   SRequestObj *pRequest = (SRequestObj *)res;
-  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
+  if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
-      pRequest->type == TSDB_SQL_INSERT ||
+      pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
-      pRequest->code != TSDB_CODE_SUCCESS ||
-      taos_num_fields(res) == 0) {
     return 0;
   }
 
@@ -352,9 +374,7 @@ int  taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
   return pResultInfo->numOfRows;
 }
 
-int  taos_validate_sql(TAOS *taos, const char *sql) {
+int taos_validate_sql(TAOS *taos, const char *sql) { return true; }
-  return true;
-}
 
 const char *taos_get_server_info(TAOS *taos) {
   if (taos == NULL) {
@@ -437,11 +457,26 @@ int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) {
   return -1;
 }
 
+int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) {
+    // TODO
+    return -1;
+}
+
+int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) {
+    // TODO
+    return -1;
+}
+
 int taos_stmt_add_batch(TAOS_STMT* stmt) {
     // TODO
     return -1;
 }
 
+TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
+    // TODO
+    return NULL;
+}
+
 int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) {
     // TODO
     return -1;

source/client/src/tmq.c

@@ -26,6 +26,18 @@
 #include "tqueue.h"
 #include "tref.h"
 
+typedef struct {
+  int32_t curBlock;
+  int32_t curRow;
+  void**  uData;
+} SMqRowIter;
+
+struct tmq_message_t {
+  SMqPollRsp msg;
+  void*      vg;
+  SMqRowIter iter;
+};
+
 struct tmq_list_t {
   SArray container;
 };
@@ -103,9 +115,10 @@ typedef struct {
   char*       sql;
   char*       topicName;
   int64_t     topicId;
-  int32_t        nextVgIdx;
   SArray*     vgs;  // SArray<SMqClientVg>
-  SSchemaWrapper schema;
+  int8_t      isSchemaAdaptive;
+  int32_t     numOfFields;
+  TAOS_FIELD* fields;
 } SMqClientTopic;
 
 typedef struct {
@@ -131,10 +144,10 @@ typedef struct {
 
 typedef struct {
   tmq_t*         tmq;
-  /*SMqClientVg* pVg;*/
   int32_t        async;
   tsem_t         rspSem;
   tmq_resp_err_t rspErr;
+  /*SMqClientVg* pVg;*/
 } SMqCommitCbParam;
 
 tmq_conf_t* tmq_conf_new() {
@@ -471,7 +484,12 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
     tNameExtractFullName(&name, topicFname);
     tscDebug("subscribe topic: %s", topicFname);
     SMqClientTopic topic = {
-        .nextVgIdx = 0, .sql = NULL, .sqlLen = 0, .topicId = 0, .topicName = topicFname, .vgs = NULL};
+        .sql = NULL,
+        .sqlLen = 0,
+        .topicId = 0,
+        .topicName = topicFname,
+        .vgs = NULL,
+    };
     topic.vgs = taosArrayInit(0, sizeof(SMqClientVg));
     taosArrayPush(tmq->clientTopics, &topic);
     taosArrayPush(req.topicNames, &topicFname);
@@ -615,6 +633,7 @@ _return:
   return pRequest;
 }
 
+#if 0
 TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) {
   STscObj*     pTscObj = (STscObj*)taos;
   SRequestObj* pRequest = NULL;
@@ -700,6 +719,7 @@ _return:
 
   return pRequest;
 }
+#endif
 
 static char* formatTimestamp(char* buf, int64_t val, int precision) {
   time_t  tt;

source/client/test/tmqTest.cpp

@@ -55,8 +55,9 @@ TEST(testCase, create_topic_ctb_Test) {
 
   taos_free_result(pRes);
 
-  char* sql = "select * from tu";
+  // char* sql = "select * from tu";
-  pRes = tmq_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql));
+  // pRes = tmq_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql));
+  pRes = taos_query(pConn, "create test_ctb_topic_1 as select * from tu");
   taos_free_result(pRes);
   taos_close(pConn);
 }
@@ -79,8 +80,9 @@ TEST(testCase, create_topic_stb_Test) {
 
   taos_free_result(pRes);
 
-  char* sql = "select * from st1";
+  // char* sql = "select * from st1";
-  pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
+  // pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
+  pRes = taos_query(pConn, "create test_ctb_topic_1 as select * from st1");
   taos_free_result(pRes);
   taos_close(pConn);
 }

source/common/src/tmsgcb.c

@@ -24,7 +24,7 @@ int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) {
   return (*pMsgCb->qsizeFp)(pMsgCb->pWrapper, vgId, qtype);
 }
 
-int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq) {
+int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) {
   return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq);
 }
 
@@ -32,4 +32,12 @@ int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq) {
   return (*pMsgCb->sendMnodeReqFp)(pMsgCb->pWrapper, pReq);
 }
 
-void tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp) { return (*pMsgCb->sendRspFp)(pMsgCb->pWrapper, pRsp); }
+void tmsgSendRsp(const SMsgCb* pMsgCb, const SRpcMsg* pRsp) { return (*pMsgCb->sendRspFp)(pMsgCb->pWrapper, pRsp); }
+
+void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) {
+  (*pMsgCb->registerBrokenLinkArgFp)(pMsgCb->pWrapper, pMsg);
+}
+
+void tmsgReleaseHandle(const SMsgCb* pMsgCb, void* handle, int8_t type) {
+  (*pMsgCb->releaseHandleFp)(pMsgCb->pWrapper, handle, type);
+}

source/dnode/mgmt/bnode/src/bmInt.c

@@ -24,6 +24,7 @@ static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) {
   msgCb.sendReqFp = dndSendReqToDnode;
   msgCb.sendMnodeReqFp = dndSendReqToMnode;
   msgCb.sendRspFp = dndSendRsp;
+  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
   pOption->msgCb = msgCb;
 }
 

source/dnode/mgmt/container/inc/dnd.h

@@ -137,8 +137,9 @@ void       dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc
 void       dndSendMonitorReport(SDnode *pDnode);
 
 int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
-int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg);
+int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg);
-void    dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp);
+void    dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
+void    dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 
 int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
 int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed);

source/dnode/mgmt/container/inc/dndInt.h

@@ -56,6 +56,7 @@ void    dndCleanupServer(SDnode *pDnode);
 int32_t dndInitClient(SDnode *pDnode);
 void    dndCleanupClient(SDnode *pDnode);
 int32_t dndInitMsgHandle(SDnode *pDnode);
+void    dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/container/src/dndExec.c

@@ -116,11 +116,12 @@ static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) {
   }
 }
 
-static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t msgLen, void *pCont, int32_t contLen) {
+static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
+                                 ProcFuncType ftype) {
   SRpcMsg *pRpc = &pMsg->rpcMsg;
   pRpc->pCont = pCont;
-  dTrace("msg:%p, get from child process queue, type:%s handle:%p app:%p", pMsg, TMSG_INFO(pRpc->msgType),
+  dTrace("msg:%p, get from child queue, type:%s handle:%p app:%p", pMsg, TMSG_INFO(pRpc->msgType), pRpc->handle,
-         pRpc->handle, pRpc->ahandle);
+         pRpc->ahandle);
 
   NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)];
   int32_t   code = (*msgFp)(pWrapper, pMsg);
@@ -138,13 +139,21 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t
   }
 }
 
-static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, int32_t msgLen, void *pCont, int32_t contLen) {
+static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
-  pRpc->pCont = pCont;
+                                  ProcFuncType ftype) {
-  dTrace("msg:%p, get from parent process queue, type:%s handle:%p app:%p", pRpc, TMSG_INFO(pRpc->msgType),
+  pMsg->pCont = pCont;
-         pRpc->handle, pRpc->ahandle);
+  dTrace("msg:%p, get from parent queue, type:%s handle:%p app:%p", pMsg, TMSG_INFO(pMsg->msgType), pMsg->handle,
+         pMsg->ahandle);
 
-  dndSendRsp(pWrapper, pRpc);
+  switch (ftype) {
-  taosMemoryFree(pRpc);
+    case PROC_REGISTER:
+      rpcRegisterBrokenLinkArg(pMsg);
+      break;
+    default:
+      dndSendRpcRsp(pWrapper, pMsg);
+      break;
+  }
+  taosMemoryFree(pMsg);
 }
 
 static int32_t dndRunInMultiProcess(SDnode *pDnode) {

source/dnode/mgmt/container/src/dndMsg.c

@@ -42,6 +42,8 @@ static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) {
   }
 
   memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN);
+  pMsg->clientIp = connInfo.clientIp;
+  pMsg->clientPort = connInfo.clientPort;
   memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg));
   return 0;
 }
@@ -66,7 +68,7 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) {
   } else if (pWrapper->procType == PROC_PARENT) {
     dTrace("msg:%p, is created and will put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle,
            pRpc->ahandle, pMsg->user);
-    code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen);
+    code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ);
   } else {
     dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle,
            pMsg->user);

source/dnode/mgmt/container/src/dndTransport.c

@@ -309,7 +309,7 @@ int32_t dndInitMsgHandle(SDnode *pDnode) {
   return 0;
 }
 
-static int32_t dndSendRpcReq(STransMgmt *pMgmt, SEpSet *pEpSet, SRpcMsg *pReq) {
+static int32_t dndSendRpcReq(STransMgmt *pMgmt, const SEpSet *pEpSet, SRpcMsg *pReq) {
   if (pMgmt->clientRpc == NULL) {
     terrno = TSDB_CODE_DND_OFFLINE;
     return -1;
@@ -319,7 +319,7 @@ static int32_t dndSendRpcReq(STransMgmt *pMgmt, SEpSet *pEpSet, SRpcMsg *pReq) {
   return 0;
 }
 
-int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pReq) {
+int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) {
   if (pWrapper->procType == PROC_CHILD) {
   } else {
     SDnode *pDnode = pWrapper->pDnode;
@@ -348,7 +348,7 @@ int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) {
   }
 }
 
-static void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
+void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) {
   if (pRsp->code == TSDB_CODE_APP_NOT_READY) {
     SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE);
     if (pDnodeWrapper != NULL) {
@@ -362,11 +362,11 @@ static void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
   }
 }
 
-void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
+void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) {
   if (pWrapper->procType == PROC_CHILD) {
     int32_t code = -1;
     do {
-      code = taosProcPutToParentQueue(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen);
+      code = taosProcPutToParentQ(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_RSP);
       if (code != 0) {
         taosMsleep(10);
       }
@@ -375,3 +375,17 @@ void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
     dndSendRpcRsp(pWrapper, pRsp);
   }
 }
+
+void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
+  if (pWrapper->procType == PROC_CHILD) {
+    int32_t code = -1;
+    do {
+      code = taosProcPutToParentQ(pWrapper->pProc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, PROC_REGISTER);
+      if (code != 0) {
+        taosMsleep(10);
+      }
+    } while (code != 0);
+  } else {
+    rpcRegisterBrokenLinkArg(pMsg);
+  }
+}

source/dnode/mgmt/dnode/inc/dm.h

@@ -29,7 +29,7 @@ void dmInitMsgHandles(SMgmtWrapper *pWrapper);
 
 void    dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
 void    dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
-void    dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
+void    dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg);
 
 #ifdef __cplusplus
 }

source/dnode/mgmt/dnode/src/dmInt.c

@@ -54,7 +54,7 @@ void dmGetDnodeEp(SMgmtWrapper *pWrapper, int32_t dnodeId, char *pEp, char *pFqd
   taosRUnLockLatch(&pMgmt->latch);
 }
 
-void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pReq) {
+void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pReq) {
   SDnode *pDnode = pMgmt->pDnode;
 
   SEpSet epSet = {0};

source/dnode/mgmt/mnode/src/mmInt.c

@@ -52,6 +52,7 @@ static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
   msgCb.sendReqFp = dndSendReqToDnode;
   msgCb.sendMnodeReqFp = dndSendReqToMnode;
   msgCb.sendRspFp = dndSendRsp;
+  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
   pOption->msgCb = msgCb;
 }
 

source/dnode/mgmt/qnode/src/qmInt.c

@@ -27,6 +27,7 @@ static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) {
   msgCb.sendReqFp = dndSendReqToDnode;
   msgCb.sendMnodeReqFp = dndSendReqToMnode;
   msgCb.sendRspFp = dndSendRsp;
+  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
   pOption->msgCb = msgCb;
 }
 

source/dnode/mgmt/snode/src/smInt.c

@@ -24,6 +24,7 @@ static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) {
   msgCb.sendReqFp = dndSendReqToDnode;
   msgCb.sendMnodeReqFp = dndSendReqToMnode;
   msgCb.sendRspFp = dndSendRsp;
+  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
   pOption->msgCb = msgCb;
 }
 

source/dnode/mgmt/vnode/src/vmInt.c

@@ -137,6 +137,7 @@ static void *vmOpenVnodeFunc(void *param) {
     msgCb.sendReqFp = dndSendReqToDnode;
     msgCb.sendMnodeReqFp = dndSendReqToMnode;
     msgCb.sendRspFp = dndSendRsp;
+    msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
     SVnodeCfg cfg = {.msgCb = msgCb, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
     SVnode   *pImpl = vnodeOpen(pCfg->path, &cfg);
     if (pImpl == NULL) {

source/dnode/mgmt/vnode/src/vmMsg.c

@@ -91,6 +91,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
   msgCb.sendReqFp = dndSendReqToDnode;
   msgCb.sendMnodeReqFp = dndSendReqToMnode;
   msgCb.sendRspFp = dndSendRsp;
+  msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
 
   vnodeCfg.msgCb = msgCb;
   vnodeCfg.pTfs = pMgmt->pTfs;

source/dnode/mnode/impl/inc/mndDef.h

@@ -731,10 +731,10 @@ typedef struct {
   int32_t  vgNum;
   SRWLatch lock;
   int8_t   status;
-  int8_t   sourceType;
-  int8_t   sinkType;
   // int32_t  sqlLen;
-  int32_t        sinkVgId;  // 0 for automatic
+  int8_t         createdBy;      // STREAM_CREATED_BY__USER or SMA
+  int32_t        fixedSinkVgId;  // 0 for shuffle
+  int64_t        smaId;          // 0 for unused
   char*          sql;
   char*          logicalPlan;
   char*          physicalPlan;

source/dnode/mnode/impl/inc/mndScheduler.h

@@ -27,7 +27,7 @@ void    mndCleanupScheduler(SMnode* pMnode);
 
 int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub);
 
-int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, int64_t smaId);
+int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndStream.h

@@ -31,7 +31,7 @@ void        mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
 SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
 SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
 
-int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans, int64_t smaId);
+int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndDef.c

@@ -26,8 +26,11 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
   if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1;
   if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
   if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
+  if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1;
+  if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
+  if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1;
   if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
-  if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;
+  /*if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;*/
   if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
   // TODO encode tasks
   if (pObj->tasks) {
@@ -69,8 +72,11 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
   if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
   if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
   if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1;
   if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;
-  if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;
+  /*if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;*/
   if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1;
   pObj->tasks = NULL;
   int32_t sz;

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -20,7 +20,8 @@
 #define SYSTABLE_SCH_DB_NAME_LEN    ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
 
 //!!!! Note: only APPEND columns in below tables, NO insert !!!!
-static const SInfosTableSchema dnodesSchema[] = {{.name = "id",             .bytes = 2,   .type = TSDB_DATA_TYPE_SMALLINT},
+static const SInfosTableSchema dnodesSchema[] = {
+    {.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
     {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
     {.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
@@ -28,21 +29,25 @@ static const SInfosTableSchema dnodesSchema[] = {{.name = "id",             .byt
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
 };
-static const SInfosTableSchema mnodesSchema[] = {{.name = "id",             .bytes = 4,   .type = TSDB_DATA_TYPE_INT},
+static const SInfosTableSchema mnodesSchema[] = {
+    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
 };
-static const SInfosTableSchema modulesSchema[] = {{.name = "id",             .bytes = 4,   .type = TSDB_DATA_TYPE_INT},
+static const SInfosTableSchema modulesSchema[] = {
+    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
 };
-static const SInfosTableSchema qnodesSchema[] = {{.name = "id",             .bytes = 4,   .type = TSDB_DATA_TYPE_INT},
+static const SInfosTableSchema qnodesSchema[] = {
+    {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
 };
-static const SInfosTableSchema userDBSchema[] = {{.name = "name",             .bytes = SYSTABLE_SCH_DB_NAME_LEN,   .type = TSDB_DATA_TYPE_BINARY},
+static const SInfosTableSchema userDBSchema[] = {
+    {.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
     {.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
@@ -59,15 +64,18 @@ static const SInfosTableSchema userDBSchema[] = {{.name = "name",             .b
     {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
     {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
     {.name = "precision", .bytes = 3 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
-//                                                 {.name = "update",           .bytes = 1,   .type = TSDB_DATA_TYPE_TINYINT},  // disable update
+    //                                                 {.name = "update",           .bytes = 1,   .type =
+    //                                                 TSDB_DATA_TYPE_TINYINT},  // disable update
 };
-static const SInfosTableSchema userFuncSchema[] = {{.name = "name",           .bytes = 32,   .type = TSDB_DATA_TYPE_BINARY},
+static const SInfosTableSchema userFuncSchema[] = {
+    {.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
 };
-static const SInfosTableSchema userIdxSchema[] = {{.name = "db_name",          .bytes = 32,   .type = TSDB_DATA_TYPE_BINARY},
+static const SInfosTableSchema userIdxSchema[] = {
+    {.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "index_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
@@ -75,7 +83,8 @@ static const SInfosTableSchema userIdxSchema[] = {{.name = "db_name",          .
     {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY},
 };
-static const SInfosTableSchema userStbsSchema[] = {{.name = "stable_name",     .bytes = SYSTABLE_SCH_TABLE_NAME_LEN,  .type = TSDB_DATA_TYPE_BINARY},
+static const SInfosTableSchema userStbsSchema[] = {
+    {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
@@ -84,7 +93,8 @@ static const SInfosTableSchema userStbsSchema[] = {{.name = "stable_name",     .
     {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT},
 };
-static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name",  .bytes = SYSTABLE_SCH_DB_NAME_LEN,  .type = TSDB_DATA_TYPE_BINARY},
+static const SInfosTableSchema userStreamsSchema[] = {
+    {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
@@ -101,7 +111,8 @@ static const SInfosTableSchema userTblsSchema[] = {
     {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
 };
-static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name",                .bytes = 32,   .type = TSDB_DATA_TYPE_BINARY},
+static const SInfosTableSchema userTblDistSchema[] = {
+    {.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
@@ -115,12 +126,14 @@ static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name",
     {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
 };
-static const SInfosTableSchema userUsersSchema[] = {{.name = "name",           .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE,   .type = TSDB_DATA_TYPE_BINARY},
+static const SInfosTableSchema userUsersSchema[] = {
+    {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
     {.name = "account", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
 };
-static const SInfosTableSchema vgroupsSchema[] = {{.name = "vgroup_id",        .bytes = 4,   .type = TSDB_DATA_TYPE_INT},
+static const SInfosTableSchema vgroupsSchema[] = {
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
     {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
@@ -136,7 +149,34 @@ static const SInfosTableSchema vgroupsSchema[] = {{.name = "vgroup_id",        .
     {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
 };
 
-static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)},
+// TODO put into perf schema
+static const SInfosTableSchema topicSchema[] = {
+    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
+    {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+};
+
+static const SInfosTableSchema consumerSchema[] = {
+    {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    // ep
+    // up time
+    // topics
+};
+
+static const SInfosTableSchema subscribeSchema[] = {
+    {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
+    {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
+};
+
+static const SInfosTableMeta infosMeta[] = {
+    {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)},
     {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)},
     {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)},
     {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)},
@@ -251,5 +291,3 @@ void mndCleanupInfos(SMnode *pMnode) {
   pMnode->infosMeta = NULL;
 }
 
-
-

source/dnode/mnode/impl/src/mndProfile.c

@@ -44,7 +44,8 @@ typedef struct {
   SQueryDesc *pQueries;
 } SConnObj;
 
-static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, const char *app, int64_t startTime);
+static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, uint32_t ip, uint16_t port, int32_t pid,
+                               const char *app, int64_t startTime);
 static void      mndFreeConn(SConnObj *pConn);
 static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId);
 static void      mndReleaseConn(SMnode *pMnode, SConnObj *pConn);
@@ -94,7 +95,8 @@ void mndCleanupProfile(SMnode *pMnode) {
   }
 }
 
-static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, const char *app, int64_t startTime) {
+static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, uint32_t ip, uint16_t port, int32_t pid,
+                               const char *app, int64_t startTime) {
   SProfileMgmt *pMgmt = &pMnode->profileMgmt;
 
   int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1);
@@ -104,8 +106,8 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
   SConnObj connObj = {.id = connId,
                       .appStartTimeMs = startTime,
                       .pid = pid,
-                      .ip = pInfo->clientIp,
+                      .ip = ip,
-                      .port = pInfo->clientPort,
+                      .port = port,
                       .killed = 0,
                       .loginTimeMs = taosGetTimestampMs(),
                       .lastAccessTimeMs = 0,
@@ -114,17 +116,17 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
                       .pQueries = NULL};
 
   connObj.lastAccessTimeMs = connObj.loginTimeMs;
-  tstrncpy(connObj.user, pInfo->user, TSDB_USER_LEN);
+  tstrncpy(connObj.user, user, TSDB_USER_LEN);
   tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN);
 
   int32_t   keepTime = tsShellActivityTimer * 3;
   SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000);
   if (pConn == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
-    mError("conn:%d, failed to put into cache since %s, user:%s", connId, pInfo->user, terrstr());
+    mError("conn:%d, failed to put into cache since %s, user:%s", connId, user, terrstr());
     return NULL;
   } else {
-    mTrace("conn:%d, is created, data:%p user:%s", pConn->id, pConn, pInfo->user);
+    mTrace("conn:%d, is created, data:%p user:%s", pConn->id, pConn, user);
     return pConn;
   }
 }
@@ -184,20 +186,14 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
   SConnObj   *pConn = NULL;
   int32_t     code = -1;
   SConnectReq connReq = {0};
+  char        ip[30] = {0};
 
   if (tDeserializeSConnectReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &connReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
     goto CONN_OVER;
   }
 
-  SRpcConnInfo info = {0};
+  taosIp2String(pReq->clientIp, ip);
-  if (rpcGetConnInfo(pReq->rpcMsg.handle, &info) != 0) {
-    mError("user:%s, failed to login while get connection info since %s", pReq->user, terrstr());
-    goto CONN_OVER;
-  }
-
-  char ip[30];
-  taosIp2String(info.clientIp, ip);
 
   pUser = mndAcquireUser(pMnode, pReq->user);
   if (pUser == NULL) {
@@ -216,7 +212,8 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
     }
   }
 
-  pConn = mndCreateConn(pMnode, &info, connReq.pid, connReq.app, connReq.startTime);
+  pConn =
+      mndCreateConn(pMnode, pReq->user, pReq->clientIp, pReq->clientPort, connReq.pid, connReq.app, connReq.startTime);
   if (pConn == NULL) {
     mError("user:%s, failed to login from %s while create connection since %s", pReq->user, ip, terrstr());
     goto CONN_OVER;
@@ -241,7 +238,7 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
   pReq->rspLen = contLen;
   pReq->pRsp = pRsp;
 
-  mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, connReq.app);
+  mDebug("user:%s, login from %s, conn:%d, app:%s", pReq->user, ip, pConn->id, connReq.app);
 
   code = 0;
 

source/dnode/mnode/impl/src/mndQuery.c

@@ -14,15 +14,15 @@
  */
 
 #include "mndQuery.h"
-#include "mndMnode.h"
 #include "executor.h"
+#include "mndMnode.h"
 #include "qworker.h"
 
 int32_t mndProcessQueryMsg(SNodeMsg *pReq) {
-  mTrace("message in query queue is processing");
   SMnode     *pMnode = pReq->pNode;
   SReadHandle handle = {0};
 
+  mTrace("msg:%p, in query queue is processing", pReq);
   switch (pReq->rpcMsg.msgType) {
     case TDMT_VND_QUERY:
       return qWorkerProcessQueryMsg(&handle, pMnode->pQuery, &pReq->rpcMsg);
@@ -35,8 +35,8 @@ int32_t mndProcessQueryMsg(SNodeMsg *pReq) {
 }
 
 int32_t mndProcessFetchMsg(SNodeMsg *pReq) {
-  mTrace("message in fetch queue is processing");
   SMnode *pMnode = pReq->pNode;
+  mTrace("msg:%p, in fetch queue is processing", pReq);
 
   switch (pReq->rpcMsg.msgType) {
     case TDMT_VND_FETCH:
@@ -52,9 +52,9 @@ int32_t mndProcessFetchMsg(SNodeMsg *pReq) {
 }
 
 int32_t mndInitQuery(SMnode *pMnode) {
-  int32_t code = qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb);
+  if (qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) {
-  if (code) {
+    mError("failed to init qworker in mnode since %s", terrstr());
-    return code;
+    return -1;
   }
 
   mndSetMsgHandle(pMnode, TDMT_VND_QUERY, mndProcessQueryMsg);
@@ -67,4 +67,3 @@ int32_t mndInitQuery(SMnode *pMnode) {
 }
 
 void mndCleanupQuery(SMnode *pMnode) { qWorkerDestroy((void **)&pMnode->pQuery); }
-

source/dnode/mnode/impl/src/mndScheduler.c

@@ -119,7 +119,7 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) {
   return pVgroup;
 }
 
-int32_t mndAddSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, int64_t smaId) {
+int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
   SSdb*   pSdb = pMnode->pSdb;
   void*   pIter = NULL;
   SArray* tasks = taosArrayGetP(pStream->tasks, 0);
@@ -151,9 +151,9 @@ int32_t mndAddSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream,
     pTask->execType = TASK_EXEC__NONE;
 
     // sink
-    if (smaId != -1) {
+    if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
       pTask->sinkType = TASK_SINK__SMA;
-      pTask->smaSink.smaId = smaId;
+      pTask->smaSink.smaId = pStream->smaId;
     } else {
       pTask->sinkType = TASK_SINK__TABLE;
     }
@@ -166,7 +166,45 @@ int32_t mndAddSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream,
   return 0;
 }
 
-int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, int64_t smaId) {
+int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
+  ASSERT(pStream->fixedSinkVgId != 0);
+  SArray*      tasks = taosArrayGetP(pStream->tasks, 0);
+  SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+  if (pTask == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  taosArrayPush(tasks, &pTask);
+
+  pTask->nodeId = pStream->fixedSinkVgId;
+  SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId);
+  if (pVgroup == NULL) {
+    return -1;
+  }
+  pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
+  // source
+  pTask->sourceType = TASK_SOURCE__MERGE;
+
+  // exec
+  pTask->execType = TASK_EXEC__NONE;
+
+  // sink
+  if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
+    pTask->sinkType = TASK_SINK__SMA;
+    pTask->smaSink.smaId = pStream->smaId;
+  } else {
+    pTask->sinkType = TASK_SINK__TABLE;
+  }
+  //
+  // dispatch
+  pTask->dispatchType = TASK_DISPATCH__NONE;
+
+  mndPersistTaskDeployReq(pTrans, pTask, &pTask->epSet, TDMT_VND_TASK_DEPLOY, pVgroup->vgId);
+
+  return 0;
+}
+
+int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
   SSdb*       pSdb = pMnode->pSdb;
   SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
   if (pPlan == NULL) {
@@ -185,7 +223,11 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
     taosArrayPush(pStream->tasks, &taskOneLevel);
     // add extra sink
     hasExtraSink = true;
-    mndAddSinkToStream(pMnode, pTrans, pStream, smaId);
+    if (pStream->fixedSinkVgId == 0) {
+      mndAddShuffledSinkToStream(pMnode, pTrans, pStream);
+    } else {
+      mndAddFixedSinkToStream(pMnode, pTrans, pStream);
+    }
   }
 
   for (int32_t level = 0; level < totLevel; level++) {
@@ -221,12 +263,14 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
           pTask->sinkType = TASK_SINK__SHOW;
           pTask->showSink.reserved = 0;
           if (!hasExtraSink) {
-            if (smaId != -1) {
+#if 1
+            if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
               pTask->sinkType = TASK_SINK__SMA;
-              pTask->smaSink.smaId = smaId;
+              pTask->smaSink.smaId = pStream->smaId;
             } else {
               pTask->sinkType = TASK_SINK__TABLE;
             }
+#endif
           }
         } else {
           pTask->sinkType = TASK_SINK__NONE;
@@ -286,6 +330,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
 
       if (hasExtraSink) {
         // add dispatcher
+        if (pStream->fixedSinkVgId == 0) {
           pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
 
           pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
@@ -316,6 +361,17 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
               }
             }
           }
+        } else {
+          pTask->dispatchType = TASK_DISPATCH__FIXED;
+          pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
+          SArray* pArray = taosArrayGetP(pStream->tasks, 0);
+          // one sink only
+          ASSERT(taosArrayGetSize(pArray) == 1);
+          SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0);
+          pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId;
+          pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId;
+          pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet;
+        }
       }
 #endif
 

source/dnode/mnode/impl/src/mndSma.c

@@ -415,6 +415,10 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
   streamObj.dbUid = pDb->uid;
   streamObj.version = 1;
   streamObj.sql = pCreate->sql;
+  streamObj.createdBy = STREAM_CREATED_BY__SMA;
+  // TODO
+  streamObj.fixedSinkVgId = 0;
+  streamObj.smaId = smaObj.uid;
   /*streamObj.physicalPlan = "";*/
   streamObj.logicalPlan = "not implemented";
 
@@ -428,7 +432,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
   if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
   if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
   if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER;
-  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans, smaObj.uid) != 0) goto _OVER;
+  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;

source/dnode/mnode/impl/src/mndStream.c

@@ -246,7 +246,7 @@ static int32_t mndStreamGetPlanString(const char *ast, char **pStr) {
   return code;
 }
 
-int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans, int64_t smaId) {
+int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) {
   SNode *pAst = NULL;
 
   if (nodesStringToNode(ast, &pAst) < 0) {
@@ -271,7 +271,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast
     return -1;
   }
 
-  if (mndScheduleStream(pMnode, pTrans, pStream, smaId) < 0) {
+  if (mndScheduleStream(pMnode, pTrans, pStream) < 0) {
     mError("stream:%ld, schedule stream since %s", pStream->uid, terrstr());
     return -1;
   }
@@ -300,6 +300,10 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
   streamObj.dbUid = pDb->uid;
   streamObj.version = 1;
   streamObj.sql = pCreate->sql;
+  streamObj.createdBy = STREAM_CREATED_BY__USER;
+  // TODO
+  streamObj.fixedSinkVgId = 0;
+  streamObj.smaId = 0;
   /*streamObj.physicalPlan = "";*/
   streamObj.logicalPlan = "not implemented";
 
@@ -310,7 +314,7 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
   }
   mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name);
 
-  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans, -1) != 0) {
+  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) {
     mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr());
     mndTransDrop(pTrans);
     return -1;

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -62,8 +62,6 @@ static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqT
 
 static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp);
 
-static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub);
-
 int32_t mndInitSubscribe(SMnode *pMnode) {
   SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
                      .keyType = SDB_KEY_BINARY,
@@ -98,14 +96,6 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
     return NULL;
   }
 
-#if 0
-  if (mndInitUnassignedVg(pMnode, pTopic, pSub) < 0) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    tDeleteSMqSubscribeObj(pSub);
-    taosMemoryFree(pSub);
-    return NULL;
-  }
-#endif
   // TODO: disable alter subscribed table
   return pSub;
 }
@@ -210,45 +200,6 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq
   return 0;
 }
 
-#if 0
-static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg) {
-  SMnode             *pMnode = pMsg->pNode;
-  uint8_t            *str = pMsg->rpcMsg.pCont;
-  SMqCMResetOffsetReq req;
-
-  SCoder decoder;
-  tCoderInit(&decoder, TD_LITTLE_ENDIAN, str, pMsg->rpcMsg.contLen, TD_DECODER);
-  tDecodeSMqCMResetOffsetReq(&decoder, &req);
-
-  SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
-  if (pHash == NULL) {
-    return -1;
-  }
-
-  for (int32_t i = 0; i < req.num; i++) {
-    SMqOffset    *pOffset = &req.offsets[i];
-    SMqVgOffsets *pVgOffset = taosHashGet(pHash, &pOffset->vgId, sizeof(int32_t));
-    if (pVgOffset == NULL) {
-      pVgOffset = taosMemoryMalloc(sizeof(SMqVgOffsets));
-      if (pVgOffset == NULL) {
-        return -1;
-      }
-      pVgOffset->offsets = taosArrayInit(0, sizeof(void *));
-      taosArrayPush(pVgOffset->offsets, &pOffset);
-    }
-    taosHashPut(pHash, &pOffset->vgId, sizeof(int32_t), &pVgOffset, sizeof(void *));
-  }
-
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
-  if (pTrans == NULL) {
-    mError("mq-reset-offset: failed since %s", terrstr());
-    return -1;
-  }
-
-  return 0;
-}
-#endif
-
 static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
   SMnode           *pMnode = pMsg->pNode;
   SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
@@ -574,251 +525,6 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
   return 0;
 }
 
-#if 0
-      for (int32_t j = 0; j < consumerNum; j++) {
-        bool            changed = false;
-        bool            unfished = false;
-
-        bool            canUseLeft = imbalanceSolved < imbalanceVg;
-        bool            mustUseLeft = canUseLeft && (imbalanceVg - imbalanceSolved >= consumerNum - j);
-        ASSERT(imbalanceVg - imbalanceSolved <= consumerNum - j);
-
-        int32_t maxVg = vgEachConsumer + canUseLeft;
-        int32_t minVg = vgEachConsumer + mustUseLeft;
-
-        SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
-        int32_t         vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
-        int32_t         vgThisConsumerAfterRb;
-        if (vgThisConsumerBeforeRb > maxVg) {
-          vgThisConsumerAfterRb = maxVg; 
-          imbalanceSolved++;
-          changed = true;
-        } else if (vgThisConsumerBeforeRb < minVg) {
-          vgThisConsumerAfterRb = minVg;
-          if (mustUseLeft) imbalanceSolved++;
-          changed = true;
-        } else {
-          vgThisConsumerAfterRb = vgThisConsumerBeforeRb;
-        }
-
-        if (vgThisConsumerBeforeRb > vgThisConsumerAfterRb) {
-          while (taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) {
-            // put into unassigned
-            SMqConsumerEp *pConsumerEp = taosArrayPop(pSubConsumer->vgInfo);
-            ASSERT(pConsumerEp != NULL);
-            ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId);
-            taosArrayPush(unassignedVgStash, pConsumerEp);
-          }
-
-        } else if (vgThisConsumerBeforeRb < vgThisConsumerAfterRb) {
-          // assign from unassigned
-          while (taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerAfterRb) {
-            // if no unassgined, save j
-            if (taosArrayGetSize(unassignedVgStash) == 0) {
-              taosArrayPush(unassignedConsumerIdx, &j);
-              unfished = true;
-              break;
-            }
-            // assign vg to consumer
-            SMqConsumerEp *pConsumerEp = taosArrayPop(unassignedVgStash);
-            ASSERT(pConsumerEp != NULL);
-            pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
-            pConsumerEp->consumerId = pSubConsumer->consumerId;
-            taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
-            // build msg and persist into trans
-          }
-        }
-
-        if (changed && !unfished) {
-          SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
-          pRebConsumer->epoch++;
-          if (vgThisConsumerAfterRb != 0) {
-            atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
-          } else {
-            atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE);
-          }
-          SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
-          sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
-          mndTransAppendRedolog(pTrans, pConsumerRaw);
-          mndReleaseConsumer(pMnode, pRebConsumer);
-          // TODO: save history
-        }
-      }
-
-      for (int32_t j = 0; j < taosArrayGetSize(unassignedConsumerIdx); j++) {
-        bool            canUseLeft = imbalanceSolved < imbalanceVg;
-        int32_t         consumerIdx = *(int32_t *)taosArrayGet(unassignedConsumerIdx, j);
-        SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, consumerIdx);
-        if (canUseLeft) imbalanceSolved++;
-        // must use
-        int32_t vgThisConsumerAfterRb = taosArrayGetSize(pSubConsumer->vgInfo) + canUseLeft;
-        while (taosArrayGetSize(pSubConsumer->vgInfo) < vgEachConsumer + canUseLeft) {
-          // assign vg to consumer
-          SMqConsumerEp *pConsumerEp = taosArrayPop(unassignedVgStash);
-          ASSERT(pConsumerEp != NULL);
-          pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
-          pConsumerEp->consumerId = pSubConsumer->consumerId;
-          taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
-          // build msg and persist into trans
-        }
-        SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
-        pRebConsumer->epoch++;
-        atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
-        SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
-        sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
-        mndTransAppendRedolog(pTrans, pConsumerRaw);
-        mndReleaseConsumer(pMnode, pRebConsumer);
-        // TODO: save history
-      }
-#endif
-
-#if 0
-    //update consumer status for the subscribption
-    for (int32_t i = 0; i < taosArrayGetSize(pSub->assigned); i++) {
-      SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i);
-      int64_t        consumerId = pCEp->consumerId;
-      if (pCEp->status != -1) {
-        int32_t consumerHbStatus = atomic_fetch_add_32(&pCEp->consumerHbStatus, 1);
-        if (consumerHbStatus < MND_SUBSCRIBE_REBALANCE_CNT) {
-          continue;
-        }
-        // put consumer into lostConsumer
-        SMqConsumerEp* lostConsumer = taosArrayPush(pSub->lostConsumer, pCEp);
-        lostConsumer->qmsg = NULL;
-        // put vg into unassigned
-        taosArrayPush(pSub->unassignedVg, pCEp);
-        // remove from assigned
-        // TODO: swap with last one, reduce size and reset i
-        taosArrayRemove(pSub->assigned, i);
-        // remove from available consumer
-        for (int32_t j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) {
-          if (*(int64_t *)taosArrayGet(pSub->availConsumer, i) == pCEp->consumerId) {
-            taosArrayRemove(pSub->availConsumer, j);
-            break;
-          }
-          // TODO: acquire consumer, set status to unavail
-        }
-#if 0
-        SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, consumerId);
-        pConsumer->epoch++;
-        printf("current epoch %ld size %ld", pConsumer->epoch, pConsumer->topics->size);
-        SSdbRaw* pRaw = mndConsumerActionEncode(pConsumer);
-        sdbSetRawStatus(pRaw, SDB_STATUS_READY);
-        sdbWriteNotFree(pMnode->pSdb, pRaw);
-        mndReleaseConsumer(pMnode, pConsumer);
-#endif
-      }
-    }
-    // no available consumer, skip rebalance
-    if (taosArrayGetSize(pSub->availConsumer) == 0) {
-      continue;
-    }
-    taosArrayGet(pSub->availConsumer, 0);
-    // rebalance condition1 : have unassigned vg
-    // assign vg to a consumer, trying to find the least assigned one
-    if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0) {
-      char *topic = NULL;
-      char *cgroup = NULL;
-      mndSplitSubscribeKey(pSub->key, &topic, &cgroup);
-
-      SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
-      STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg);
-      for (int32_t i = 0; i < sz; i++) {
-        int64_t        consumerId = *(int64_t *)taosArrayGet(pSub->availConsumer, pSub->nextConsumerIdx);
-        pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer);
-
-        SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg);
-        pCEp->oldConsumerId = pCEp->consumerId;
-        pCEp->consumerId = consumerId;
-        taosArrayPush(pSub->assigned, pCEp);
-
-        SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
-        pConsumer->epoch++;
-        SSdbRaw* pConsumerRaw = mndConsumerActionEncode(pConsumer);
-        sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
-        sdbWrite(pMnode->pSdb, pConsumerRaw);
-        mndReleaseConsumer(pMnode, pConsumer);
-
-        void* msg;
-        int32_t msgLen;
-        mndBuildRebalanceMsg(&msg, &msgLen, pTopic, pCEp, cgroup, topic);
-
-        // persist msg
-        STransAction action = {0};
-        action.epSet = pCEp->epSet;
-        action.pCont = msg;
-        action.contLen = sizeof(SMsgHead) + msgLen;
-        action.msgType = TDMT_VND_MQ_SET_CONN;
-        mndTransAppendRedoAction(pTrans, &action);
-
-        // persist data
-        SSdbRaw *pRaw = mndSubActionEncode(pSub);
-        sdbSetRawStatus(pRaw, SDB_STATUS_READY);
-        mndTransAppendRedolog(pTrans, pRaw);
-      }
-
-      if (mndTransPrepare(pMnode, pTrans) != 0) {
-        mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
-      }
-      mndReleaseTopic(pMnode, pTopic);
-      mndTransDrop(pTrans);
-      taosMemoryFreeClear(topic);
-      taosMemoryFreeClear(cgroup);
-    }
-    // rebalance condition2 : imbalance assignment
-  }
-  return 0;
-}
-#endif
-
-#if 0
-static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) {
-  SSdb      *pSdb = pMnode->pSdb;
-  SVgObj    *pVgroup = NULL;
-  SQueryPlan *pPlan = qStringToQueryPlan(pTopic->physicalPlan);
-  SArray    *pArray = NULL;
-  SNodeListNode    *inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
-  SSubplan  *plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
-  SArray    *unassignedVg = pSub->unassignedVg;
-
-  void *pIter = NULL;
-  while (1) {
-    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
-    if (pIter == NULL) break;
-    if (pVgroup->dbUid != pTopic->dbUid) {
-      sdbRelease(pSdb, pVgroup);
-      continue;
-    }
-
-    pSub->vgNum++;
-    plan->execNode.nodeId = pVgroup->vgId;
-    plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup);
-
-    if (schedulerConvertDagToTaskList(pPlan, &pArray) < 0) {
-      terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
-      mError("unsupport topic: %s, sql: %s", pTopic->name, pTopic->sql);
-      return -1;
-    }
-
-    SMqConsumerEp consumerEp = {0};
-    consumerEp.status = 0;
-    consumerEp.consumerId = -1;
-    STaskInfo *pTaskInfo = taosArrayGet(pArray, 0);
-    consumerEp.epSet = pTaskInfo->addr.epset;
-    consumerEp.vgId = pTaskInfo->addr.nodeId;
-
-    ASSERT(consumerEp.vgId == pVgroup->vgId);
-    consumerEp.qmsg = strdup(pTaskInfo->msg->msg);
-    taosArrayPush(unassignedVg, &consumerEp);
-    // TODO: free taskInfo
-    taosArrayDestroy(pArray);
-  }
-
-  /*qDestroyQueryDag(pDag);*/
-  return 0;
-}
-#endif
-
 static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
                                       const SMqConsumerEp *pConsumerEp) {
   ASSERT(pConsumerEp->oldConsumerId == -1);

source/dnode/mnode/impl/src/mndTopic.c

@@ -67,7 +67,9 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
 
   int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1;
   int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
-  int32_t  size = sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + MND_TOPIC_RESERVE_SIZE;
+  int32_t swLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema);
+  int32_t size =
+      sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + swLen + MND_TOPIC_RESERVE_SIZE;
   SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size);
   if (pRaw == NULL) goto TOPIC_ENCODE_OVER;
 
@@ -86,7 +88,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
   SDB_SET_INT32(pRaw, dataPos, physicalPlanLen, TOPIC_ENCODE_OVER);
   SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER);
 
-  int32_t swLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema);
   void *swBuf = taosMemoryMalloc(swLen);
   if (swBuf == NULL) {
     goto TOPIC_ENCODE_OVER;

source/dnode/mnode/impl/src/mndTrans.c

@@ -54,7 +54,7 @@ static bool    mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans);
 static bool    mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans);
 
 static void    mndTransExecute(SMnode *pMnode, STrans *pTrans);
-static void    mndTransSendRpcRsp(STrans *pTrans);
+static void    mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
 static int32_t mndProcessTransReq(SNodeMsg *pReq);
 static int32_t mndProcessKillTransReq(SNodeMsg *pReq);
 
@@ -737,7 +737,7 @@ static int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) {
   return 0;
 }
 
-static void mndTransSendRpcRsp(STrans *pTrans) {
+static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
   bool sendRsp = false;
 
   if (pTrans->stage == TRN_STAGE_FINISHED) {
@@ -771,7 +771,7 @@ static void mndTransSendRpcRsp(STrans *pTrans) {
                       .ahandle = pTrans->rpcAHandle,
                       .pCont = rpcCont,
                       .contLen = pTrans->rpcRspLen};
-    rpcSendResponse(&rspMsg);
+    tmsgSendRsp(&pMnode->msgCb, &rspMsg);
     pTrans->rpcHandle = NULL;
     pTrans->rpcRsp = NULL;
     pTrans->rpcRspLen = 0;
@@ -1158,7 +1158,7 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
     }
   }
 
-  mndTransSendRpcRsp(pTrans);
+  mndTransSendRpcRsp(pMnode, pTrans);
 }
 
 static int32_t mndProcessTransReq(SNodeMsg *pReq) {

source/dnode/vnode/src/tq/tq.c

@@ -275,6 +275,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
     pMsg->pCont = NULL;
     pMsg->contLen = 0;
     pMsg->code = -1;
+    ASSERT(0);
     rpcSendResponse(pMsg);
     return 0;
   }
@@ -356,6 +357,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
   void*   buf = rpcMallocCont(tlen);
   if (buf == NULL) {
     pMsg->code = -1;
+    ASSERT(0);
     return -1;
   }
   ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;

source/dnode/vnode/src/tq/tqRead.c

@@ -114,10 +114,9 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
   SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
 
   int32_t numOfRows = pHandle->pBlock->numOfRows;
-  int32_t numOfCols = pHandle->pSchema->numOfCols;
+  /*int32_t numOfCols = pHandle->pSchema->numOfCols;*/
   int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
 
-  // TODO: stable case
   if (colNumNeed > pSchemaWrapper->nCols) {
     colNumNeed = pSchemaWrapper->nCols;
   }
@@ -138,56 +137,24 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
       colNeed++;
     } else {
       SColumnInfoData colInfo = {0};
-      int             sz = numOfRows * pColSchema->bytes;
+      /*int             sz = numOfRows * pColSchema->bytes;*/
       colInfo.info.bytes = pColSchema->bytes;
       colInfo.info.colId = pColSchema->colId;
       colInfo.info.type = pColSchema->type;
 
-      colInfo.pData = taosMemoryCalloc(1, sz);
+      if (blockDataEnsureColumnCapacity(&colInfo, numOfRows) < 0) {
-      if (colInfo.pData == NULL) {
+        taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock);
-        // TODO free
-        taosArrayDestroy(pArray);
         return NULL;
       }
-
-      blockDataEnsureColumnCapacity(&colInfo, numOfRows);
       taosArrayPush(pArray, &colInfo);
       colMeta++;
       colNeed++;
     }
   }
 
-  int j = 0;
-  for (int32_t i = 0; i < colNumNeed; i++) {
-    col_id_t colId = *(col_id_t*)taosArrayGet(pHandle->pColIdList, i);
-    while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) {
-      j++;
-    }
-    if (j >= pSchemaWrapper->nCols) {
-      continue;
-    }
-    SSchema*        pColSchema = &pSchemaWrapper->pSchema[j];
-    SColumnInfoData colInfo = {0};
-    int             sz = numOfRows * pColSchema->bytes;
-    colInfo.info.bytes = pColSchema->bytes;
-    colInfo.info.colId = colId;
-    colInfo.info.type = pColSchema->type;
-
-    colInfo.pData = taosMemoryCalloc(1, sz);
-    if (colInfo.pData == NULL) {
-      // TODO free
-      taosArrayDestroy(pArray);
-      return NULL;
-    }
-
-    blockDataEnsureColumnCapacity(&colInfo, numOfRows);
-    taosArrayPush(pArray, &colInfo);
-  }
-
   STSRowIter iter = {0};
   tdSTSRowIterInit(&iter, pTschema);
   STSRow* row;
-  // int32_t kvIdx = 0;
   int32_t curRow = 0;
   tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter);
   while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
@@ -200,25 +167,9 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
       if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
         break;
       }
-      memcpy(POINTER_SHIFT(pColData->pData, curRow * pColData->info.bytes), sVal.val, pColData->info.bytes);
+      // TODO handle null
+      colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL);
     }
-#if 0
-    for (int32_t i = 0; i < colNumNeed; i++) {
-      SColumnInfoData* pColData = taosArrayGet(pArray, i);
-      STColumn*        pCol = schemaColAt(pTschema, i);
-      // TODO
-      if(pCol->colId != pColData->info.colId) {
-        continue;
-      }
-      // void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx);
-      SCellVal sVal = {0};
-      if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) {
-        // TODO: reach end
-        break;
-      }
-      memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes);
-    }
-#endif
     curRow++;
   }
   return pArray;

source/libs/executor/src/executor.c

@@ -14,12 +14,12 @@
  */
 
 #include "executor.h"
-#include "tdatablock.h"
 #include "executorimpl.h"
 #include "planner.h"
+#include "tdatablock.h"
 #include "vnode.h"
 
-static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void** input, size_t numOfBlocks, int32_t type, char* id) {
+static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) {
   ASSERT(pOperator != NULL);
   if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
     if (pOperator->numOfDownstream == 0) {
@@ -44,17 +44,18 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void** input, size_t n
     }
 
     if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
-      if (tqReadHandleSetMsg(pInfo->readerHandle, input[0], 0) < 0) {
+      if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) {
         qError("submit msg messed up when initing stream block, %s" PRIx64, id);
         return TSDB_CODE_QRY_APP_ERROR;
       }
     } else {
       for (int32_t i = 0; i < numOfBlocks; ++i) {
-        SSDataBlock* pDataBlock = input[i];
+        SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i];
 
         SSDataBlock* p = createOneDataBlock(pDataBlock);
         p->info = pDataBlock->info;
 
+        taosArrayClear(p->pDataBlock);
         taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock);
         taosArrayPush(pInfo->pBlockLists, &p);
       }
@@ -65,10 +66,10 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void** input, size_t n
 }
 
 int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type) {
-  qSetMultiStreamInput(tinfo, (void**) &input, 1, type);
+  return qSetMultiStreamInput(tinfo, input, 1, type);
 }
 
-int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, void** pBlocks, size_t numOfBlocks, int32_t type) {
+int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) {
   if (tinfo == NULL) {
     return TSDB_CODE_QRY_APP_ERROR;
   }

source/libs/index/inc/index_fst.h

@@ -21,9 +21,9 @@ extern "C" {
 #endif
 
 #include "indexInt.h"
-#include "index_fst_node.h"
 #include "index_fst_automation.h"
 #include "index_fst_counting_writer.h"
+#include "index_fst_node.h"
 #include "index_fst_registry.h"
 #include "index_fst_util.h"
 
@@ -325,10 +325,10 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
 FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut);
 
 void fstStreamBuilderDestroy(FstStreamBuilder* b);
-// set up bound range
-// refator, simple code by marco
 
-FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type);
+// set up bound range
+// refator later:  to simple code by marco
+void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type);
 
 #ifdef __cplusplus
 }

source/libs/index/src/index_fst.c

@@ -1184,7 +1184,7 @@ StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstB
   sws->aut = automation;
   sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t));
 
-  sws->emptyOutput.null = false;
+  sws->emptyOutput.null = true;
   sws->emptyOutput.out = 0;
 
   sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
@@ -1239,8 +1239,8 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
   for (uint32_t i = 0; i < len; i++) {
     uint8_t  b = data[i];
     uint64_t res = 0;
-    bool     null = fstNodeFindInput(node, b, &res);
+    bool     find = fstNodeFindInput(node, b, &res);
-    if (null == false) {
+    if (find == true) {
       FstTransition trn;
       fstNodeGetTransitionAt(node, res, &trn);
       void* preState = autState;
@@ -1293,7 +1293,6 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
 
   return false;
 }
-
 StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) {
   AutomationCtx* aut = sws->aut;
   FstOutput      output = sws->emptyOutput;
@@ -1356,6 +1355,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
       sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
       taosMemoryFreeClear(buf);
       fstSliceDestroy(&slice);
+      taosArrayDestroy(nodes);
       return NULL;
     }
     if (FST_NODE_IS_FINAL(nextNode) && isMatch) {
@@ -1364,15 +1364,12 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
       taosMemoryFreeClear(buf);
       fstSliceDestroy(&slice);
       taosArrayDestroy(nodes);
+      nodes = NULL;
       return result;
     }
     taosMemoryFreeClear(buf);
     fstSliceDestroy(&slice);
-  }
+  };
-  for (size_t i = 0; i < taosArrayGetSize(nodes); i++) {
-    FstNode** node = (FstNode**)taosArrayGet(nodes, i);
-    fstNodeDestroy(*node);
-  }
   taosArrayDestroy(nodes);
   return NULL;
 }
@@ -1425,9 +1422,9 @@ void fstStreamBuilderDestroy(FstStreamBuilder* b) {
   taosMemoryFreeClear(b->max);
   taosMemoryFree(b);
 }
-FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
+void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
   if (b == NULL) {
-    return NULL;
+    return;
   }
   if (type == GE) {
     b->min->type = Included;
@@ -1446,5 +1443,4 @@ FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, Rang
     fstSliceDestroy(&(b->max->data));
     b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1);
   }
-  return b;
 }

source/libs/index/src/index_fst_automation.c

@@ -85,10 +85,20 @@ static void* prefixStart(AutomationCtx* ctx) {
 };
 static bool prefixIsMatch(AutomationCtx* ctx, void* sv) {
   StartWithStateValue* ssv = (StartWithStateValue*)sv;
+  if (ssv == NULL) {
+    return false;
+  }
+  if (ssv->type == FST_INT) {
     return ssv->val == strlen(ctx->data);
+  } else {
+    return false;
+  }
 }
 static bool prefixCanMatch(AutomationCtx* ctx, void* sv) {
   StartWithStateValue* ssv = (StartWithStateValue*)sv;
+  if (ssv == NULL) {
+    return false;
+  }
   return ssv->val >= 0;
 }
 static bool  prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
@@ -154,15 +164,7 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
     // add more search type
   }
 
-  char* dst = NULL;
+  ctx->data = (data != NULL ? strdup((char*)data) : NULL);
-  if (data != NULL) {
-    char*  src = (char*)data;
-    size_t len = strlen(src);
-    dst = (char*)taosMemoryCalloc(1, len * sizeof(char) + 1);
-    memcpy(dst, src, len);
-  }
-
-  ctx->data = dst;
   ctx->type = atype;
   ctx->stdata = (void*)sv;
   return ctx;

source/libs/index/test/fstTest.cc

@@ -96,11 +96,41 @@ class FstReadMemory {
       char*       ch = (char*)fstSliceData(s, &sz);
       std::string key(ch, sz);
       printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
+      result.push_back(rt->out.out);
       swsResultDestroy(rt);
     }
-    for (size_t i = 0; i < result.size(); i++) {
+    streamWithStateDestroy(st);
+    fstStreamBuilderDestroy(sb);
+    return true;
   }
-    std::cout << std::endl;
+  bool SearchRange(AutomationCtx* ctx, const std::string& low, RangeType lowType, const std::string& high,
+                   RangeType highType, std::vector<uint64_t>& result) {
+    FstStreamBuilder* sb = fstSearch(_fst, ctx);
+
+    FstSlice l = fstSliceCreate((uint8_t*)low.c_str(), low.size());
+    FstSlice h = fstSliceCreate((uint8_t*)high.c_str(), high.size());
+
+    // range [low, high);
+    fstStreamBuilderSetRange(sb, &l, lowType);
+    fstStreamBuilderSetRange(sb, &h, highType);
+
+    fstSliceDestroy(&l);
+    fstSliceDestroy(&h);
+
+    StreamWithState*       st = streamBuilderIntoStream(sb);
+    StreamWithStateResult* rt = NULL;
+    while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
+      // result.push_back((uint64_t)(rt->out.out));
+      FstSlice*   s = &rt->data;
+      int32_t     sz = 0;
+      char*       ch = (char*)fstSliceData(s, &sz);
+      std::string key(ch, sz);
+      printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
+      result.push_back(rt->out.out);
+      swsResultDestroy(rt);
+    }
+    streamWithStateDestroy(st);
+    fstStreamBuilderDestroy(sb);
     return true;
   }
   bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
@@ -233,7 +263,7 @@ void checkFstLongTerm() {
   // taosMemoryFree(ctx);
   // delete m;
 }
-void checkFstCheckIterator() {
+void checkFstCheckIterator1() {
   FstWriter* fw = new FstWriter;
   int64_t    s = taosGetTimestampUs();
   int        count = 2;
@@ -243,8 +273,7 @@ void checkFstCheckIterator() {
   std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
 
   fw->Put("Hello world", 1);
-  fw->Put("hello world", 2);
+  fw->Put("Hello worle", 2);
-  fw->Put("hello worle", 3);
   fw->Put("hello worlf", 4);
   delete fw;
 
@@ -258,15 +287,227 @@ void checkFstCheckIterator() {
   // prefix search
   std::vector<uint64_t> result;
 
-  AutomationCtx* ctx = automCtxCreate((void*)"H", AUTOMATION_PREFIX);
+  AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
   m->Search(ctx, result);
   std::cout << "size: " << result.size() << std::endl;
   // assert(result.size() == count);
   for (int i = 0; i < result.size(); i++) {
     // assert(result[i] == i);  // check result
   }
+  automCtxDestroy(ctx);
 
-  taosMemoryFree(ctx);
+  delete m;
+}
+void checkFstCheckIterator2() {
+  FstWriter* fw = new FstWriter;
+  int64_t    s = taosGetTimestampUs();
+  int        count = 2;
+  // Performance_fstWriteRecords(fw);
+  int64_t e = taosGetTimestampUs();
+
+  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
+
+  fw->Put("a", 1);
+  fw->Put("b", 2);
+  fw->Put("c", 4);
+  delete fw;
+
+  FstReadMemory* m = new FstReadMemory(1024 * 64);
+  if (m->init() == false) {
+    std::cout << "init readMemory failed" << std::endl;
+    delete m;
+    return;
+  }
+
+  // prefix search
+  std::vector<uint64_t> result;
+
+  AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
+  m->Search(ctx, result);
+  std::cout << "size: " << result.size() << std::endl;
+  // assert(result.size() == count);
+  for (int i = 0; i < result.size(); i++) {
+    // assert(result[i] == i);  // check result
+  }
+  automCtxDestroy(ctx);
+
+  delete m;
+}
+void checkFstCheckIteratorPrefix() {
+  FstWriter* fw = new FstWriter;
+  int64_t    s = taosGetTimestampUs();
+  int        count = 2;
+  // Performance_fstWriteRecords(fw);
+  int64_t e = taosGetTimestampUs();
+
+  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
+
+  fw->Put("Hello world", 1);
+  fw->Put("Hello worle", 2);
+  fw->Put("hello worlf", 4);
+  fw->Put("ja", 4);
+  fw->Put("jb", 4);
+  fw->Put("jc", 4);
+  fw->Put("jddddddddd", 4);
+  fw->Put("jefffffff", 4);
+  delete fw;
+
+  FstReadMemory* m = new FstReadMemory(1024 * 64);
+  if (m->init() == false) {
+    std::cout << "init readMemory failed" << std::endl;
+    delete m;
+    return;
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+
+    AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_PREFIX);
+    m->Search(ctx, result);
+    assert(result.size() == 1);
+    automCtxDestroy(ctx);
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+
+    AutomationCtx* ctx = automCtxCreate((void*)"Hello", AUTOMATION_PREFIX);
+    m->Search(ctx, result);
+    assert(result.size() == 2);
+    automCtxDestroy(ctx);
+  }
+  {
+    std::vector<uint64_t> result;
+
+    AutomationCtx* ctx = automCtxCreate((void*)"jddd", AUTOMATION_PREFIX);
+    m->Search(ctx, result);
+    assert(result.size() == 1);
+    automCtxDestroy(ctx);
+  }
+  delete m;
+}
+void checkFstCheckIteratorRange1() {
+  FstWriter* fw = new FstWriter;
+  int64_t    s = taosGetTimestampUs();
+  int        count = 2;
+  // Performance_fstWriteRecords(fw);
+  int64_t e = taosGetTimestampUs();
+
+  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
+
+  fw->Put("a", 1);
+  fw->Put("b", 2);
+  fw->Put("c", 3);
+  fw->Put("d", 4);
+  fw->Put("e", 5);
+  fw->Put("f", 5);
+  fw->Put("G", 5);
+  delete fw;
+
+  FstReadMemory* m = new FstReadMemory(1024 * 64);
+  if (m->init() == false) {
+    std::cout << "init readMemory failed" << std::endl;
+    delete m;
+    return;
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GE, "e", LT, result);
+    assert(result.size() == 3);
+    automCtxDestroy(ctx);
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GT, "e", LT, result);
+    assert(result.size() == 2);
+    automCtxDestroy(ctx);
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GT, "e", LE, result);
+    assert(result.size() == 3);
+    automCtxDestroy(ctx);
+  }
+  {
+    // prefix search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GE, "e", LE, result);
+    assert(result.size() == 4);
+    automCtxDestroy(ctx);
+  }
+  delete m;
+}
+void checkFstCheckIteratorRange2() {
+  FstWriter* fw = new FstWriter;
+  int64_t    s = taosGetTimestampUs();
+  int        count = 2;
+  // Performance_fstWriteRecords(fw);
+  int64_t e = taosGetTimestampUs();
+
+  std::cout << "insert data count :  " << count << "elapas time: " << e - s << std::endl;
+
+  fw->Put("ab", 1);
+  fw->Put("b", 2);
+  fw->Put("cdd", 3);
+  fw->Put("cde", 3);
+  fw->Put("ddd", 4);
+  fw->Put("ed", 5);
+  delete fw;
+
+  FstReadMemory* m = new FstReadMemory(1024 * 64);
+  if (m->init() == false) {
+    std::cout << "init readMemory failed" << std::endl;
+    delete m;
+    return;
+  }
+  {
+    // range  search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GE, "ed", LT, result);
+    assert(result.size() == 4);
+    automCtxDestroy(ctx);
+  }
+  {
+    // range  search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GE, "ed", LE, result);
+    assert(result.size() == 5);
+    automCtxDestroy(ctx);
+    // taosMemoryFree(ctx);
+  }
+  {
+    // range  search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GT, "ed", LE, result);
+    assert(result.size() == 4);
+    automCtxDestroy(ctx);
+  }
+  {
+    // range  search
+    std::vector<uint64_t> result;
+    AutomationCtx*        ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
+    // [b, e)
+    m->SearchRange(ctx, "b", GT, "ed", LT, result);
+    assert(result.size() == 3);
+    automCtxDestroy(ctx);
+  }
   delete m;
 }
 
@@ -332,7 +573,11 @@ int main(int argc, char* argv[]) {
   // path suid colName ver
   // iterTFileReader(argv[1], argv[2], argv[3], argv[4]);
   //}
-  checkFstCheckIterator();
+  checkFstCheckIterator1();
+  checkFstCheckIterator2();
+  checkFstCheckIteratorPrefix();
+  checkFstCheckIteratorRange1();
+  checkFstCheckIteratorRange2();
   // checkFstLongTerm();
   // checkFstPrefixSearch();
 

source/libs/index/test/fstUT.cc

@@ -13,9 +13,9 @@
 #include "index_fst_util.h"
 #include "index_tfile.h"
 #include "tglobal.h"
+#include "tlog.h"
 #include "tskiplist.h"
 #include "tutil.h"
-#include "tlog.h"
 
 static std::string dir = "/tmp/index";
 

source/libs/nodes/src/nodesCloneFuncs.c

@@ -260,9 +260,9 @@ static SNode* logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) {
 }
 
 static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* pDst) {
+  COPY_ALL_SCALAR_FIELDS;
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
   CLONE_NODE_LIST_FIELD(pProjections);
-  COPY_CHAR_ARRAY_FIELD(stmtName);
   return (SNode*)pDst;
 }
 
@@ -307,10 +307,8 @@ static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) {
 }
 
 static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) {
-  COPY_SCALAR_FIELD(dataBlockId);
+  COPY_ALL_SCALAR_FIELDS;
   CLONE_NODE_LIST_FIELD(pSlots);
-  COPY_SCALAR_FIELD(resultRowSize);
-  COPY_SCALAR_FIELD(precision);
   return (SNode*)pDst;
 }
 

source/libs/nodes/src/nodesCodeFuncs.c

@@ -475,6 +475,10 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
 }
 
 static const char* jkProjectLogicPlanProjections = "Projections";
+static const char* jkProjectLogicPlanLimit = "Limit";
+static const char* jkProjectLogicPlanOffset = "Offset";
+static const char* jkProjectLogicPlanSlimit = "SLimit";
+static const char* jkProjectLogicPlanSoffset = "SOffset";
 
 static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
   const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj;
@@ -483,6 +487,18 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkProjectLogicPlanProjections, pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanLimit, pNode->limit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanOffset, pNode->offset);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanSlimit, pNode->slimit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, pNode->soffset);
+  }
 
   return code;
 }
@@ -494,6 +510,18 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkProjectLogicPlanProjections, &pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanLimit, &pNode->limit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanOffset, &pNode->offset);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanSlimit, &pNode->slimit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkScanLogicPlanTableMetaSize, &pNode->soffset);
+  }
 
   return code;
 }
@@ -802,6 +830,10 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) {
 }
 
 static const char* jkProjectPhysiPlanProjections = "Projections";
+static const char* jkProjectPhysiPlanLimit = "Limit";
+static const char* jkProjectPhysiPlanOffset = "Offset";
+static const char* jkProjectPhysiPlanSlimit = "SLimit";
+static const char* jkProjectPhysiPlanSoffset = "SOffset";
 
 static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
   const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj;
@@ -810,6 +842,18 @@ static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkProjectPhysiPlanProjections, pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanLimit, pNode->limit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanOffset, pNode->offset);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanSlimit, pNode->slimit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanSoffset, pNode->soffset);
+  }
 
   return code;
 }
@@ -821,6 +865,18 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkProjectPhysiPlanProjections, &pNode->pProjections);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanLimit, &pNode->limit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanOffset, &pNode->offset);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanSlimit, &pNode->slimit);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanSoffset, &pNode->soffset);
+  }
 
   return code;
 }
@@ -2057,19 +2113,26 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) {
 
 static const char* jkDataBlockDescDataBlockId = "DataBlockId";
 static const char* jkDataBlockDescSlots = "Slots";
-static const char* jkDataBlockResultRowSize = "ResultRowSize";
+static const char* jkDataBlockTotalRowSize = "TotalRowSize";
+static const char* jkDataBlockOutputRowSize = "OutputRowSize";
+static const char* jkDataBlockPrecision = "Precision";
 
 static int32_t dataBlockDescNodeToJson(const void* pObj, SJson* pJson) {
   const SDataBlockDescNode* pNode = (const SDataBlockDescNode*)pObj;
 
   int32_t code = tjsonAddIntegerToObject(pJson, jkDataBlockDescDataBlockId, pNode->dataBlockId);
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddIntegerToObject(pJson, jkDataBlockResultRowSize, pNode->resultRowSize);
+    code = tjsonAddIntegerToObject(pJson, jkDataBlockTotalRowSize, pNode->totalRowSize);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDataBlockOutputRowSize, pNode->outputRowSize);
   }
-
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkDataBlockDescSlots, pNode->pSlots);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkDataBlockPrecision, pNode->precision);
+  }
 
   return code;
 }
@@ -2079,12 +2142,17 @@ static int32_t jsonToDataBlockDescNode(const SJson* pJson, void* pObj) {
 
   int32_t code = tjsonGetSmallIntValue(pJson, jkDataBlockDescDataBlockId, &pNode->dataBlockId);
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonGetIntValue(pJson, jkDataBlockResultRowSize, &pNode->resultRowSize);
+    code = tjsonGetIntValue(pJson, jkDataBlockTotalRowSize, &pNode->totalRowSize);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetIntValue(pJson, jkDataBlockOutputRowSize, &pNode->outputRowSize);
   }
-
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkDataBlockDescSlots, &pNode->pSlots);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetSmallIntValue(pJson, jkDataBlockPrecision, &pNode->precision);
+  }
 
   return code;
 }

source/libs/nodes/src/nodesUtilFuncs.c

@@ -582,11 +582,11 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char*
   nodesWalkSelectStmt(pSelect, clause, collectColumns, &cxt);
   taosHashCleanup(cxt.pColIdHash);
   if (TSDB_CODE_SUCCESS != cxt.errCode) {
-    nodesDestroyList(cxt.pCols);
+    nodesClearList(cxt.pCols);
     return cxt.errCode;
   }
   if (0 == LIST_LENGTH(cxt.pCols)) {
-    nodesDestroyList(cxt.pCols);
+    nodesClearList(cxt.pCols);
     cxt.pCols = NULL;
   }
   *pCols = cxt.pCols;

source/libs/parser/inc/sql.y

@@ -349,6 +349,7 @@ literal(A) ::= NK_STRING(B).
 literal(A) ::= NK_BOOL(B).                                                        { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B)); }
 literal(A) ::= TIMESTAMP(B) NK_STRING(C).                                         { A = createRawExprNodeExt(pCxt, &B, &C, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &C)); }
 literal(A) ::= duration_literal(B).                                               { A = B; }
+literal(A) ::= NULL(B).                                                           { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL)); }
 
 duration_literal(A) ::= NK_VARIABLE(B).                                           { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
 
@@ -372,6 +373,7 @@ signed_literal(A) ::= NK_STRING(B).
 signed_literal(A) ::= NK_BOOL(B).                                                 { A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); }
 signed_literal(A) ::= TIMESTAMP NK_STRING(B).                                     { A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); }
 signed_literal(A) ::= duration_literal(B).                                        { A = releaseRawExprNode(pCxt, B); }
+signed_literal(A) ::= NULL.                                                       { A = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); }
 
 %type literal_list                                                                { SNodeList* }
 %destructor literal_list                                                          { nodesDestroyList($$); }

source/libs/parser/src/parAstCreater.c

@@ -577,11 +577,13 @@ SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pC
 SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral) {
   SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
   CHECK_OUT_OF_MEM(val);
+  if (NULL != pLiteral) {
     val->literal = strndup(pLiteral->z, pLiteral->n);
     if (TK_NK_ID != pLiteral->type && (IS_VAR_DATA_TYPE(dataType) || TSDB_DATA_TYPE_TIMESTAMP == dataType)) {
       trimString(pLiteral->z, pLiteral->n, val->literal, pLiteral->n);
     }
     CHECK_OUT_OF_MEM(val->literal);
+  }
   val->node.resType.type = dataType;
   val->node.resType.bytes = IS_VAR_DATA_TYPE(dataType) ? strlen(val->literal) : tDataTypes[dataType].bytes;
   if (TSDB_DATA_TYPE_TIMESTAMP == dataType) {
@@ -717,8 +719,10 @@ SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft
 SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) {
   SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT);
   CHECK_OUT_OF_MEM(limitNode);
-  // limitNode->limit = limit;
+  limitNode->limit = strtol(pLimit->z, NULL, 10);
-  // limitNode->offset = offset;
+  if (NULL != pOffset) {
+    limitNode->offset = strtol(pOffset->z, NULL, 10);
+  }
   return (SNode*)limitNode;
 }
 

source/libs/parser/src/parTokenizer.c

@@ -99,6 +99,7 @@ static SKeyword keywordTable[] = {
     {"ON",            TK_ON},
     {"OR",            TK_OR},
     {"ORDER",         TK_ORDER},
+    {"PARTITION",     TK_PARTITION},
     {"PASS",          TK_PASS},
     {"PORT",          TK_PORT},
     {"PRECISION",     TK_PRECISION},

source/libs/parser/src/parTranslater.c

@@ -235,6 +235,8 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol
   nodesListAppend(pExpr->pAssociationList, (SNode*)pCol);
   if (NULL != pTable) {
     strcpy(pCol->tableAlias, pTable->tableAlias);
+  } else if (QUERY_NODE_COLUMN == nodeType(pExpr)) {
+    strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias);
   }
   strcpy(pCol->colName, pExpr->aliasName);
   pCol->node.resType = pExpr->resType;
@@ -617,7 +619,6 @@ static int32_t toVgroupsInfo(SArray* pVgs, SVgroupsInfo** pVgsInfo) {
 }
 
 static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
-  // todo release
   if (0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) {
     return TSDB_CODE_SUCCESS;
   }
@@ -631,11 +632,6 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea
   }
 
   if (TSDB_CODE_SUCCESS == code) {
-    // todo remove
-    //if (NULL != vgroupList && taosArrayGetSize(vgroupList) > 0 && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) {
-    //  taosArrayPopTailBatch(vgroupList, taosArrayGetSize(vgroupList) - 1);
-    //}
-
     code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList);
   }
   taosArrayDestroy(vgroupList);
@@ -1488,12 +1484,6 @@ static int32_t nodeTypeToShowType(ENodeType nt) {
 
 static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) {
   SShowReq showReq = { .type = nodeTypeToShowType(nodeType(pStmt)) };
-  // if ('\0' != pStmt->dbName[0]) {
-  //   SName name = {0};
-  //   tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
-  //   char dbFname[TSDB_DB_FNAME_LEN] = {0};
-  //   tNameGetFullDbName(&name, showReq.db);
-  // }
 
   pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo));
   if (NULL == pCxt->pCmdMsg) {

source/libs/parser/test/parserAstTest.cpp

@@ -488,6 +488,9 @@ TEST_F(ParserTest, createTable) {
        "if not exists test.t1 using test.st1 (tag1, tag2) tags(1, 'abc') "
        "if not exists test.t2 using test.st1 (tag1, tag2) tags(2, 'abc') "
        "if not exists test.t3 using test.st1 (tag1, tag2) tags(3, 'abc') "
+       "if not exists test.t4 using test.st1 (tag1, tag2) tags(3, null) "
+       "if not exists test.t5 using test.st1 (tag1, tag2) tags(null, 'abc') "
+       "if not exists test.t6 using test.st1 (tag1, tag2) tags(null, null)"
       );
   ASSERT_TRUE(run());
 

source/libs/planner/src/planLogicCreater.c

@@ -142,6 +142,46 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanCols, STab
   return SCAN_TYPE_TAG;
 }
 
+static SNodeptr createPrimaryKeyCol(uint64_t tableId) {
+  SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
+  if (NULL == pCol) {
+    return NULL;
+  }
+  pCol->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
+  pCol->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
+  pCol->tableId = tableId;
+  pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
+  pCol->colType = COLUMN_TYPE_COLUMN;
+  strcpy(pCol->colName, "#primarykey");
+  return pCol;
+}
+
+static int32_t addPrimaryKeyCol(uint64_t tableId, SNodeList** pCols) {
+  if (NULL == *pCols) {
+    *pCols = nodesMakeList();
+    if (NULL == *pCols) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+
+  bool found = false;
+  SNode* pCol = NULL;
+  FOREACH(pCol, *pCols) {
+    if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pCol)->colId) {
+      found = true;
+      break;
+    }
+  }
+
+  if (!found) {
+    if (TSDB_CODE_SUCCESS != nodesListStrictAppend(*pCols, createPrimaryKeyCol(tableId))) {
+      nodesDestroyList(*pCols);
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) {
   SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN);
   if (NULL == pScan) {
@@ -161,7 +201,11 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
   // set columns to scan
   SNodeList* pCols = NULL;
   int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols);
-  if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
+  if (TSDB_CODE_SUCCESS == code) {
+    code = addPrimaryKeyCol(pScan->pMeta->uid, &pCols);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
     pScan->pScanCols = nodesCloneList(pCols);
     if (NULL == pScan) {
       code = TSDB_CODE_OUT_OF_MEMORY;
@@ -171,13 +215,15 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
   pScan->scanType = getScanType(pCxt, pCols, pScan->pMeta);
 
   // set output
-  if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
+  if (TSDB_CODE_SUCCESS == code) {
     pScan->node.pTargets = nodesCloneList(pCols);
     if (NULL == pScan) {
       code = TSDB_CODE_OUT_OF_MEMORY;
     }
   }
 
+  nodesClearList(pCols);
+
   if (TSDB_CODE_SUCCESS == code) {
     *pLogicNode = (SLogicNode*)pScan;
   } else {
@@ -548,6 +594,22 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  if (NULL != pSelect->pLimit) {
+    pProject->limit = ((SLimitNode*)pSelect->pLimit)->limit;
+    pProject->offset = ((SLimitNode*)pSelect->pLimit)->offset;
+  } else {
+    pProject->limit = -1;
+    pProject->offset = -1;
+  }
+
+  if (NULL != pSelect->pSlimit) {
+    pProject->slimit = ((SLimitNode*)pSelect->pSlimit)->limit;
+    pProject->soffset = ((SLimitNode*)pSelect->pSlimit)->offset;
+  } else {
+    pProject->slimit = -1;
+    pProject->soffset = -1;
+  }
+
   int32_t code = TSDB_CODE_SUCCESS;
 
   pProject->pProjections = nodesCloneList(pSelect->pProjectionList);
@@ -569,15 +631,42 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel
   return code;
 }
 
+static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
+  if (NULL == pSelect->pPartitionByList) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SPartitionLogicNode* pPartition = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PARTITION);
+  if (NULL == pPartition) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
+  if (!pSelect->isDistinct) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
   SLogicNode* pRoot = NULL;
   int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot);
   if (TSDB_CODE_SUCCESS == code) {
     code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createChildLogicNode(pCxt, pSelect, createPartitionLogicNode, &pRoot);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createChildLogicNode(pCxt, pSelect, createDistinctLogicNode, &pRoot);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = createChildLogicNode(pCxt, pSelect, createSortLogicNode, &pRoot);
   }

source/libs/planner/src/planPhysiCreater.c

@@ -131,7 +131,8 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD
       code = putSlotToHash(pDataBlockDesc->dataBlockId, slotId, pNode, pHash);
     }
     if (TSDB_CODE_SUCCESS == code) {
-      pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes;
+      pDataBlockDesc->totalRowSize += ((SExprNode*)pNode)->resType.bytes;
+      pDataBlockDesc->outputRowSize += ((SExprNode*)pNode)->resType.bytes;
       ++slotId;
     } else {
       break;
@@ -189,7 +190,10 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList,
       if (TSDB_CODE_SUCCESS == code) {
         code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash);
       }
-      pDataBlockDesc->resultRowSize += ((SExprNode*)pExpr)->resType.bytes;
+      pDataBlockDesc->totalRowSize += ((SExprNode*)pExpr)->resType.bytes;
+      if (output) {
+        pDataBlockDesc->outputRowSize += ((SExprNode*)pExpr)->resType.bytes;
+      }
       slotId = nextSlotId;
       ++nextSlotId;
     } else {
@@ -311,20 +315,6 @@ static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pL
   return TSDB_CODE_SUCCESS;
 }
 
-static SNodeptr createPrimaryKeyCol(SPhysiPlanContext* pCxt, uint64_t tableId) {
-  SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
-  if (NULL == pCol) {
-    return NULL;
-  }
-  pCol->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
-  pCol->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
-  pCol->tableId = tableId;
-  pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
-  pCol->colType = COLUMN_TYPE_COLUMN;
-  strcpy(pCol->colName, "#primarykey");
-  return pCol;
-}
-
 static int32_t colIdCompare(const void* pLeft, const void* pRight) {
   SColumnNode* pLeftCol = *(SColumnNode**)pLeft;
   SColumnNode* pRightCol = *(SColumnNode**)pRight;
@@ -353,35 +343,10 @@ static int32_t sortScanCols(SNodeList* pScanCols) {
 }
 
 static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode, SNodeList* pScanCols) {
-  if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanPhysiNode)
-      || QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN == nodeType(pScanPhysiNode)) {
-    pScanPhysiNode->pScanCols = nodesMakeList();
-    if (NULL == pScanPhysiNode->pScanCols) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-
-    SNode* pNode;
-    FOREACH(pNode, pScanCols) {
-      if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) {
-        SColumnNode* pCol = nodesListGetNode(pScanPhysiNode->pScanCols, 0);
-        strcpy(pCol->tableAlias, ((SColumnNode*)pNode)->tableAlias);
-        strcpy(pCol->colName, ((SColumnNode*)pNode)->colName);
-        continue;
-      }
-      if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pScanPhysiNode->pScanCols, nodesCloneNode(pNode))) {
-        return TSDB_CODE_OUT_OF_MEMORY;
-      }
-    }
-  } else {
   pScanPhysiNode->pScanCols = nodesCloneList(pScanCols);
   if (NULL == pScanPhysiNode->pScanCols) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  }
-
   return sortScanCols(pScanPhysiNode->pScanCols);
 }
 
@@ -708,6 +673,11 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  pProject->limit = pProjectLogicNode->limit;
+  pProject->offset = pProjectLogicNode->offset;
+  pProject->slimit = pProjectLogicNode->slimit;
+  pProject->soffset = pProjectLogicNode->soffset;
+
   int32_t code = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, pProjectLogicNode->pProjections, &pProject->pProjections);
   if (TSDB_CODE_SUCCESS == code) {
     code = addDataBlockSlotsForProject(pCxt, pProjectLogicNode->stmtName, pProject->pProjections, pProject->node.pOutputDataBlockDesc);

source/libs/planner/test/plannerTest.cpp

@@ -207,7 +207,7 @@ TEST_F(PlannerTest, sessionWindow) {
 TEST_F(PlannerTest, orderBy) {
   setDatabase("root", "test");
 
-  bind("SELECT * FROM t1 order by c1");
+  bind("SELECT c1 FROM t1 order by c1");
   ASSERT_TRUE(run());
 
   bind("SELECT c1 FROM t1 order by c2");
@@ -217,6 +217,32 @@ TEST_F(PlannerTest, orderBy) {
   ASSERT_TRUE(run());
 }
 
+TEST_F(PlannerTest, limit) {
+  setDatabase("root", "test");
+
+  bind("SELECT * FROM t1 limit 2");
+  ASSERT_TRUE(run());
+
+  bind("SELECT * FROM t1 limit 5 offset 2");
+  ASSERT_TRUE(run());
+
+  bind("SELECT * FROM t1 limit 2, 5");
+  ASSERT_TRUE(run());
+}
+
+TEST_F(PlannerTest, slimit) {
+  setDatabase("root", "test");
+
+  bind("SELECT * FROM t1 partition by c1 slimit 2");
+  ASSERT_TRUE(run());
+
+  bind("SELECT * FROM t1 partition by c1 slimit 5 soffset 2");
+  ASSERT_TRUE(run());
+
+  bind("SELECT * FROM t1 partition by c1 slimit 2, 5");
+  ASSERT_TRUE(run());
+}
+
 TEST_F(PlannerTest, showTables) {
   setDatabase("root", "test");
 

source/libs/qworker/inc/qworkerMsg.h

@@ -30,21 +30,20 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg);
 int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg);
 int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req);
 
-int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code);
+int32_t qwBuildAndSendDropRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code);
-int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code);
+int32_t qwBuildAndSendCancelRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code);
-int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code);
+int32_t qwBuildAndSendFetchRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength,
+                               int32_t code);
 void    qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete);
 int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn);
-int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code);
+int32_t qwBuildAndSendReadyRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code);
-int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code);
+int32_t qwBuildAndSendQueryRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code);
 void    qwFreeFetchRsp(void *msg);
 int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp);
 int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp);
-int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *rsp, int32_t code);
+int32_t qwBuildAndSendHbRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, SSchedulerHbRsp *rsp, int32_t code);
 int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn);
 
-
-
 #ifdef __cplusplus
 }
 #endif

source/libs/qworker/src/qworker.c

@@ -764,7 +764,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
         QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
         dropConnection = NULL;
 
-        qwBuildAndSendDropRsp(&ctx->connInfo, code);    
+        qwBuildAndSendDropRsp(&mgmt->msgCb, &ctx->connInfo, code);
         QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code));
 
         QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
@@ -802,7 +802,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
         QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
         dropConnection = NULL;
 
-        qwBuildAndSendDropRsp(&ctx->connInfo, code);
+        qwBuildAndSendDropRsp(&mgmt->msgCb, &ctx->connInfo, code);
         QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code));
 
         QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
@@ -830,12 +830,12 @@ _return:
   }
 
   if (dropConnection) {
-    qwBuildAndSendDropRsp(dropConnection, code);    
+    qwBuildAndSendDropRsp(&mgmt->msgCb, dropConnection, code);
     QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", dropConnection->handle, code, tstrerror(code));
   }
 
   if (cancelConnection) {
-    qwBuildAndSendCancelRsp(cancelConnection, code);    
+    qwBuildAndSendCancelRsp(&mgmt->msgCb, cancelConnection, code);
     QW_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", cancelConnection->handle, code, tstrerror(code));
   }
 
@@ -886,7 +886,7 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp
       QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
     }
 
-    qwBuildAndSendDropRsp(&ctx->connInfo, code);    
+    qwBuildAndSendDropRsp(&mgmt->msgCb, &ctx->connInfo, code);
     QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code));
 
     QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
@@ -918,7 +918,7 @@ _return:
   }
 
   if (readyConnection) {
-    qwBuildAndSendReadyRsp(readyConnection, code);    
+    qwBuildAndSendReadyRsp(&mgmt->msgCb, readyConnection, code);    
     QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", readyConnection->handle, code, tstrerror(code));
   }
 
@@ -970,7 +970,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) {
     QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
   }
 
-  QW_ERR_JRET(qwBuildAndSendQueryRsp(&qwMsg->connInfo, code));
+  QW_ERR_JRET(qwBuildAndSendQueryRsp(&mgmt->msgCb, &qwMsg->connInfo, code));
   QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
 
   queryRsped = true;
@@ -988,7 +988,7 @@ _return:
   code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL);
 
   if (!queryRsped) {
-    qwBuildAndSendQueryRsp(&qwMsg->connInfo, code);
+    qwBuildAndSendQueryRsp(&mgmt->msgCb, &qwMsg->connInfo, code);
     QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
   }
 
@@ -1051,7 +1051,7 @@ _return:
   }
 
   if (needRsp) {
-    qwBuildAndSendReadyRsp(&qwMsg->connInfo, code);
+    qwBuildAndSendReadyRsp(&mgmt->msgCb, &qwMsg->connInfo, code);
     QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
   }
 
@@ -1095,7 +1095,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
         qwMsg->connInfo = ctx->connInfo;
         QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);            
         
-        qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code);                
+        qwBuildAndSendFetchRsp(&mgmt->msgCb, &qwMsg->connInfo, rsp, dataLen, code);                
         QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen);
       } else {
         atomic_store_8((int8_t*)&ctx->queryContinue, 1);
@@ -1114,7 +1114,7 @@ _return:
       rsp = NULL;
       
       qwMsg->connInfo = ctx->connInfo;
-      qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, 0, code);
+      qwBuildAndSendFetchRsp(&mgmt->msgCb, &qwMsg->connInfo, rsp, 0, code);
       QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0);
     }
 
@@ -1195,7 +1195,7 @@ _return:
   }
 
   if (code || rsp) {
-    qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code);
+    qwBuildAndSendFetchRsp(&mgmt->msgCb, &qwMsg->connInfo, rsp, dataLen, code);
     QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen);
   }
 
@@ -1226,7 +1226,7 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
     QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx));
     qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING);
   } else if (ctx->phase > 0) {
-    qwBuildAndSendDropRsp(&qwMsg->connInfo, code);
+    qwBuildAndSendDropRsp(&mgmt->msgCb, &qwMsg->connInfo, code);
     QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
 
     QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
@@ -1261,7 +1261,7 @@ _return:
   }
 
   if (TSDB_CODE_SUCCESS != code) {
-    qwBuildAndSendDropRsp(&qwMsg->connInfo, code);
+    qwBuildAndSendDropRsp(&mgmt->msgCb, &qwMsg->connInfo, code);
     QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
   }
 
@@ -1297,7 +1297,7 @@ int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
 
 _return:
 
-  qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code);
+  qwBuildAndSendHbRsp(&mgmt->msgCb, &qwMsg->connInfo, &rsp, code);
   QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
   
   QW_RET(TSDB_CODE_SUCCESS);
@@ -1351,8 +1351,9 @@ _return:
   QW_UNLOCK(QW_READ, &mgmt->schLock);
 
   for (int32_t j = 0; j < i; ++j) {
-    qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code);
+    qwBuildAndSendHbRsp(&mgmt->msgCb, &rspList[j].connInfo, &rspList[j].rsp, code);
-    QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code), (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));    
+    QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code),
+            (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));
     tFreeSSchedulerHbRsp(&rspList[j].rsp);
   }
 

source/libs/qworker/src/qworkerMsg.c

@@ -46,7 +46,7 @@ void qwFreeFetchRsp(void *msg) {
   }
 }
 
-int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code) {
+int32_t qwBuildAndSendQueryRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code) {
   SQueryTableRsp rsp = {.code = code};
   
   int32_t contLen = tSerializeSQueryTableRsp(NULL, 0, &rsp);
@@ -62,12 +62,12 @@ int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code) {
     .code    = code,
   };
 
-  rpcSendResponse(&rpcRsp);
+  tmsgSendRsp(pMsgCb, &rpcRsp);
 
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) {
+int32_t qwBuildAndSendReadyRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code) {
   SResReadyRsp *pRsp = (SResReadyRsp *)rpcMallocCont(sizeof(SResReadyRsp));
   pRsp->code = code;
 
@@ -80,12 +80,12 @@ int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) {
     .code    = code,
   };
 
-  rpcSendResponse(&rpcRsp);
+  tmsgSendRsp(pMsgCb, &rpcRsp);
 
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) {
+int32_t qwBuildAndSendHbRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) {
   int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus);
   void *pRsp = rpcMallocCont(contLen);
   tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus);
@@ -99,12 +99,12 @@ int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_
     .code    = code,
   };
 
-  rpcSendResponse(&rpcRsp);
+  tmsgSendRsp(pMsgCb, &rpcRsp);
 
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) {
+int32_t qwBuildAndSendFetchRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) {
   if (NULL == pRsp) {
     pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
     memset(pRsp, 0, sizeof(SRetrieveTableRsp));
@@ -120,12 +120,12 @@ int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int3
     .code    = code,
   };
 
-  rpcSendResponse(&rpcRsp);
+  tmsgSendRsp(pMsgCb, &rpcRsp);
 
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code) {
+int32_t qwBuildAndSendCancelRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code) {
   STaskCancelRsp *pRsp = (STaskCancelRsp *)rpcMallocCont(sizeof(STaskCancelRsp));
   pRsp->code = code;
 
@@ -138,11 +138,11 @@ int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code) {
     .code    = code,
   };
 
-  rpcSendResponse(&rpcRsp);
+  tmsgSendRsp(pMsgCb, &rpcRsp);
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code) {
+int32_t qwBuildAndSendDropRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code) {
   STaskDropRsp *pRsp = (STaskDropRsp *)rpcMallocCont(sizeof(STaskDropRsp));
   pRsp->code = code;
 
@@ -155,11 +155,11 @@ int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code) {
     .code    = code,
   };
 
-  rpcSendResponse(&rpcRsp);
+  tmsgSendRsp(pMsgCb, &rpcRsp);
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) {
+int32_t qwBuildAndSendShowRsp(const SMsgCb *pMsgCb, SRpcMsg *pMsg, int32_t code) {
   int32_t         numOfCols = 6;
   SVShowTablesRsp showRsp = {0};
 
@@ -210,11 +210,11 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) {
       .code = code,
   };
 
-  rpcSendResponse(&rpcMsg);
+  tmsgSendRsp(pMsgCb, &rpcMsg);
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchReq) {
+int32_t qwBuildAndSendShowFetchRsp(const SMsgCb *pMsgCb, SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchReq) {
   SVShowTablesFetchRsp *pRsp = (SVShowTablesFetchRsp *)rpcMallocCont(sizeof(SVShowTablesFetchRsp));
   int32_t handle = htonl(pFetchReq->id);
 
@@ -227,7 +227,7 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe
       .code    = 0,
   };
 
-  rpcSendResponse(&rpcMsg);
+  tmsgSendRsp(pMsgCb, &rpcMsg);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -287,7 +287,7 @@ int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) {
     .code    = TSDB_CODE_RPC_NETWORK_UNAVAIL,
   };
   
-  rpcRegisterBrokenLinkArg(&pMsg);
+  tmsgRegisterBrokenLinkArg(&mgmt->msgCb, &pMsg);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -498,7 +498,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
 
 _return:
 
-  QW_ERR_RET(qwBuildAndSendCancelRsp(&qwMsg.connInfo, code));
+  QW_ERR_RET(qwBuildAndSendCancelRsp(&mgmt->msgCb, &qwMsg.connInfo, code));
   QW_SCH_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", qwMsg.connInfo.handle, code, tstrerror(code));
 
   return TSDB_CODE_SUCCESS;
@@ -579,15 +579,16 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
   if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
     return TSDB_CODE_QRY_INVALID_INPUT;
   }
 
+  SQWorkerMgmt *pMgmt = qWorkerMgmt;
+
   int32_t code = 0;
   SVShowTablesReq *pReq = pMsg->pCont;
-  QW_RET(qwBuildAndSendShowRsp(pMsg, code));
+  QW_RET(qwBuildAndSendShowRsp(&pMgmt->msgCb, pMsg, code));
 }
 
 int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
@@ -595,8 +596,8 @@ int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg)
     return TSDB_CODE_QRY_INVALID_INPUT;
   }
 
+  SQWorkerMgmt *pMgmt = qWorkerMgmt;
+
   SVShowTablesFetchReq *pFetchReq = pMsg->pCont;
-  QW_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq));
+  QW_RET(qwBuildAndSendShowFetchRsp(&pMgmt->msgCb, pMsg, pFetchReq));
 }
-
-

source/libs/stream/src/tstream.c

@@ -74,7 +74,7 @@ static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg*
   pMsg->contLen = tlen;
   pMsg->code = 0;
   pMsg->msgType = pTask->dispatchMsgType;
-  /*pMsg->noResp = 1;*/
+  pMsg->noResp = 1;
 
   return 0;
 }
@@ -124,10 +124,11 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in
       }
     } else if (inputType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
       const SArray* blocks = (const SArray*)input;
-      int32_t       sz = taosArrayGetSize(blocks);
+      /*int32_t       sz = taosArrayGetSize(blocks);*/
-      for (int32_t i = 0; i < sz; i++) {
+      /*for (int32_t i = 0; i < sz; i++) {*/
-        SSDataBlock* pBlock = taosArrayGet(blocks, i);
+      /*SSDataBlock* pBlock = taosArrayGet(blocks, i);*/
-        qSetStreamInput(exec, pBlock, inputType);
+      /*qSetStreamInput(exec, pBlock, inputType);*/
+      qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
       while (1) {
         SSDataBlock* output;
         uint64_t     ts;
@@ -139,7 +140,7 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in
         }
         taosArrayPush(pRes, output);
       }
-      }
+      /*}*/
     } else {
       ASSERT(0);
     }

source/libs/transport/src/transSrv.c

@@ -214,9 +214,11 @@ static void uvHandleReq(SSrvConn* pConn) {
     // no ref here
   }
 
-  if (pHead->noResp == 0) {
+  // if pHead->noResp = 1,
+  // 1. server application should not send resp on handle
+  // 2. once send out data, cli conn released to conn pool immediately
+  // 3. not mixed with persist
   transMsg.handle = pConn;
-  }
 
   STrans* pTransInst = pConn->pTransInst;
   (*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);

source/util/src/tprocess.c

@@ -24,7 +24,6 @@
 #include <sys/wait.h>
 
 #define SHM_DEFAULT_SIZE (20 * 1024 * 1024)
-#define CEIL8(n)         (ceil((float)(n) / 8) * 8)
 typedef void *(*ProcThreadFp)(void *param);
 
 typedef struct SProcQueue {
@@ -58,6 +57,11 @@ typedef struct SProcObj {
   bool        stopFlag;
 } SProcObj;
 
+static inline int32_t CEIL8(int32_t v) {
+  const int32_t c = ceil((float)(v) / 8) * 8;
+  return c < 8 ? 8 : c;
+}
+
 static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) {
   TdThreadMutex    *pMutex = NULL;
   TdThreadMutexAttr mattr = {0};
@@ -203,7 +207,8 @@ static void taosProcCleanupQueue(SProcQueue *pQueue) {
   }
 }
 
-static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHeadLen, char *pBody, int32_t rawBodyLen) {
+static int32_t taosProcQueuePush(SProcQueue *pQueue, const char *pHead, int16_t rawHeadLen, const char *pBody,
+                                 int32_t rawBodyLen, ProcFuncType funcType) {
   const int32_t headLen = CEIL8(rawHeadLen);
   const int32_t bodyLen = CEIL8(rawBodyLen);
   const int32_t fullLen = headLen + bodyLen + 8;
@@ -221,10 +226,12 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHea
   }
 
   if (pQueue->tail < pQueue->total) {
-    *(int32_t *)(pQueue->pBuffer + pQueue->head) = headLen;
+    *(int16_t *)(pQueue->pBuffer + pQueue->head) = headLen;
+    *(int8_t *)(pQueue->pBuffer + pQueue->head + 2) = (int8_t)funcType;
     *(int32_t *)(pQueue->pBuffer + pQueue->head + 4) = bodyLen;
   } else {
-    *(int32_t *)(pQueue->pBuffer) = headLen;
+    *(int16_t *)(pQueue->pBuffer) = headLen;
+    *(int8_t *)(pQueue->pBuffer + pQueue->head + 2) = (int8_t)funcType;
     *(int32_t *)(pQueue->pBuffer + 4) = bodyLen;
   }
 
@@ -264,13 +271,13 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHea
   taosThreadMutexUnlock(pQueue->mutex);
   tsem_post(&pQueue->sem);
 
-  uTrace("proc:%s, push msg to queue:%p remains:%d, head:%d:%p body:%d:%p", pQueue->name, pQueue, pQueue->items,
+  uTrace("proc:%s, push msg to queue:%p remains:%d, head:%d:%p body:%d:%p ftype:%d", pQueue->name, pQueue, pQueue->items,
-         headLen, pHead, bodyLen, pBody);
+         headLen, pHead, bodyLen, pBody, funcType);
   return 0;
 }
 
-static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHeadLen, void **ppBody,
+static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHeadLen, void **ppBody,
-                                int32_t *pBodyLen) {
+                                int32_t *pBodyLen, ProcFuncType *pFuncType) {
   tsem_wait(&pQueue->sem);
 
   taosThreadMutexLock(pQueue->mutex);
@@ -281,13 +288,16 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHea
     return 0;
   }
 
-  int32_t headLen = 0;
+  int16_t headLen = 0;
+  int8_t  ftype = 0;
   int32_t bodyLen = 0;
   if (pQueue->head < pQueue->total) {
-    headLen = *(int32_t *)(pQueue->pBuffer + pQueue->head);
+    headLen = *(int16_t *)(pQueue->pBuffer + pQueue->head);
+    ftype = *(int8_t *)(pQueue->pBuffer + pQueue->head + 2);
     bodyLen = *(int32_t *)(pQueue->pBuffer + pQueue->head + 4);
   } else {
-    headLen = *(int32_t *)(pQueue->pBuffer);
+    headLen = *(int16_t *)(pQueue->pBuffer);
+    ftype = *(int8_t *)(pQueue->pBuffer + 2);
     bodyLen = *(int32_t *)(pQueue->pBuffer + 4);
   }
 
@@ -341,9 +351,10 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHea
   *ppBody = pBody;
   *pHeadLen = headLen;
   *pBodyLen = bodyLen;
+  *pFuncType = (ProcFuncType)ftype;
 
-  uTrace("proc:%s, pop msg from queue:%p remains:%d, head:%d:%p body:%d:%p", pQueue->name, pQueue, pQueue->items,
+  uTrace("proc:%s, pop msg from queue:%p remains:%d, head:%d:%p body:%d:%p ftype:%d", pQueue->name, pQueue, pQueue->items,
-         headLen, pHead, bodyLen, pBody);
+         headLen, pHead, bodyLen, pBody, ftype);
   return 1;
 }
 
@@ -396,12 +407,14 @@ static void taosProcThreadLoop(SProcQueue *pQueue) {
   ProcConsumeFp consumeFp = pQueue->consumeFp;
   void         *pParent = pQueue->pParent;
   void         *pHead, *pBody;
-  int32_t       headLen, bodyLen;
+  int16_t       headLen;
+  ProcFuncType  ftype;
+  int32_t       bodyLen;
 
   uDebug("proc:%s, start to get msg from queue:%p", pQueue->name, pQueue);
 
   while (1) {
-    int32_t numOfMsgs = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen);
+    int32_t numOfMsgs = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen, &ftype);
     if (numOfMsgs == 0) {
       uDebug("proc:%s, get no msg from queue:%p and exit the proc thread", pQueue->name, pQueue);
       break;
@@ -410,7 +423,7 @@ static void taosProcThreadLoop(SProcQueue *pQueue) {
       taosMsleep(1);
       continue;
     } else {
-      (*consumeFp)(pParent, pHead, headLen, pBody, bodyLen);
+      (*consumeFp)(pParent, pHead, headLen, pBody, bodyLen, ftype);
     }
   }
 }
@@ -458,10 +471,12 @@ void taosProcCleanup(SProcObj *pProc) {
   }
 }
 
-int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen) {
+int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-  return taosProcQueuePush(pProc->pChildQueue, pHead, headLen, pBody, bodyLen);
+                            ProcFuncType funcType) {
+  return taosProcQueuePush(pProc->pChildQueue, pHead, headLen, pBody, bodyLen, funcType);
 }
 
-int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen) {
+int32_t taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
-  return taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen);
+                             ProcFuncType funcType) {
+  return taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen, funcType);
 }

tests/script/jenkins/basic.txt

@@ -21,6 +21,10 @@
 ./test.sh -f tsim/insert/backquote.sim
 ./test.sh -f tsim/insert/null.sim
 
+# ---- parser
+#./test.sh -f tsim/parser/groupby-basic.sim
+#./test.sh -f tsim/parser/fourArithmetic-basic.sim
+
 # ---- query
 ./test.sh -f tsim/query/interval.sim
 ./test.sh -f tsim/query/interval-offset.sim
@@ -33,5 +37,6 @@
 
 # ---- tmq
 ./test.sh -f tsim/tmq/basic.sim
+./test.sh -f tsim/tmq/basic1.sim
 
 #======================b1-end===============

tests/script/tsim/insert/basic1.sim

@@ -41,7 +41,7 @@ print =============== insert data, mode1: one row one table in sql
 print =============== insert data, mode1: mulit rows one table in sql
 print =============== insert data, mode1: one rows mulit table in sql
 print =============== insert data, mode1: mulit rows mulit table in sql
-sql insert into c1 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40)
+sql insert into c1 values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40)
 sql insert into c1 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+2s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40)
 
 print =============== query data

tests/script/tsim/parser/fourArithmetic-basic.sim

@@ -0,0 +1,110 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$dbNamme = d0
+print =============== create database
+sql create database $dbNamme vgroups 1
+sql show databases
+print $data00 $data01 $data02
+if $rows != 2 then 
+  return -1
+endi
+
+sql use $dbNamme
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double) tags (t1 int)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+sql create table ct0 using stb tags(1000)
+#sql create table ct1 using stb tags(2000)
+#sql create table ct3 using stb tags(3000)
+
+sql show tables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== insert data
+
+$tbPrefix = ct
+$tbNum = 1
+$rowNum = 10
+$tstart = 1640966400000  # 2022-01-01 00:00:00.000
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+
+  $x = 0
+  while $x < $rowNum
+    $c2 = $x + 10
+    $c3 = $x * 10
+    $c4 = $x - 10
+
+    sql insert into $tb values ($tstart , $x , $c2 ,  $c3 , $c4 )
+    $tstart = $tstart + 1
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+  $tstart = 1640966400000
+endw
+
+sql select ts, c2-c1, c3/c1, c4+c1, c1*9, c1%3 from ct0
+print ===> rows: $rows 
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+if $rows != 10 then
+ return -1
+endi
+
+if $data01 != 10.000000000 then
+ return -1
+endi
+if $data02 != -nan then
+ return -1
+endi
+if $data03 != -10.000000000 then
+ return -1
+endi
+
+if $data91 != 10.000000000 then
+ return -1
+endi
+if $data92 != 10.000000000 then
+ return -1
+endi
+if $data93 != 8.000000000 then
+ return -1
+endi
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/parser/groupby-basic.sim

@@ -0,0 +1,786 @@
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 1
+system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$dbPrefix = group_db
+$tbPrefix = group_tb
+$mtPrefix = group_mt
+$tbNum = 8
+$rowNum = 100
+$totalNum = $tbNum * $rowNum
+
+print =============== groupby.sim
+$i = 0
+$db = $dbPrefix . $i
+$mt = $mtPrefix . $i
+
+$i = 0
+$db = $dbPrefix . $i
+$mt = $mtPrefix . $i
+
+$tstart = 1640966400000  # 2022-01-01 00:00:00.000
+
+print ==== create db, stable, ctables, insert data
+sql drop database if exists $db -x step1
+step1:
+sql create database if not exists $db keep 3650
+sql use $db
+
+sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+  $tg2 = ' . abc
+  $tg2 = $tg2 . '
+
+  sql create table $tb using $mt tags( $i , $tg2 )
+
+  $x = 0
+  while $x < $rowNum
+    $c = $x / 10
+    $c = $c * 10
+    $c = $x - $c
+
+    $binary = ' . binary
+    $binary = $binary . $c
+    $binary = $binary . '
+
+    $nchar = ' . nchar
+    $nchar = $nchar . $c
+    $nchar = $nchar . '
+
+    sql insert into $tb values ($tstart , $c , $c , $x , $x , $c , $c , $c , $binary , $nchar )
+    #print ==== insert into $tb values ($tstart , $c , $c , $x , $x , $c , $c , $c , $binary , $nchar )
+    $tstart = $tstart + 1
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+  $tstart = 1640966400000
+endw
+
+sleep 100
+
+$i1 = 1
+$i2 = 0
+
+$db = $dbPrefix . $i
+$mt = $mtPrefix . $i
+
+$dbPrefix = group_db
+$tbPrefix = group_tb
+$mtPrefix = group_mt
+
+$tb1 = $tbPrefix . $i1
+$tb2 = $tbPrefix . $i2
+$ts1 = $tb1 . .ts
+$ts2 = $tb2 . .ts
+
+print ===============================groupby_operation
+print 
+print ==== select count(*), c1 from group_tb0 group by c1
+sql select count(*), c1 from group_tb0 group by c1
+print rows: $rows
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+print $data20 $data21 $data22 $data23
+print $data80 $data81 $data82 $data83
+print $data90 $data91 $data92 $data93
+if $rows != 10 then
+  return -1
+endi
+#if $data00 != 10 then
+# return -1
+#endi
+if $data01 != 0 then
+  return -1
+endi
+#if $data10 != 10 then
+#  return -1
+#endi
+if $data11 != 1 then
+  return -1
+endi
+#if $data90 != 10 then
+#  return -1
+#endi
+if $data91 != 9 then
+  return -1
+endi
+
+sql select first(ts),c1 from group_tb0 group by c1;
+print rows: $rows
+print $data00 $data01 $data02 $data03
+print $data10 $data11 $data12 $data13
+print $data20 $data21 $data22 $data23
+print $data80 $data81 $data82 $data83
+print $data90 $data91 $data92 $data93
+if $row != 10 then
+  return -1
+endi
+
+if $data00 != @2022-01-01 00:00:00.000@ then
+  return -1
+endi
+if $data01 != 0 then
+  return -1
+endi
+if $data90 != @2022-01-01 00:00:00.009@ then
+  return -1
+endi
+if $data91 != 9 then
+  return -1
+endi
+
+
+sql select sum(c1), c1, avg(c1), min(c1), max(c2) from group_tb0 where c1 < 20 group by c1;
+if $row != 20 then
+  return -1
+endi
+
+if $data00 != 0 then
+  return -1
+endi
+
+if $data01 != 0 then
+  return -1
+endi
+
+print $data02
+if $data02 != 0.000000000 then
+  return -1
+endi
+
+if $data03 != 0 then
+  return -1
+endi
+
+print $data04
+if $data04 != 0.00000 then
+  return -1
+endi
+
+if $data10 != 100 then
+  return -1
+endi
+
+if $data11 != 1 then
+  return -1
+endi
+
+print $data12
+if $data12 != 1.000000000 then
+  return -1
+endi
+
+if $data13 != 1 then
+  return -1
+endi
+
+if $data14 != 1.00000 then
+  print expect 1.00000, actual:$data14
+  return -1
+endi
+
+sql_error select sum(c1), ts, c1 from group_tb0 where c1<20 group by c1;
+sql_error select first(ts), ts, c2 from group_tb0 where c1 < 20 group by c1;
+sql_error select sum(c3), ts, c2 from group_tb0 where c1 < 20 group by c1;
+sql_error select sum(c3), first(ts), c2 from group_tb0 where c1 < 20 group by c1;
+sql_error select first(c3), ts, c1, c2 from group_tb0 where c1 < 20 group by c1;
+sql_error select first(c3), last(c3), ts, c1 from group_tb0 where c1 < 20 group by c1;
+sql_error select ts from group_tb0 group by c1;
+
+#===========================interval=====not support======================
+sql_error select count(*), c1 from group_tb0 where c1<20 interval(1y) group by c1;
+#=====tbname must be the first in the group by clause=====================
+sql_error select count(*) from group_tb0 where c1 < 20 group by c1, tbname;
+
+#super table group by normal columns
+sql select count(*), c1 from group_mt0 where c1< 20 group by c1;
+if $row != 20 then
+  return -1
+endi
+
+if $data00 != 800 then
+  return -1
+endi
+
+if $data01 != 0 then
+  return -1
+endi
+
+if $data10 != 800 then
+  return -1
+endi
+
+if $data11 != 1 then
+  return -1
+endi
+
+if $data90 != 800 then
+  return -1
+endi
+
+if $data91 != 9 then
+  return -1
+endi
+
+sql select first(c1), c1, ts from group_mt0 where c1<20 group by c1;
+if $row != 20 then
+  return -1
+endi
+
+if $data00 != $data01 then
+  return -1
+endi
+
+if $data02 != @70-01-01 08:01:40.000@ then
+  return -1
+endi
+
+if $data10 != $data11 then
+  return -1
+endi
+
+if $data12 != @70-01-01 08:01:40.001@ then
+  return -1
+endi
+
+if $data20 != $data21 then
+  return -1
+endi
+
+if $data22 != @70-01-01 08:01:40.002@ then
+  return -1
+endi
+
+if $data90 != $data91 then
+  return -1
+endi
+
+if $data92 != @70-01-01 08:01:40.009@ then
+  return -1
+endi
+
+sql select first(c1), last(ts), first(ts), last(c1),c1,sum(c1),avg(c1),count(*) from group_mt0 where c1<20 group by c1;
+if $row != 20 then
+  return -1
+endi
+
+if $data00 != $data03 then
+  return -1
+endi
+
+if $data01 != @70-01-01 08:01:49.900@ then
+  return -1
+endi
+
+if $data02 != @70-01-01 08:01:40.000@ then
+  return -1
+endi
+
+if $data07 != 800 then
+  return -1
+endi
+
+if $data10 != $data13 then
+  return -1
+endi
+
+if $data11 != @70-01-01 08:01:49.901@ then
+  return -1
+endi
+
+if $data12 != @70-01-01 08:01:40.001@ then
+  return -1
+endi
+
+if $data17 != 800 then
+  return -1
+endi
+
+if $data90 != $data93 then
+  return -1
+endi
+
+if $data91 != @70-01-01 08:01:49.909@ then
+  return -1
+endi
+
+if $data92 != @70-01-01 08:01:40.009@ then
+  return -1
+endi
+
+if $data97 != 800 then
+  return -1
+endi
+
+if $data95 != 7200 then
+  return -1
+endi
+
+if $data94 != 9 then
+  return -1
+endi
+
+sql select c1,sum(c1),avg(c1),count(*) from group_mt0 where c1<5 group by c1;
+if $row != 5 then
+  return -1
+endi
+
+if $data00 != 0 then
+  return -1
+endi
+
+if $data11 != 800 then
+  return -1
+endi
+
+sql select first(c1), last(ts), first(ts), last(c1),sum(c1),avg(c1),count(*) from group_mt0 where c1<20 group by tbname,c1;
+if $row != 160 then
+  return -1
+endi
+
+print $data00
+if $data00 != 0 then
+  return -1
+endi
+
+if $data01 != @70-01-01 08:01:49.900@ then
+  return -1
+endi
+
+print $data01
+if $data02 != @70-01-01 08:01:40.000@ then
+  return -1
+endi
+
+if $data03 != 0 then
+  return -1
+endi
+
+if $data04 != 0 then
+  return -1
+endi
+
+if $data06 != 100 then
+  return -1
+endi
+
+if $data07 != @group_tb0@ then
+  return -1
+endi
+
+if $data90 != 9 then
+  return -1
+endi
+
+if $data91 != @70-01-01 08:01:49.909@ then
+  return -1
+endi
+
+if $data92 != @70-01-01 08:01:40.009@ then
+  return -1
+endi
+
+if $data93 != 9 then
+  return -1
+endi
+
+if $data94 != 900 then
+  return -1
+endi
+
+if $data96 != 100 then
+  return -1
+endi
+
+if $data97 != @group_tb0@ then
+  return -1
+endi
+
+sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4;
+if $rows != 10000 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+if $data01 != @70-01-01 08:01:40.000@ then
+  return -1
+endi
+
+if $data02 !=  @70-01-01 08:01:40.000@ then
+  return -1
+endi
+
+if $data03 != 0 then
+  return -1
+endi
+
+sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 limit 1;
+if $rows != 1 then
+  return -1
+endi
+
+sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 limit 20 offset 9990;
+if $rows != 10 then
+  return -1
+endi
+
+sql select count(*),first(ts),last(ts),min(c3),max(c3),sum(c3),avg(c3),sum(c4)/count(c4) from group_tb1 group by c4;
+if $rows != 10000 then
+  return -1
+endi
+
+print ---------------------------------> group by binary|nchar data add cases
+sql select count(*) from group_tb1 group by c8;
+if $rows != 100 then
+  return -1
+endi
+
+sql select count(*),sum(c4), count(c4), sum(c4)/count(c4) from group_tb1 group by c8
+if $rows != 100 then
+  return -1
+endi
+
+if $data00 != 100 then
+  return -1
+endi
+
+if $data01 != 495000 then
+  return -1
+endi
+
+if $data02 != 100 then
+  return -1
+endi
+
+if $data03 != 4950.000000000 then
+  print expect 4950.000000000 , acutal $data03
+  return -1
+endi
+
+if $data10 != 100 then
+  return -1
+endi
+
+if $data11 != 495100 then
+  return -1
+endi
+
+if $data13 != 4951.000000000 then
+  return -1
+endi
+
+print ====================> group by normal column + slimit + soffset
+sql select count(*), c8 from group_mt0 group by c8 limit 1 offset 0;
+if $rows != 100 then
+  return -1
+endi
+
+sql select sum(c2),c8,avg(c2), sum(c2)/count(*) from group_mt0 group by c8 slimit 2 soffset 99
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 79200.000000000 then
+  return -1
+endi
+
+if $data01 != @binary99@ then
+  return -1
+endi
+
+if $data02 != 99.000000000 then
+  return -1
+endi
+
+if $data03 != 99.000000000 then
+  return -1
+endi
+
+print ============>td-1765
+sql select percentile(c4, 49),min(c4),max(c4),avg(c4),stddev(c4) from group_tb0 group by c8;
+if $rows != 100 then
+  return -1
+endi
+
+if $data00 != 4851.000000000 then
+  return -1
+endi
+
+if $data01 != 0 then
+  return -1
+endi
+
+if $data02 != 9900 then
+  return -1
+endi
+
+if $data03 != 4950.000000000 then
+  return -1
+endi
+
+if $data04 != 2886.607004772 then
+  return -1
+endi
+
+if $data10 != 4852.000000000 then
+  return -1
+endi
+
+if $data11 != 1 then
+  return -1
+endi
+
+if $data12 != 9901 then
+  return -1
+endi
+
+if $data13 != 4951.000000000 then
+  return -1
+endi
+
+if $data14 != 2886.607004772 then
+  return -1
+endi
+
+print ================>td-2090
+sql select leastsquares(c2, 1, 1) from group_tb1  group by c8;
+if $rows != 100 then
+ return -1
+endi
+
+if $data00 != @{slop:0.000000, intercept:0.000000}@ then
+  return -1
+endi
+
+if $data10 != @{slop:0.000000, intercept:1.000000}@ then
+  return -1
+endi
+
+if $data90 != @{slop:0.000000, intercept:9.000000}@ then
+  return -1
+endi
+
+#=========================== group by multi tags ======================
+sql create table st (ts timestamp, c int) tags (t1 int, t2 int, t3 int, t4 int);
+sql create table t1 using st tags(1, 1, 1, 1);
+sql create table t2 using st tags(1, 2, 2, 2);
+sql insert into t1 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; 
+sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; 
+sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; 
+sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; 
+
+print =================>TD-2665
+sql_error create table txx as select avg(c) as t from st;
+sql_error create table txx1 as select avg(c) as t from t1;
+
+sql select stddev(c),stddev(c) from st group by c;
+if $rows != 4 then
+  return -1
+endi
+
+print =================>TD-2236
+sql select first(ts),last(ts) from t1 group by c;
+if $rows != 4 then
+  return -1
+endi
+
+if $data00 != @20-03-27 04:11:16.000@ then
+  return -1
+endi
+
+if $data01 != @20-03-27 04:21:16.000@ then
+  return -1
+endi
+
+if $data10 != @20-03-27 04:11:17.000@ then
+  return -1
+endi
+
+if $data11 != @20-03-27 04:31:17.000@ then
+  return -1
+endi
+
+if $data20 != @20-03-27 04:11:18.000@ then
+  return -1
+endi
+
+if $data21 != @20-03-27 04:51:18.000@ then
+  return -1
+endi
+
+if $data30 != @20-03-27 04:11:19.000@ then
+  return -1
+endi
+
+if $data31 != @20-03-27 05:10:19.000@ then
+  return -1
+endi
+
+print ===============>
+sql select stddev(c),c from st where t2=1 or t2=2  group by c;
+if $rows != 4 then
+  return -1
+endi
+
+if $data00 != 0.000000000 then
+  return -1
+endi
+
+if $data01 != 1 then
+  return -1
+endi
+
+if $data10 != 0.000000000 then
+  return -1
+endi
+
+if $data11 != 2 then
+  return -1
+endi
+
+if $data20 != 0.000000000 then
+  return -1
+endi
+
+if $data21 != 3 then
+  return -1
+endi
+
+if $data30 != 0.000000000 then
+  return -1
+endi
+
+if $data31 != 4 then
+  return -1
+endi
+
+sql_error select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732'  interval(1m) sliding(15s) group by tbname,c;
+sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732'  interval(1m) sliding(15s) group by tbname,t1,t2;
+if $rows != 40 then
+  return -1
+endi
+
+if $data01 != 1.000000000 then
+  return -1
+endi
+if $data02 != t1 then
+  return -1
+endi
+if $data03 != 1 then
+  return -1
+endi
+if $data04 != 1 then
+  return -1
+endi
+
+if $data11 != 1.000000000 then
+  return -1
+endi
+if $data12 != t1 then
+  return -1
+endi
+if $data13 != 1 then
+  return -1
+endi
+if $data14 != 1 then
+  return -1
+endi
+
+sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732'  interval(1m) sliding(15s) group by tbname,t1,t2 limit 1;
+if $rows != 2 then
+  return -1
+endi
+
+if $data11 != 1.000000000 then
+  return -1
+endi
+if $data12 != t2 then
+  return -1
+endi
+if $data13 != 1 then
+  return -1
+endi
+if $data14 != 2 then
+  return -1
+endi
+
+sql create table m1 (ts timestamp, k int, f1 int) tags(a int);
+sql create table tm0 using m1 tags(0);
+sql create table tm1 using m1 tags(1);
+
+sql insert into tm0 values('2020-1-1 1:1:1', 1, 10);
+sql insert into tm0 values('2020-1-1 1:1:2', 1, 20);
+sql insert into tm1 values('2020-2-1 1:1:1', 2, 10);
+sql insert into tm1 values('2020-2-1 1:1:2', 2, 20);
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+sleep 100
+system sh/exec.sh -n dnode1 -s start
+sleep 100
+
+sql connect
+sleep 100
+sql use group_db0;
+
+print =========================>TD-4894
+sql select count(*),k from m1 group by k;
+if $rows != 2 then
+  return -1
+endi
+
+if $data00 != 2 then
+  return -1
+endi
+
+if $data01 != 1 then
+  return -1
+endi
+
+if $data10 != 2 then
+  return -1
+endi
+
+if $data11 != 2 then
+  return -1
+endi
+
+sql_error select count(*) from m1 group by tbname,k,f1;
+sql_error select count(*) from m1 group by tbname,k,a;
+sql_error select count(*) from m1 group by k, tbname;
+sql_error select count(*) from m1 group by k,f1;
+sql_error select count(*) from tm0 group by tbname;
+sql_error select count(*) from tm0 group by a;
+sql_error select count(*) from tm0 group by k,f1;
+
+sql_error select count(*),f1 from m1 group by tbname,k;
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/query/interval-offset.sim

@@ -41,6 +41,38 @@ sql insert into ct1 values ( '2022-01-01 01:01:26.000', 6 )
 sql insert into ct1 values ( '2022-01-01 01:01:30.000', 7 )
 sql insert into ct1 values ( '2022-01-01 01:01:36.000', 8 )
 
+print =============== insert data into child table ct2 (d)
+sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1 )
+sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2 )
+sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3 )
+sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4 )
+sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5 )
+sql insert into ct2 values ( '2022-01-03 10:00:01.000', 6 )
+sql insert into ct2 values ( '2022-01-03 20:00:01.000', 7 )
+
+print =============== insert data into child table ct3 (n)
+sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL )
+sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1 )
+sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2 )
+sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3 )
+sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4 )
+sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5 )
+sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6 )
+sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7 )
+sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8 )
+
+print =============== insert data into child table ct4 (y)
+sql insert into ct4 values ( '2020-10-21 01:01:01.000', 1 )
+sql insert into ct4 values ( '2020-12-31 01:01:01.000', 2 )
+sql insert into ct4 values ( '2021-01-01 01:01:06.000', 3 )
+sql insert into ct4 values ( '2021-05-07 01:01:10.000', 4 )
+sql insert into ct4 values ( '2021-09-30 01:01:16.000', 5 )
+sql insert into ct4 values ( '2022-02-01 01:01:20.000', 6 )
+sql insert into ct4 values ( '2022-10-28 01:01:26.000', 7 )
+sql insert into ct4 values ( '2022-12-01 01:01:30.000', 8 )
+sql insert into ct4 values ( '2022-12-31 01:01:36.000', 9 )
+
+print ================ start query ======================
 sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
 print ===> rows: $rows
@@ -102,15 +134,6 @@ if $data80 != 1 then
   return -1
 endi 
 
-print =============== insert data into child table ct2 (d)
-sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1 )
-sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2 )
-sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3 )
-sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4 )
-sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5 )
-sql insert into ct2 values ( '2022-01-03 10:00:01.000', 6 )
-sql insert into ct2 values ( '2022-01-03 20:00:01.000', 7 )
-
 sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h)
 print ===> rows: $rows
@@ -140,44 +163,18 @@ print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45
 print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55 
 print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65 
 print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75 
-#if $rows != 8 then
+if $rows != 8 then
-#  return -1
+  return -1
-#endi 
+endi 
-#if $data00 != 1 then
+if $data00 != 1 then
-#  return -1
+  return -1
-#endi  
+endi  
-#if $data10 != 2 then
+if $data10 != 2 then
-#  return -1
+  return -1
-#endi   
+endi   
-#if $data20 != 2 then
+if $data70 != 1 then
-#  return -1
+  return -1
-#endi   
+endi
-#if $data30 != 2 then
-#  return -1
-#endi   
-#if $data40 != 2 then
-#  return -1
-#endi   
-#if $data50 != 2 then
-#  return -1
-#endi   
-#if $data60 != 2 then
-#  return -1
-#endi 
-#if $data70 != 1 then
-#  return -1
-#endi
-
-print =============== insert data into child table ct3 (n)
-sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL );
-sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1 );
-sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2 );
-sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3 );
-sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4 );
-sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5 );
-sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6 );
-sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7 );
-sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8 );
  
 sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
@@ -236,18 +233,6 @@ print ===> rows7: $data70 $data71 $data72 $data73 $data74
 #  return -1
 #endi 
 
-
-print =============== insert data into child table ct4 (y)
-sql insert into ct4 values ( '2020-10-21 01:01:01.000', 1 )
-sql insert into ct4 values ( '2020-12-31 01:01:01.000', 2 )
-sql insert into ct4 values ( '2021-01-01 01:01:06.000', 3 )
-sql insert into ct4 values ( '2021-05-07 01:01:10.000', 4 )
-sql insert into ct4 values ( '2021-09-30 01:01:16.000', 5 )
-sql insert into ct4 values ( '2022-02-01 01:01:20.000', 6 )
-sql insert into ct4 values ( '2022-10-28 01:01:26.000', 7 )
-sql insert into ct4 values ( '2022-12-01 01:01:30.000', 8 )
-sql insert into ct4 values ( '2022-12-31 01:01:36.000', 9 )
-
 sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n)
 print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n)
 print ===> rows: $rows
@@ -305,6 +290,27 @@ print ===> rows7: $data70 $data71 $data72 $data73 $data74
 #  return -1
 #endi 
 
+#=================================================
+print =============== stop and restart taosd
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
 
 
 

tests/script/tsim/testCaseSuite.sim

@@ -2,6 +2,8 @@
 run tsim/user/basic1.sim
 
 run tsim/db/basic1.sim
+run tsim/db/basic2.sim
+run tsim/db/basic3.sim
 run tsim/db/basic6.sim
 run tsim/db/basic7.sim
 run tsim/db/error1.sim
@@ -13,12 +15,15 @@ run tsim/insert/basic1.sim
 run tsim/insert/backquote.sim
 run tsim/insert/null.sim
 
+#run tsim/parser/groupby-basic.sim
+#run tsim/parser/fourArithmetic-basic.sim
+
 run tsim/query/interval.sim
-#run tsim/query/interval-offset.sim  # TD-14266
+run tsim/query/interval-offset.sim
 
 run tsim/show/basic.sim
 
 run tsim/table/basic1.sim
 
 run tsim/tmq/basic.sim
-
+run tsim/tmq/basic1.sim

tests/script/tsim/tmq/basic1.sim

@@ -0,0 +1,183 @@
+#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
+# vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
+# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
+# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
+#
+# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
+#
+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1
+system sh/exec.sh -n dnode1 -s start
+
+$loop_cnt = 0
+check_dnode_ready:
+	$loop_cnt = $loop_cnt + 1
+	sleep 200
+	if $loop_cnt == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
+if $data00 != 1 then
+  return -1
+endi
+if $data04 != ready then
+  goto check_dnode_ready
+endi
+
+sql connect
+
+$dbNamme = d0
+print =============== create database , vgroup 1
+sql create database $dbNamme vgroups 1
+sql show databases
+print $data00 $data01 $data02
+if $rows != 2 then 
+  return -1
+endi
+
+sql use $dbNamme
+
+print =============== create super table
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
+
+sql show stables
+if $rows != 1 then 
+  return -1
+endi
+
+print =============== create child table
+sql create table ct0 using stb tags(1000)
+sql create table ct1 using stb tags(2000)
+#sql create table ct3 using stb tags(3000)
+
+print =============== create normal table
+sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
+
+print =============== create multi topics. notes: now only support:
+print =============== 1. columns from stb; 2. * from ctb; 3. columns from ctb
+print =============== will support: * from stb; function from stb/ctb
+
+sql create topic topic_stb_column as select ts, c1, c3 from stb
+#sql create topic topic_stb_all as select * from stb
+#sql create topic topic_stb_function as select ts, abs(c1), sina(c2) from stb
+
+sql create topic topic_ctb_column as select ts, c1, c3 from ct0
+sql create topic topic_ctb_all as select * from ct0
+#sql create topic topic_ctb_function as select ts, abs(c1), sina(c2) from ct0
+
+sql create topic topic_ntb_column as select ts, c1, c3 from ntb
+sql create topic topic_ntb_all as select * from ntb
+#sql create topic topic_ntb_function as select ts, abs(c1), sina(c2) from ntb
+
+sql show tables
+if $rows != 3 then 
+  return -1
+endi
+
+print =============== insert data
+
+$tbPrefix = ct
+$tbNum = 2
+$rowNum = 10
+$tstart = 1640966400000  # 2022-01-01 00:00:00.000
+
+$i = 0
+while $i < $tbNum
+  $tb = $tbPrefix . $i
+
+  $x = 0
+  while $x < $rowNum
+    $c = $x / 10
+    $c = $c * 10
+    $c = $x - $c
+
+    $binary = ' . binary
+    $binary = $binary . $c
+    $binary = $binary . '
+
+    sql insert into $tb values ($tstart , $c , $x , $binary )
+    sql insert into ntb values ($tstart , $c , $x , $binary )
+    $tstart = $tstart + 1
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+  $tstart = 1640966400000
+endw
+
+#root@trd02 /home $ tmq_sim --help                                                                          
+# -c      Configuration directory, default is 
+# -d      The name of the database for cosumer, no default 
+# -t      The topic string for cosumer, no default 
+# -k      The key-value string for cosumer, no default 
+# -g      showMsgFlag, default is 0
+#        
+
+$totalMsgCnt = $rowNum * $tbNum
+print inserted totalMsgCnt: $totalMsgCnt
+
+# supported key:
+# group.id:<xxx>
+# enable.auto.commit:<true | false>
+# auto.offset.reset:<earliest | latest | none>
+# td.connect.ip:<fqdn | ipaddress>
+# td.connect.user:root
+# td.connect.pass:taosdata
+# td.connect.port:6030
+# td.connect.db:db
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
+print cmd result----> $system_content
+if $system_content != @{consume success: 20, 0}@ then
+  return -1
+endi
+
+#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
+#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
+#print cmd result----> $system_content
+#if $system_content != @{consume success: 20, 0}@ then
+#  return -1
+#endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
+print cmd result----> $system_content
+if $system_content != @{consume success: 10, 0}@ then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
+print cmd result----> $system_content
+if $system_content != @{consume success: 10, 0}@ then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
+print cmd result----> $system_content
+if $system_content != @{consume success: 20, 0}@ then
+  return -1
+endi
+
+print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
+system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
+print cmd result----> $system_content
+if $system_content != @{consume success: 20, 0}@ then
+  return -1
+endi
+
+print =============== create database , vgroup 4
+$dbNamme = d1
+sql create database $dbNamme vgroups 4
+
+
+#system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/test/c/tmqDemo.c

@@ -67,7 +67,7 @@ static SConfInfo g_stConfInfo = {
     "tmqdb",
     "stb",
     "./tmqResult.txt",	// output_file
-    "/data2/dnode/data/vnode/vnode2/wal",
+    "",                 // /data2/dnode/data/vnode/vnode2/wal",
     1, // threads
     1, // tables
     1, // vgroups
@@ -662,7 +662,7 @@ int main(int32_t argc, char *argv[]) {
 	float	msgsSpeed   = totalMsgs / seconds;
 	
 
-	if (0 == g_stConfInfo.simCase) {
+	if ((0 == g_stConfInfo.simCase) && (strlen(g_stConfInfo.vnodeWalPath))) {
 	  walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath);
 	  if (walLogSize <= 0) {
 	    printf("%s size incorrect!", g_stConfInfo.vnodeWalPath);

tests/test/c/tmqSim.c

@@ -98,7 +98,7 @@ void parseArgument(int32_t argc, char *argv[]) {
     }
   }
 
-#if 1
+#if 0
   pPrint("%s configDir:%s %s", GREEN, configDir, NC);
   pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC);
   pPrint("%s topicString:%s %s", GREEN, g_stConfInfo.topicString, NC);
@@ -151,8 +151,6 @@ void parseInputString() {
     token = strtok(NULL, delim);
   }
 
-  printf("\n\n");
-
   token = strtok(g_stConfInfo.keyString, delim);
   while(token != NULL) {
     //printf("%s\n", token );
@@ -198,6 +196,8 @@ tmq_t* build_consumer() {
   TAOS_RES* pRes = taos_query(pConn, sqlStr);
   if (taos_errno(pRes) != 0) {
     printf("error in use db, reason:%s\n", taos_errstr(pRes));
+	taos_free_result(pRes);
+	exit(-1);
   }
   taos_free_result(pRes);
 
@@ -219,61 +219,33 @@ tmq_list_t* build_topic_list() {
   return topic_list;
 }
 
-void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
+void loop_consume(tmq_t* tmq) {
-  static const int MIN_COMMIT_COUNT = 1000;
-
-  int            msg_count = 0;
   tmq_resp_err_t err;
 
-  if ((err = tmq_subscribe(tmq, topics))) {
-    fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err));
-    return;
-  }
-
-  while (running) {
-    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1);
-    if (tmqmessage) {
-      msg_process(tmqmessage);
-      tmq_message_destroy(tmqmessage);
-
-      if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);
-    }
-  }
-
-  err = tmq_consumer_close(tmq);
-  if (err)
-    fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err));
-  else
-    fprintf(stderr, "%% Consumer closed\n");
-}
-
-void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
-  tmq_resp_err_t err;
-
-  if ((err = tmq_subscribe(tmq, topics))) {
-    printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err));
-    return;
-  }
-  
   int32_t totalMsgs = 0;
+  int32_t totalRows = 0;
   int32_t skipLogNum = 0;
-  //int64_t startTime = taosGetTimestampUs();
   while (running) {
-    tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1);
+    tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 1);
-    if (tmqmessage) {
+    if (tmqMsg) {
 	  totalMsgs++;
-      skipLogNum += tmqGetSkipLogNum(tmqmessage);
+
-	  if (0 != g_stConfInfo.showMsgFlag) {
+	  #if 0
-        msg_process(tmqmessage);
+	  TAOS_ROW row;
+	  while (NULL != (row = tmq_get_row(tmqMsg))) {
+        totalRows++;
 	  }
-      tmq_message_destroy(tmqmessage);
+	  #endif
+	  
+      skipLogNum += tmqGetSkipLogNum(tmqMsg);
+	  if (0 != g_stConfInfo.showMsgFlag) {
+        msg_process(tmqMsg);
+	  }
+      tmq_message_destroy(tmqMsg);
     } else {
       break;
     }
   }
-  //int64_t endTime = taosGetTimestampUs();
-  //double consumeTime = (double)(endTime - startTime) / 1000000;
-
 
   err = tmq_consumer_close(tmq);
   if (err) {
@@ -281,7 +253,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
 	exit(-1);
   }
   
-  printf("{consume success: %d}", totalMsgs);
+  printf("{consume success: %d, %d}", totalMsgs, totalRows);
 }
 
 int main(int32_t argc, char *argv[]) {
@@ -294,7 +266,21 @@ int main(int32_t argc, char *argv[]) {
     return -1;
   }
   
-  perf_loop(tmq, topic_list);
+  tmq_resp_err_t err = tmq_subscribe(tmq, topic_list);
+  if (err) {
+    printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err));
+    exit(-1);
+  }
+  
+  loop_consume(tmq);
+
+  #if 0
+  err = tmq_unsubscribe(tmq);
+  if (err) {
+    printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
+    exit(-1);
+  }
+  #endif
 
   return 0;
 }