Merge branch '3.0' into feature/qnode

include/common/tcommon.h

@@ -77,7 +77,9 @@ typedef struct SDataBlockInfo {
   int16_t     numOfCols;
   int16_t     hasVarCol;
   int32_t     capacity;
-  EStreamType type;
+  // TODO: optimize and remove following
+  int32_t     childId;  // used for stream, do not serialize
+  EStreamType type;     // used for stream, do not serialize
 } SDataBlockInfo;
 
 typedef struct SSDataBlock {
@@ -105,14 +107,14 @@ typedef struct SColumnInfoData {
 } SColumnInfoData;
 
 typedef struct SQueryTableDataCond {
-  //STimeWindow  twindow;
+  // STimeWindow  twindow;
   int32_t      order;  // desc|asc order to iterate the data block
   int32_t      numOfCols;
-  SColumnInfo *colList;
+  SColumnInfo* colList;
   bool         loadExternalRows;  // load external rows or not
   int32_t      type;              // data block load type:
   int32_t      numOfTWindows;
-  STimeWindow *twindows;
+  STimeWindow* twindows;
 } SQueryTableDataCond;
 
 void*   blockDataDestroy(SSDataBlock* pBlock);
@@ -202,17 +204,17 @@ typedef struct SExprInfo {
 } SExprInfo;
 
 typedef struct {
-  const char*   key;
+  const char* key;
-  int32_t       keyLen;
+  int32_t     keyLen;
-  uint8_t       type;
+  uint8_t     type;
-  union{
+  union {
     const char* value;
     int64_t     i;
     uint64_t    u;
     double      d;
     float       f;
   };
-  int32_t       length;
+  int32_t length;
 } SSmlKv;
 
 #define QUERY_ASC_FORWARD_STEP  1
@@ -220,7 +222,7 @@ typedef struct {
 
 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
 
-#define SORT_QSORT_T 0x1
+#define SORT_QSORT_T              0x1
 #define SORT_SPILLED_MERGE_SORT_T 0x2
 typedef struct SSortExecInfo {
   int32_t sortMethod;

include/common/tmsg.h

@@ -177,8 +177,8 @@ typedef struct {
 typedef struct SField {
   char    name[TSDB_COL_NAME_LEN];
   uint8_t type;
-  int32_t bytes;
   int8_t  flags;
+  int32_t bytes;
 } SField;
 
 typedef struct SRetention {
@@ -1122,13 +1122,13 @@ typedef struct {
   SSchema* pSchemas;
 } STableMetaRsp;
 
-typedef struct {  
+typedef struct {
   STableMetaRsp* pMeta;
 } SMAlterStbRsp;
 
-int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp);
+int32_t tEncodeSMAlterStbRsp(SEncoder* pEncoder, const SMAlterStbRsp* pRsp);
-int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp);
+int32_t tDecodeSMAlterStbRsp(SDecoder* pDecoder, SMAlterStbRsp* pRsp);
-void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp);
+void    tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp);
 
 int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp);
 int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp);
@@ -2303,23 +2303,23 @@ typedef struct {
 } SVgEpSet;
 
 typedef struct {
-  int8_t   version;       // for compatibility(default 0)
+  int8_t    version;       // for compatibility(default 0)
-  int8_t   intervalUnit;  // MACRO: TIME_UNIT_XXX
+  int8_t    intervalUnit;  // MACRO: TIME_UNIT_XXX
-  int8_t   slidingUnit;   // MACRO: TIME_UNIT_XXX
+  int8_t    slidingUnit;   // MACRO: TIME_UNIT_XXX
-  int8_t   timezoneInt;   // sma data expired if timezone changes.
+  int8_t    timezoneInt;   // sma data expired if timezone changes.
-  int32_t  dstVgId;
+  int32_t   dstVgId;
-  char     indexName[TSDB_INDEX_NAME_LEN];
+  char      indexName[TSDB_INDEX_NAME_LEN];
-  int32_t  exprLen;
+  int32_t   exprLen;
-  int32_t  tagsFilterLen;
+  int32_t   tagsFilterLen;
-  int32_t  numOfVgroups;
+  int32_t   numOfVgroups;
-  int64_t  indexUid;
+  int64_t   indexUid;
-  tb_uid_t tableUid;  // super/child/common table uid
+  tb_uid_t  tableUid;  // super/child/common table uid
-  int64_t  interval;
+  int64_t   interval;
-  int64_t  offset;  // use unit by precision of DB
+  int64_t   offset;  // use unit by precision of DB
-  int64_t  sliding;
+  int64_t   sliding;
-  char*    expr;  // sma expression
+  char*     expr;  // sma expression
-  char*    tagsFilter;
+  char*     tagsFilter;
-  SVgEpSet vgEpSet[];
+  SVgEpSet* pVgEpSet;
 } STSma;  // Time-range-wise SMA
 
 typedef STSma SVCreateTSmaReq;
@@ -2405,7 +2405,7 @@ static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) {
 }
 
 typedef struct {
-  int64_t indexUid;
+  int64_t     indexUid;
   STimeWindow queryWindow;
 } SVGetTsmaExpWndsReq;
 

include/libs/nodes/nodes.h

@@ -195,6 +195,7 @@ typedef enum ENodeType {
   QUERY_NODE_LOGIC_PLAN_FILL,
   QUERY_NODE_LOGIC_PLAN_SORT,
   QUERY_NODE_LOGIC_PLAN_PARTITION,
+  QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC,
   QUERY_NODE_LOGIC_SUBPLAN,
   QUERY_NODE_LOGIC_PLAN,
 
@@ -211,7 +212,7 @@ typedef enum ENodeType {
   QUERY_NODE_PHYSICAL_PLAN_MERGE,
   QUERY_NODE_PHYSICAL_PLAN_SORT,
   QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL,
-  QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL,
+  QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL,
   QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL,
   QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL,
   QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL,
@@ -222,6 +223,7 @@ typedef enum ENodeType {
   QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW,
   QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE_WINDOW,
   QUERY_NODE_PHYSICAL_PLAN_PARTITION,
+  QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC,
   QUERY_NODE_PHYSICAL_PLAN_DISPATCH,
   QUERY_NODE_PHYSICAL_PLAN_INSERT,
   QUERY_NODE_PHYSICAL_PLAN_DELETE,

include/libs/nodes/plannodes.h

@@ -42,6 +42,7 @@ typedef struct SScanLogicNode {
   SNodeList*    pScanPseudoCols;
   int8_t        tableType;
   uint64_t      tableId;
+  uint64_t      stableId;
   SVgroupsInfo* pVgroupList;
   EScanType     scanType;
   uint8_t       scanSeq[2];  // first is scan count, and second is reverse scan count
@@ -86,6 +87,11 @@ typedef struct SProjectLogicNode {
   int64_t    soffset;
 } SProjectLogicNode;
 
+typedef struct SIndefRowsFuncLogicNode {
+  SLogicNode node;
+  SNodeList* pVectorFuncs;
+} SIndefRowsFuncLogicNode;
+
 typedef enum EModifyTableType { MODIFY_TABLE_TYPE_INSERT = 1, MODIFY_TABLE_TYPE_DELETE } EModifyTableType;
 
 typedef struct SVnodeModifyLogicNode {
@@ -94,7 +100,7 @@ typedef struct SVnodeModifyLogicNode {
   int32_t          msgType;
   SArray*          pDataBlocks;
   SVgDataBlocks*   pVgDataBlocks;
-  SNode*           pModifyRows;  // SColumnNode
+  SNode*           pAffectedRows;  // SColumnNode
   uint64_t         tableId;
   int8_t           tableType;  // table type
   char             tableFName[TSDB_TABLE_FNAME_LEN];
@@ -109,6 +115,7 @@ typedef struct SExchangeLogicNode {
 typedef struct SMergeLogicNode {
   SLogicNode node;
   SNodeList* pMergeKeys;
+  SNodeList* pInputs;
   int32_t    numOfChannels;
   int32_t    srcGroupId;
 } SMergeLogicNode;
@@ -117,7 +124,7 @@ typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW
 
 typedef enum EIntervalAlgorithm {
   INTERVAL_ALGO_HASH = 1,
-  INTERVAL_ALGO_SORT_MERGE,
+  INTERVAL_ALGO_MERGE,
   INTERVAL_ALGO_STREAM_FINAL,
   INTERVAL_ALGO_STREAM_SEMI,
   INTERVAL_ALGO_STREAM_SINGLE,
@@ -220,6 +227,7 @@ typedef struct SScanPhysiNode {
   SNodeList* pScanCols;
   SNodeList* pScanPseudoCols;
   uint64_t   uid;  // unique id of the table
+  uint64_t   suid;
   int8_t     tableType;
   SName      tableName;
 } SScanPhysiNode;
@@ -264,6 +272,12 @@ typedef struct SProjectPhysiNode {
   int64_t    soffset;
 } SProjectPhysiNode;
 
+typedef struct SIndefRowsFuncPhysiNode {
+  SPhysiNode node;
+  SNodeList* pExprs;
+  SNodeList* pVectorFuncs;
+} SIndefRowsFuncPhysiNode;
+
 typedef struct SJoinPhysiNode {
   SPhysiNode node;
   EJoinType  joinType;
@@ -296,6 +310,7 @@ typedef struct SExchangePhysiNode {
 typedef struct SMergePhysiNode {
   SPhysiNode node;
   SNodeList* pMergeKeys;
+  SNodeList* pTargets;
   int32_t    numOfChannels;
   int32_t    srcGroupId;
 } SMergePhysiNode;
@@ -319,7 +334,7 @@ typedef struct SIntervalPhysiNode {
   int8_t           slidingUnit;
 } SIntervalPhysiNode;
 
-typedef SIntervalPhysiNode SSortMergeIntervalPhysiNode;
+typedef SIntervalPhysiNode SMergeIntervalPhysiNode;
 typedef SIntervalPhysiNode SStreamIntervalPhysiNode;
 typedef SIntervalPhysiNode SStreamFinalIntervalPhysiNode;
 typedef SIntervalPhysiNode SStreamSemiIntervalPhysiNode;
@@ -388,6 +403,7 @@ typedef struct SDataDeleterNode {
   int8_t        tableType;  // table type
   char          tableFName[TSDB_TABLE_FNAME_LEN];
   STimeWindow   deleteTimeRange;
+  SNode*        pAffectedRows;
 } SDataDeleterNode;
 
 typedef struct SSubplan {

include/libs/qcom/query.h

@@ -224,7 +224,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
 #define NEED_SCHEDULER_RETRY_ERROR(_code) \
   ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
 
-#define REQUEST_MAX_TRY_TIMES 1
+#define REQUEST_TOTAL_EXEC_TIMES 2
 
 #define qFatal(...)                                                                           \
   do {                                                                                        \

include/libs/stream/tstream.h

@@ -24,8 +24,8 @@
 extern "C" {
 #endif
 
-#ifndef _TSTREAM_H_
+#ifndef _STREAM_H_
-#define _TSTREAM_H_
+#define _STREAM_H_
 
 typedef struct SStreamTask SStreamTask;
 
@@ -39,6 +39,7 @@ enum {
   TASK_INPUT_STATUS__NORMAL = 1,
   TASK_INPUT_STATUS__BLOCKED,
   TASK_INPUT_STATUS__RECOVER,
+  TASK_INPUT_STATUS__PROCESSING,
   TASK_INPUT_STATUS__STOP,
   TASK_INPUT_STATUS__FAILED,
 };
@@ -60,6 +61,10 @@ enum {
   STREAM_INPUT__CHECKPOINT,
 };
 
+typedef struct {
+  int8_t type;
+} SStreamQueueItem;
+
 typedef struct {
   int8_t      type;
   int64_t     ver;
@@ -80,55 +85,51 @@ typedef struct {
   int8_t type;
 } SStreamCheckpoint;
 
+enum {
+  STREAM_QUEUE__SUCESS = 1,
+  STREAM_QUEUE__FAILED,
+  STREAM_QUEUE__PROCESSING,
+};
+
 typedef struct {
   STaosQueue* queue;
   STaosQall*  qall;
   void*       qItem;
-  int8_t      failed;
+  int8_t      status;
-} SStreamQ;
+} SStreamQueue;
 
-static FORCE_INLINE void* streamQCurItem(SStreamQ* queue) {
+SStreamQueue* streamQueueOpen();
-  //
+void          streamQueueClose(SStreamQueue* queue);
-  return queue->qItem;
+
+static FORCE_INLINE void streamQueueProcessSuccess(SStreamQueue* queue) {
+  ASSERT(atomic_load_8(&queue->status) == STREAM_QUEUE__PROCESSING);
+  queue->qItem = NULL;
+  atomic_store_8(&queue->status, STREAM_QUEUE__SUCESS);
 }
 
-static FORCE_INLINE void* streamQNextItem(SStreamQ* queue) {
+static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) {
-  int8_t failed = atomic_load_8(&queue->failed);
+  ASSERT(atomic_load_8(&queue->status) == STREAM_QUEUE__PROCESSING);
-  if (failed) {
+  atomic_store_8(&queue->status, STREAM_QUEUE__FAILED);
+}
+
+static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { return queue->qItem; }
+
+static FORCE_INLINE void* streamQueueNextItem(SStreamQueue* queue) {
+  int8_t dequeueFlag = atomic_exchange_8(&queue->status, STREAM_QUEUE__PROCESSING);
+  if (dequeueFlag == STREAM_QUEUE__FAILED) {
     ASSERT(queue->qItem != NULL);
-    return streamQCurItem(queue);
+    return streamQueueCurItem(queue);
   } else {
     taosGetQitem(queue->qall, &queue->qItem);
     if (queue->qItem == NULL) {
       taosReadAllQitems(queue->queue, queue->qall);
       taosGetQitem(queue->qall, &queue->qItem);
     }
-    return streamQCurItem(queue);
+    return streamQueueCurItem(queue);
   }
 }
 
-static FORCE_INLINE void streamQSetFail(SStreamQ* queue) { atomic_store_8(&queue->failed, 1); }
+SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq);
-
-static FORCE_INLINE void streamQSetSuccess(SStreamQ* queue) { atomic_store_8(&queue->failed, 0); }
-
-static FORCE_INLINE SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) {
-  SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
-  if (pDataSubmit == NULL) return NULL;
-  pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t));
-  if (pDataSubmit->dataRef == NULL) goto FAIL;
-  pDataSubmit->data = pReq;
-  *pDataSubmit->dataRef = 1;
-  pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT;
-  return pDataSubmit;
-FAIL:
-  taosFreeQitem(pDataSubmit);
-  return NULL;
-}
-
-static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) {
-  //
-  atomic_add_fetch_32(pDataSubmit->dataRef, 1);
-}
 
 static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) {
   int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1);
@@ -141,9 +142,31 @@ static FORCE_INLINE void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit)
 
 SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit);
 
+#if 0
 int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput);
 void*   streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput);
 
+static FORCE_INLINE int32_t streamEnqueue1(SStreamQueue* queue, SStreamQueueItem* pItem) {
+  int8_t inputStatus = atomic_load_8(&queue->enqueueStatus);
+  if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
+    if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
+      SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem);
+      if (pSubmitClone == NULL) {
+        atomic_store_8(&queue->enqueueStatus, TASK_INPUT_STATUS__FAILED);
+        return -1;
+      }
+      taosWriteQitem(queue->queue, pSubmitClone);
+    } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) {
+      taosWriteQitem(queue->queue, pItem);
+    } else if (pItem->type == STREAM_INPUT__CHECKPOINT) {
+      taosWriteQitem(queue->queue, pItem);
+    }
+    return 0;
+  }
+  return 0;
+}
+#endif
+
 typedef struct {
   int8_t parallelizable;
   char*  qmsg;
@@ -236,13 +259,17 @@ struct SStreamTask {
   int8_t  dispatchType;
   int16_t dispatchMsgType;
 
+  // node info
+  int32_t childId;
   int32_t nodeId;
   SEpSet  epSet;
 
   // exec
   STaskExec exec;
 
-  // local sink
+  // TODO: merge sink and dispatch
+
+  //  local sink
   union {
     STaskSinkTb    tbSink;
     STaskSinkSma   smaSink;
@@ -258,25 +285,61 @@ struct SStreamTask {
 
   int8_t inputStatus;
   int8_t outputStatus;
-
+#if 0
   STaosQueue* inputQ;
   STaosQall*  inputQAll;
   STaosQueue* outputQ;
   STaosQall*  outputQAll;
+#endif
+
+  SStreamQueue* inputQueue;
+  SStreamQueue* outputQueue;
 
   // application storage
   void* ahandle;
 };
 
-SStreamTask* tNewSStreamTask(int64_t streamId);
+SStreamTask* tNewSStreamTask(int64_t streamId, int32_t childId);
 int32_t      tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
 int32_t      tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
 void         tFreeSStreamTask(SStreamTask* pTask);
 
-typedef struct {
+static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) {
-  // SMsgHead     head;
+  while (1) {
-  SStreamTask* task;
+    int8_t inputStatus =
-} SStreamTaskDeployReq;
+        atomic_val_compare_exchange_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL, TASK_INPUT_STATUS__PROCESSING);
+    if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
+      break;
+    }
+    ASSERT(0);
+  }
+
+  if (pItem->type == STREAM_INPUT__DATA_SUBMIT) {
+    SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem);
+    if (pSubmitClone == NULL) {
+      atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
+      return -1;
+    }
+    taosWriteQitem(pTask->inputQueue->queue, pSubmitClone);
+  } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) {
+    taosWriteQitem(pTask->inputQueue->queue, pItem);
+  } else if (pItem->type == STREAM_INPUT__CHECKPOINT) {
+    taosWriteQitem(pTask->inputQueue->queue, pItem);
+  }
+
+  // TODO: back pressure
+  atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL);
+  return 0;
+}
+
+static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) {
+  atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
+}
+
+static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) {
+  taosWriteQitem(pTask->outputQueue->queue, pBlock);
+  return 0;
+}
 
 typedef struct {
   int32_t reserved;
@@ -289,6 +352,11 @@ typedef struct {
   SArray* data;  // SArray<SSDataBlock>
 } SStreamTaskExecReq;
 
+typedef struct {
+  // SMsgHead     head;
+  SStreamTask* task;
+} SStreamTaskDeployReq;
+
 int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq);
 void*   tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq);
 void    tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq);
@@ -297,6 +365,12 @@ typedef struct {
   int32_t reserved;
 } SStreamTaskExecRsp;
 
+typedef struct {
+  SMsgHead head;
+  int64_t  streamId;
+  int32_t  taskId;
+} SStreamTaskRunReq;
+
 typedef struct {
   // SMsgHead head;
   int64_t streamId;
@@ -304,21 +378,18 @@ typedef struct {
   SArray* res;  // SArray<SSDataBlock>
 } SStreamSinkReq;
 
-typedef struct {
-  SMsgHead head;
-  int64_t  streamId;
-  int32_t  taskId;
-} SStreamTaskRunReq;
-
 typedef struct {
   int64_t streamId;
   int32_t taskId;
   int32_t sourceTaskId;
   int32_t sourceVg;
+  int32_t sourceChildId;
 #if 0
   int64_t sourceVer;
 #endif
-  SArray* data;  // SArray<SSDataBlock>
+  int32_t blockNum;
+  SArray* dataLen;  // SArray<int32_t>
+  SArray* data;     // SArray<SRetrieveTableRsp*>
 } SStreamDispatchReq;
 
 typedef struct {
@@ -340,6 +411,8 @@ typedef struct {
   int8_t  inputStatus;
 } SStreamTaskRecoverRsp;
 
+int32_t streamTriggerByWrite(SStreamTask* pTask, int32_t vgId, SMsgCb* pMsgCb);
+
 int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input);
 int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input);
 int32_t streamDequeueOutput(SStreamTask* pTask, void** output);
@@ -356,4 +429,4 @@ int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp)
 }
 #endif
 
-#endif /* ifndef _TSTREAM_H_ */
+#endif /* ifndef _STREAM_H_ */

include/util/taoserror.h

@@ -218,6 +218,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_VGROUP_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x0390)
 #define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE       TAOS_DEF_ERROR_CODE(0, 0x0391)
 #define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE   TAOS_DEF_ERROR_CODE(0, 0x0392)
+#define TSDB_CODE_MND_VGROUP_UN_CHANGED         TAOS_DEF_ERROR_CODE(0, 0x0393)
+#define TSDB_CODE_MND_HAS_OFFLINE_DNODE         TAOS_DEF_ERROR_CODE(0, 0x0394)
 
 // mnode-stable
 #define TSDB_CODE_MND_STB_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x03A0)
@@ -235,6 +237,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_TOO_MANY_COLUMNS          TAOS_DEF_ERROR_CODE(0, 0x03AC)
 #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST      TAOS_DEF_ERROR_CODE(0, 0x03AD)
 #define TSDB_CODE_MND_COLUMN_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03AE)
+#define TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03AF)
 #define TSDB_CODE_MND_SINGLE_STB_MODE_DB        TAOS_DEF_ERROR_CODE(0, 0x03B0)
 
 // mnode-infoSchema

include/util/tcompare.h

@@ -105,8 +105,6 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
 int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
 
-int32_t compareJsonContainsKey(const void *pLeft, const void *pRight);
-
 __compar_fn_t getComparFunc(int32_t type, int32_t optr);
 __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
 int32_t       doCompare(const char *a, const char *b, int32_t type, size_t size);

source/client/inc/clientInt.h

@@ -212,6 +212,9 @@ typedef struct SRequestObj {
   SArray*              tableList;
   SQueryExecMetric     metric;
   SRequestSendRecvBody body;
+
+  uint32_t             prevCode; //previous error code: todo refactor, add update flag for catalog
+  uint32_t             retry;
 } SRequestObj;
 
 typedef struct SSyncQueryParam {
@@ -263,8 +266,8 @@ extern SAppInfo appInfo;
 extern int32_t  clientReqRefPool;
 extern int32_t  clientConnRefPool;
 
-extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
+__async_send_cb_fn_t getMsgRspHandle(int32_t msgType);
-int           genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
+
 SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
 
 void*    createTscObj(const char* user, const char* auth, const char* db, int32_t connType, SAppInstInfo* pAppInfo);
@@ -274,7 +277,7 @@ int32_t  releaseTscObj(int64_t rid);
 
 uint64_t generateRequestId();
 
-void*        createRequest(STscObj* pObj, void* param, int32_t type);
+void*        createRequest(STscObj* pObj, int32_t type);
 void         destroyRequest(SRequestObj* pRequest);
 SRequestObj* acquireRequest(int64_t rid);
 int32_t      releaseRequest(int64_t rid);
@@ -290,8 +293,6 @@ void* openTransporter(const char* user, const char* auth, int32_t numOfThreads);
 bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType);
 void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
 
-void initMsgHandleFp();
-
 TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
                             uint16_t port, int connType);
 
@@ -325,6 +326,9 @@ int32_t      scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod
 void         launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery);
 int32_t      refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
 int32_t      updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList);
+void         doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta);
+int32_t      removeMeta(STscObj* pTscObj, SArray* tbList);// todo move to clientImpl.c and become a static function
+int32_t      handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);// todo move to xxx
 
 #ifdef __cplusplus
 }

source/client/src/clientEnv.c

@@ -177,7 +177,7 @@ STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientCon
 
 int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); }
 
-void *createRequest(STscObj *pObj, void *param, int32_t type) {
+void *createRequest(STscObj *pObj, int32_t type) {
   assert(pObj != NULL);
 
   SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj));
@@ -191,8 +191,6 @@ void *createRequest(STscObj *pObj, void *param, int32_t type) {
   pRequest->requestId = generateRequestId();
   pRequest->metric.start = taosGetTimestampUs();
 
-  pRequest->body.param = param;
-
   pRequest->type = type;
   pRequest->pTscObj = pObj;
   pRequest->msgBuf = taosMemoryCalloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
@@ -280,7 +278,6 @@ void taos_init_imp(void) {
     return;
   }
 
-  initMsgHandleFp();
   initQueryModuleMsgHandle();
 
   rpcInit();

source/client/src/clientImpl.c

@@ -133,7 +133,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
 }
 
 int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest) {
-  *pRequest = createRequest(pTscObj, NULL, TSDB_SQL_SELECT);
+  *pRequest = createRequest(pTscObj, TSDB_SQL_SELECT);
   if (*pRequest == NULL) {
     tscError("failed to malloc sqlObj");
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -207,6 +207,7 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
   if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
     code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, false);
   }
+
   return code;
 }
 
@@ -235,6 +236,31 @@ static SAppInstInfo* getAppInfo(SRequestObj* pRequest) {
   return pRequest->pTscObj->pAppInfo;
 }
 
+void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
+  SRetrieveTableRsp* pRsp = NULL;
+
+  int32_t code = qExecCommand(pQuery->pRoot, &pRsp);
+  if (TSDB_CODE_SUCCESS == code && NULL != pRsp) {
+    code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, false);
+  }
+
+  SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
+
+  if (pRequest->code != TSDB_CODE_SUCCESS) {
+    pResultInfo->numOfRows = 0;
+    pRequest->code = code;
+    tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code),
+             pRequest->requestId);
+  } else {
+    tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,
+             pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed,
+             pRequest->requestId);
+  }
+
+  pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+//  pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows);
+}
+
 int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
   // drop table if exists not_exists_table
   if (NULL == pQuery->pCmdMsg) {
@@ -509,19 +535,20 @@ int32_t handleAlterTbExecRes(void* res, SCatalog* pCatalog) {
   return catalogUpdateTableMeta(pCatalog, (STableMetaRsp*)res);
 }
 
-int32_t handleExecRes(SRequestObj* pRequest) {
+int32_t handleQueryExecRsp(SRequestObj* pRequest) {
   if (NULL == pRequest->body.resInfo.execRes.res) {
     return TSDB_CODE_SUCCESS;
   }
 
-  int32_t code = 0;
   SCatalog* pCatalog = NULL;
-  code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
+  SAppInstInfo*   pAppInfo = getAppInfo(pRequest);
+
+  int32_t code = catalogGetHandle(pAppInfo->clusterId, &pCatalog);
   if (code) {
     return code;
   }
 
-  SEpSet epset = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
+  SEpSet epset = getEpSet_s(&pAppInfo->mgmtEp);
   SQueryExecRes* pRes = &pRequest->body.resInfo.execRes;
 
   switch (pRes->msgType) {
@@ -539,8 +566,9 @@ int32_t handleExecRes(SRequestObj* pRequest) {
       break;
     }
     default:
-      tscError("invalid exec result for request type %d", pRequest->type);
+      tscError("0x%"PRIx64", invalid exec result for request type %d, reqId:0x%"PRIx64, pRequest->self,
-      return TSDB_CODE_APP_ERROR;
+          pRequest->type, pRequest->requestId);
+      code = TSDB_CODE_APP_ERROR;
   }
 
   return code;
@@ -548,6 +576,25 @@ int32_t handleExecRes(SRequestObj* pRequest) {
 
 void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
   SRequestObj* pRequest = (SRequestObj*) param;
+  pRequest->code = code;
+
+  STscObj* pTscObj = pRequest->pTscObj;
+  if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) {
+    tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
+    pRequest->prevCode = code;
+    doAsyncQuery(pRequest, true);
+    return;
+  }
+
+  if (code == TSDB_CODE_SUCCESS) {
+    code = handleQueryExecRsp(pRequest);
+    ASSERT(pRequest->code ==  TSDB_CODE_SUCCESS);
+    pRequest->code = code;
+  }
+
+  if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
+    removeMeta(pTscObj, pRequest->tableList);
+  }
 
   // return to client
   pRequest->body.queryFp(pRequest->body.param, pRequest, code);
@@ -583,7 +630,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
     qDestroyQuery(pQuery);
   }
 
-  handleExecRes(pRequest);
+  handleQueryExecRsp(pRequest);
 
   if (NULL != pRequest && TSDB_CODE_SUCCESS != code) {
     pRequest->code = terrno;
@@ -617,13 +664,12 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
 }
 
 void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
-  void* pRes = NULL;
-
   int32_t code = 0;
+
   switch (pQuery->execMode) {
     case QUERY_EXEC_MODE_LOCAL:
-      code = execLocalCmd(pRequest, pQuery);
+      asyncExecLocalCmd(pRequest, pQuery);
-      break;
+      return;
     case QUERY_EXEC_MODE_RPC:
       code = asyncExecDdlQuery(pRequest, pQuery);
       break;
@@ -649,11 +695,9 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
       if (TSDB_CODE_SUCCESS == code) {
         schedulerAsyncExecJob(pAppInfo->pTransporter, pNodeList, pRequest->body.pDag, &pRequest->body.queryJob,
                               pRequest->sqlstr, pRequest->metric.start, schedulerExecCb, pRequest);
-        //        if (NULL != pRes) {
-        //          code = validateSversion(pRequest, pRes);
-        //        }
       }
 
+      //todo not to be released here
       taosArrayDestroy(pNodeList);
       break;
     }
@@ -671,12 +715,6 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) {
   if (NULL != pRequest && TSDB_CODE_SUCCESS != code) {
     pRequest->code = terrno;
   }
-
-  //    if (res) {
-  //      *res = pRes;
-  //    } else {
-//  freeRequestRes(pRequest, pRes);
-//  pRes = NULL;
 }
 
 int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {
@@ -750,7 +788,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
       pRequest->code = code;
       break;
     }
-  } while (retryNum++ < REQUEST_MAX_TRY_TIMES);
+  } while (retryNum++ < REQUEST_TOTAL_EXEC_TIMES);
 
   if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
     removeMeta(pTscObj, pRequest->tableList);
@@ -808,7 +846,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
     return pTscObj;
   }
 
-  SRequestObj* pRequest = createRequest(pTscObj, param, TDMT_MND_CONNECT);
+  SRequestObj* pRequest = createRequest(pTscObj, TDMT_MND_CONNECT);
   if (pRequest == NULL) {
     destroyTscObj(pTscObj);
     terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@@ -850,7 +888,7 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) {
 
   pMsgSendInfo->requestObjRefId = pRequest->self;
   pMsgSendInfo->requestId = pRequest->requestId;
-  pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)];
+  pMsgSendInfo->fp = getMsgRspHandle(pMsgSendInfo->msgType);
   pMsgSendInfo->param = pRequest;
 
   SConnectReq connectReq = {0};
@@ -1429,7 +1467,7 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de
   }
 
   code = statusRsp.statusCode;
-  if (details != NULL && statusRsp.details != NULL) {
+  if (details != NULL) {
     tstrncpy(details, statusRsp.details, maxlen);
   }
 

source/client/src/clientMain.c

@@ -30,6 +30,7 @@
 #define TSC_VAR_RELEASED    0
 
 static int32_t sentinel = TSC_VAR_NOT_RELEASE;
+static int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt);
 
 int taos_options(TSDB_OPTION option, const void *arg, ...) {
   static int32_t lock = 0;
@@ -192,7 +193,7 @@ TAOS_RES *taos_query(TAOS *taos, const char *sql) {
   STscObj* pTscObj = (STscObj*)taos;
 
 #if SYNC_ON_TOP_OF_ASYNC
-  SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(struct SSyncQueryParam));
+  SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
   tsem_init(&param->sem, 0, 0);
 
   taos_query_a(pTscObj, sql, syncQueryFn, param);
@@ -608,36 +609,38 @@ void retrieveMetaCallback(SMetaData* pResultMeta, void* param, int32_t code) {
   SQuery* pQuery = pWrapper->pQuery;
   SRequestObj* pRequest = pWrapper->pRequest;
 
-  if (code != TSDB_CODE_SUCCESS) {
+  if (code == TSDB_CODE_SUCCESS) {
-    goto _error;
+    code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery);
   }
 
-  code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery);
+  if (code == TSDB_CODE_SUCCESS) {
-  if (code != TSDB_CODE_SUCCESS) {
+    if (pQuery->haveResultSet) {
-    goto _error;
+      setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, pQuery->numOfResCols);
+      setResPrecision(&pRequest->body.resInfo, pQuery->precision);
+    }
+
+    TSWAP(pRequest->dbList, (pQuery)->pDbList);
+    TSWAP(pRequest->tableList, (pQuery)->pTableList);
+
+    taosMemoryFree(pWrapper);
+    launchAsyncQuery(pRequest, pQuery);
+  } else {
+    if (NEED_CLIENT_HANDLE_ERROR(code)) {
+      tscDebug("0x%"PRIx64" client retry to handle the error, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
+      pRequest->prevCode = code;
+      doAsyncQuery(pRequest, true);
+      return;
+    }
+
+    // return to app directly
+    taosMemoryFree(pWrapper);
+    tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, reqId:0x%" PRIx64, pRequest->self, tstrerror(code),
+             pRequest->requestId);
+    pRequest->code = code;
+    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
   }
-
-  if (pQuery->haveResultSet) {
-    setResSchemaInfo(&pRequest->body.resInfo, pQuery->pResSchema, (pQuery)->numOfResCols);
-    setResPrecision(&pRequest->body.resInfo, (pQuery)->precision);
-  }
-
-  TSWAP(pRequest->dbList, (pQuery)->pDbList);
-  TSWAP(pRequest->tableList, (pQuery)->pTableList);
-
-  taosMemoryFree(pWrapper);
-  launchAsyncQuery(pRequest, pQuery);
-  return;
-
-  _error:
-  taosMemoryFree(pWrapper);
-  tscError("0x%" PRIx64 " error occurs, code:%s, return to user app, reqId:%" PRIx64, pRequest->self, tstrerror(code),
-           pRequest->requestId);
-  pRequest->code = code;
-  pRequest->body.queryFp(pRequest->body.param, pRequest, code);
 }
 
-// todo add retry before return user's callback
 void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
   ASSERT(fp != NULL);
 
@@ -657,24 +660,27 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param
   }
 
   SRequestObj *pRequest = NULL;
-  int32_t      retryNum = 0;
+  int32_t code = buildRequest(taos, sql, sqlLen, &pRequest);
-  int32_t      code = 0;
-
-  //  while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
-  code = buildRequest(taos, sql, sqlLen, &pRequest);
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
-    fp(param, NULL, code);
+    fp(param, NULL, terrno);
     return;
   }
 
   pRequest->body.queryFp = fp;
   pRequest->body.param   = param;
+  doAsyncQuery(pRequest, false);
+}
 
-  STscObj *pTscObj = pRequest->pTscObj;
+int32_t createParseContext(const SRequestObj *pRequest, SParseContext** pCxt) {
+  const STscObj *pTscObj = pRequest->pTscObj;
 
-  SParseContext* pCxt = taosMemoryCalloc(1, sizeof(SParseContext));
+  *pCxt = taosMemoryCalloc(1, sizeof(SParseContext));
-  *pCxt = (SParseContext){.requestId = pRequest->requestId,
+  if (*pCxt == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  **pCxt = (SParseContext){.requestId = pRequest->requestId,
                        .acctId = pTscObj->acctId,
                        .db = pRequest->pDb,
                        .topicQuery = false,
@@ -687,6 +693,22 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param
                        .pUser = pTscObj->user,
                        .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
                        .async = true,};
+  return TSDB_CODE_SUCCESS;
+}
+
+void doAsyncQuery(SRequestObj* pRequest, bool updateMetaForce) {
+  SParseContext* pCxt = NULL;
+  STscObj *pTscObj = pRequest->pTscObj;
+
+  if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) {
+    pRequest->code = pRequest->prevCode;
+    goto _error;
+  }
+
+  int32_t code = createParseContext(pRequest, &pCxt);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
 
   pCxt->mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
   code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCxt->pCatalog);
@@ -694,39 +716,36 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param
     goto _error;
   }
 
-  SQuery *    pQuery = NULL;
+  SQuery *pQuery = NULL;
-  SCatalogReq catalogReq = {0};
+
+  SCatalogReq catalogReq = {.forceUpdate = updateMetaForce};
   code = qParseSqlSyntax(pCxt, &pQuery, &catalogReq);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
   SqlParseWrapper *pWrapper = taosMemoryCalloc(1, sizeof(SqlParseWrapper));
+  if (pWrapper == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _error;
+  }
+
   pWrapper->pCtx = pCxt;
   pWrapper->pQuery = pQuery;
   pWrapper->pRequest = pRequest;
   pWrapper->catalogReq = catalogReq;
 
   code = catalogAsyncGetAllMeta(pCxt->pCatalog, pCxt->pTransporter, &pCxt->mgmtEpSet, pRequest->requestId,
-                                  &catalogReq, retrieveMetaCallback, pWrapper, &pRequest->body.queryJob);
+                                &catalogReq, retrieveMetaCallback, pWrapper, &pRequest->body.queryJob);
-
+  if (code == TSDB_CODE_SUCCESS) {
-  if (code != TSDB_CODE_SUCCESS) {
+    return;
-    goto _error;
   }
 
-  return;
-
-  // todo handle the retry process
-
-  //    if (TSDB_CODE_SUCCESS == code || NEED_CLIENT_HANDLE_ERROR(code)) {
-  //      TSWAP(pRequest->dbList, (pQuery)->pDbList);
-  //      TSWAP(pRequest->tableList, (pQuery)->pTableList);
-  //    }
-
   _error:
+  tscError("0x%"PRIx64" error happens, code:%s, reqId:0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId);
   terrno = code;
   pRequest->code = code;
-  fp(param, pRequest, code);
+  pRequest->body.queryFp(pRequest->body.param, pRequest, code);
 }
 
 static void fetchCallback(void* pResult, void* param, int32_t code) {
@@ -751,14 +770,15 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
   pRequest->code = setQueryResultFromRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pResultInfo->pData, true, false);
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     pResultInfo->numOfRows = 0;
-
     pRequest->code = code;
-    pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
+    tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code),
+             pRequest->requestId);
+  } else {
+    tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,
+             pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed,
+             pRequest->requestId);
   }
 
-  tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,
-           pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId);
-
   pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows);
 }
 

source/client/src/clientMsgHandler.c

@@ -21,8 +21,6 @@
 #include "tdef.h"
 #include "tname.h"
 
-int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
-
 static void setErrno(SRequestObj* pRequest, int32_t code) {
   pRequest->code = code;
   terrno = code;
@@ -107,10 +105,7 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) {
 
   assert(pRequest != NULL);
   pMsgSendInfo->msgInfo = pRequest->body.requestMsg;
-
+  pMsgSendInfo->fp = getMsgRspHandle(pRequest->type);
-  pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)
-                         ? genericRspCallback
-                         : handleRequestRspFp[TMSG_INDEX(pRequest->type)];
   return pMsgSendInfo;
 }
 
@@ -209,7 +204,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
   return 0;
 }
 
-int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) {
+int32_t processCreateSTableRsp(void* param, const SDataBuf* pMsg, int32_t code) {
   assert(pMsg != NULL && param != NULL);
   SRequestObj* pRequest = param;
 
@@ -219,6 +214,7 @@ int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) {
   }
 
   if (pRequest->body.queryFp != NULL) {
+    removeMeta(pRequest->pTscObj, pRequest->tableList);
     pRequest->body.queryFp(pRequest->body.param, pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
@@ -251,30 +247,54 @@ int32_t processAlterStbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
   SRequestObj* pRequest = param;
   if (code != TSDB_CODE_SUCCESS) {
     setErrno(pRequest, code);
-    tsem_post(&pRequest->body.rspSem);
+  } else {
-    return code;
+    SMAlterStbRsp alterRsp = {0};
+    SDecoder      coder = {0};
+    tDecoderInit(&coder, pMsg->pData, pMsg->len);
+    tDecodeSMAlterStbRsp(&coder, &alterRsp);
+    tDecoderClear(&coder);
+
+    pRequest->body.resInfo.execRes.msgType = TDMT_MND_ALTER_STB;
+    pRequest->body.resInfo.execRes.res = alterRsp.pMeta;
   }
 
-  SMAlterStbRsp alterRsp = {0};
+  if (pRequest->body.queryFp != NULL) {
-  SDecoder coder = {0};
+    SQueryExecRes* pRes = &pRequest->body.resInfo.execRes;
-  tDecoderInit(&coder, pMsg->pData, pMsg->len);
-  tDecodeSMAlterStbRsp(&coder, &alterRsp);
-  tDecoderClear(&coder);
 
-  pRequest->body.resInfo.execRes.msgType = TDMT_MND_ALTER_STB;
+    if (code == TSDB_CODE_SUCCESS) {
-  pRequest->body.resInfo.execRes.res = alterRsp.pMeta;
+      SCatalog* pCatalog = NULL;
+      int32_t   ret = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
+      if (pRes->res != NULL) {
+        ret = handleAlterTbExecRes(pRes->res, pCatalog);
+      }
 
-  tsem_post(&pRequest->body.rspSem);
+      if (ret != TSDB_CODE_SUCCESS) {
+        code = ret;
+      }
+    }
+
+    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+  } else {
+    tsem_post(&pRequest->body.rspSem);
+  }
   return code;
 }
 
-
+__async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
-// todo refactor: this arraylist is too large
+  switch (msgType) {
-void initMsgHandleFp() {
+    case TDMT_MND_CONNECT:
-  handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp;
+      return processConnectRsp;
-  handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp;
+    case TDMT_MND_CREATE_DB:
-  handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp;
+      return processCreateDbRsp;
-  handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = processCreateTableRsp;
+    case TDMT_MND_USE_DB:
-  handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp;
+      return processUseDbRsp;
-  handleRequestRspFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = processAlterStbRsp;
+    case TDMT_MND_CREATE_STB:
+      return processCreateSTableRsp;
+    case TDMT_MND_DROP_DB:
+      return processDropDbRsp;
+    case TDMT_MND_ALTER_STB:
+      return processAlterStbRsp;
+    default:
+      return genericRspCallback;
+  }
 }

source/client/src/clientSml.c

@@ -2364,7 +2364,7 @@ static int32_t isSchemalessDb(SSmlHandle* info){
  */
 
 TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) {
-  SRequestObj* request = (SRequestObj*)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj* request = (SRequestObj*)createRequest((STscObj *)taos, TSDB_SQL_INSERT);
   if(!request){
     uError("SML:taos_schemaless_insert error request is null");
     return NULL;

source/client/test/clientTests.cpp

@@ -778,7 +778,35 @@ TEST(testCase, async_api_test) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
 
-  taos_query_a(pConn, "drop table test.tm0", queryCallback, pConn);
+  taos_query(pConn, "use test");
+
+  TAOS_RES* pRes = taos_query(pConn, "select * from t1");
+
+  taos_query(pConn, "alter table t1 add column b int");
+  pRes = taos_query(pConn, "insert into t1 values(now, 1, 2)");
+  if (taos_errno(pRes) != 0) {
+    printf("failed, reason:%s\n", taos_errstr(pRes));
+  }
+
+//  int32_t n = 0;
+//  TAOS_ROW    pRow = NULL;
+//  TAOS_FIELD* pFields = taos_fetch_fields(pRes);
+//  int32_t     numOfFields = taos_num_fields(pRes);
+//
+//  char str[512] = {0};
+//  while ((pRow = taos_fetch_row(pRes)) != NULL) {
+//    int32_t* length = taos_fetch_lengths(pRes);
+//    for(int32_t i = 0; i < numOfFields; ++i) {
+//      printf("(%d):%d " , i, length[i]);
+//    }
+//    printf("\n");
+//
+//    int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
+//    printf("%s\n", str);
+//    memset(str, 0, sizeof(str));
+//  }
+
+  taos_query_a(pConn, "alter table test.m1 comment 'abcde' ", queryCallback, pConn);
   getchar();
   taos_close(pConn);
 }

source/client/test/smlTest.cpp

@@ -486,7 +486,7 @@ TEST(testCase, smlProcess_influx_Test) {
   pRes = taos_query(taos, "use inflx_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -607,7 +607,7 @@ TEST(testCase, smlParseLine_error_Test) {
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -656,7 +656,7 @@ TEST(testCase, smlProcess_telnet_Test) {
   pRes = taos_query(taos, "use telnet_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -710,7 +710,7 @@ TEST(testCase, smlProcess_json1_Test) {
   pRes = taos_query(taos, "use json_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -779,7 +779,7 @@ TEST(testCase, smlProcess_json2_Test) {
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -823,7 +823,7 @@ TEST(testCase, smlProcess_json3_Test) {
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -895,7 +895,7 @@ TEST(testCase, smlProcess_json4_Test) {
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -957,7 +957,7 @@ TEST(testCase, smlParseTelnetLine_error_Test) {
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -1006,7 +1006,7 @@ TEST(testCase, smlParseTelnetLine_diff_type_Test) {
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -1033,7 +1033,7 @@ TEST(testCase, smlParseTelnetLine_json_error_Test) {
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -1101,7 +1101,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) {
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -1146,7 +1146,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) {
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -1191,7 +1191,7 @@ TEST(testCase, sml_TD15662_Test) {
   pRes = taos_query(taos, "use db_15662");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj *)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS);
@@ -1218,7 +1218,7 @@ TEST(testCase, sml_TD15735_Test) {
   pRes = taos_query(taos, "use sml_db");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS);
@@ -1244,7 +1244,7 @@ TEST(testCase, sml_TD15742_Test) {
   pRes = taos_query(taos, "use TD15742");
   taos_free_result(pRes);
 
-  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, NULL, TSDB_SQL_INSERT);
+  SRequestObj *request = (SRequestObj *)createRequest((STscObj*)taos, TSDB_SQL_INSERT);
   ASSERT_NE(request, nullptr);
 
   SSmlHandle *info = smlBuildSmlInfo(taos, request, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS);

source/common/src/tdatablock.c

@@ -1590,7 +1590,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
       for (int32_t k = 0; k < colNum; ++k) {  // iterate by column
         SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
         STColumn*         pCol = &pTSchema->columns[k];
-        ASSERT(pCol->type == pColInfoData->info.type);
         void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
         switch (pColInfoData->info.type) {
           case TSDB_DATA_TYPE_TIMESTAMP:
@@ -1614,20 +1613,39 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks
           case TSDB_DATA_TYPE_VARBINARY:
           case TSDB_DATA_TYPE_DECIMAL:
           case TSDB_DATA_TYPE_BLOB:
+          case TSDB_DATA_TYPE_JSON:
           case TSDB_DATA_TYPE_MEDIUMBLOB:
             uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type);
             TASSERT(0);
             break;
           default:
             if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) {
-              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pColInfoData->info.type, TD_VTYPE_NORM, var, true, offset, k);
+              char tv[8] = {0};
+              if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) {
+                float v = 0;
+                GET_TYPED_DATA(v, float, pColInfoData->info.type, var);
+                SET_TYPED_DATA(&tv, pCol->type, v);
+              } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) {
+                double v = 0;
+                GET_TYPED_DATA(v, double, pColInfoData->info.type, var);
+                SET_TYPED_DATA(&tv, pCol->type, v);
+              } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) {
+                int64_t v = 0;
+                GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var);
+                SET_TYPED_DATA(&tv, pCol->type, v);
+              } else {
+                uint64_t v = 0;
+                GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var);
+                SET_TYPED_DATA(&tv, pCol->type, v);
+              }
+              tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, k);
             } else {
               uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type);
               TASSERT(0);
             }
             break;
         }
-        offset += TYPE_BYTES[pColInfoData->info.type];
+        offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation
       }
       dataLen += TD_ROW_LEN(rb.pBuf);
 #ifdef TD_DEBUG_PRINT_ROW

source/common/src/tmsg.c

@@ -694,7 +694,6 @@ void tFreeSMAltertbReq(SMAlterStbReq *pReq) {
   pReq->pFields = NULL;
 }
 
-
 int32_t tSerializeSEpSet(void *buf, int32_t bufLen, const SEpSet *pEpset) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -3674,12 +3673,12 @@ int32_t tEncodeTSma(SEncoder *pCoder, const STSma *pSma) {
     if (tEncodeCStr(pCoder, pSma->tagsFilter) < 0) return -1;
   }
   for (int32_t v = 0; v < pSma->numOfVgroups; ++v) {
-    if (tEncodeI32(pCoder, pSma->vgEpSet[v].vgId) < 0) return -1;
+    if (tEncodeI32(pCoder, pSma->pVgEpSet[v].vgId) < 0) return -1;
-    if (tEncodeI8(pCoder, pSma->vgEpSet[v].epSet.inUse) < 0) return -1;
+    if (tEncodeI8(pCoder, pSma->pVgEpSet[v].epSet.inUse) < 0) return -1;
-    int8_t numOfEps = pSma->vgEpSet[v].epSet.numOfEps;
+    int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps;
     if (tEncodeI8(pCoder, numOfEps) < 0) return -1;
     for (int32_t n = 0; n < numOfEps; ++n) {
-      const SEp *pEp = &pSma->vgEpSet[v].epSet.eps[n];
+      const SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n];
       if (tEncodeCStr(pCoder, pEp->fqdn) < 0) return -1;
       if (tEncodeU16(pCoder, pEp->port) < 0) return -1;
     }
@@ -3712,15 +3711,25 @@ int32_t tDecodeTSma(SDecoder *pCoder, STSma *pSma) {
   } else {
     pSma->tagsFilter = NULL;
   }
-  for (int32_t v = 0; v < pSma->numOfVgroups; ++v) {
+  if (pSma->numOfVgroups > 0) {
-    if (tDecodeI32(pCoder, &pSma->vgEpSet[v].vgId) < 0) return -1;
+    pSma->pVgEpSet = (SVgEpSet *)tDecoderMalloc(pCoder, pSma->numOfVgroups * sizeof(SVgEpSet));
-    if (tDecodeI8(pCoder, &pSma->vgEpSet[v].epSet.inUse) < 0) return -1;
+    if (!pSma->pVgEpSet) {
-    if (tDecodeI8(pCoder, &pSma->vgEpSet[v].epSet.numOfEps) < 0) return -1;
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
-    int8_t numOfEps = pSma->vgEpSet[v].epSet.numOfEps;
+      return -1;
-    for (int32_t n = 0; n < numOfEps; ++n) {
+    }
-      SEp *pEp = &pSma->vgEpSet[v].epSet.eps[n];
+
-      if (tDecodeCStrTo(pCoder, pEp->fqdn) < 0) return -1;
+    memset(pSma->pVgEpSet, 0, pSma->numOfVgroups * sizeof(SVgEpSet));
-      if (tDecodeU16(pCoder, &pEp->port) < 0) return -1;
+
+    for (int32_t v = 0; v < pSma->numOfVgroups; ++v) {
+      if (tDecodeI32(pCoder, &pSma->pVgEpSet[v].vgId) < 0) return -1;
+      if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.inUse) < 0) return -1;
+      if (tDecodeI8(pCoder, &pSma->pVgEpSet[v].epSet.numOfEps) < 0) return -1;
+      int8_t numOfEps = pSma->pVgEpSet[v].epSet.numOfEps;
+      for (int32_t n = 0; n < numOfEps; ++n) {
+        SEp *pEp = &pSma->pVgEpSet[v].epSet.eps[n];
+        if (tDecodeCStrTo(pCoder, pEp->fqdn) < 0) return -1;
+        if (tDecodeU16(pCoder, &pEp->port) < 0) return -1;
+      }
     }
   }
 
@@ -3765,7 +3774,7 @@ int32_t tDecodeSVDropTSmaReq(SDecoder *pCoder, SVDropTSmaReq *pReq) {
   return 0;
 }
 
-int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder* pCoder, const SVGetTsmaExpWndsReq* pReq) {
+int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder *pCoder, const SVGetTsmaExpWndsReq *pReq) {
   if (tStartEncode(pCoder) < 0) return -1;
 
   if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1;
@@ -3773,10 +3782,10 @@ int32_t tEncodeSVGetTSmaExpWndsReq(SEncoder* pCoder, const SVGetTsmaExpWndsReq*
   if (tEncodeI64(pCoder, pReq->queryWindow.ekey) < 0) return -1;
 
   tEndEncode(pCoder);
-  return 0;  
+  return 0;
 }
 
-int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder* pCoder, SVGetTsmaExpWndsReq* pReq) {
+int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder *pCoder, SVGetTsmaExpWndsReq *pReq) {
   if (tStartDecode(pCoder) < 0) return -1;
 
   if (tDecodeI64(pCoder, &pReq->indexUid) < 0) return -1;
@@ -3787,7 +3796,7 @@ int32_t tDecodeSVGetTsmaExpWndsReq(SDecoder* pCoder, SVGetTsmaExpWndsReq* pReq)
   return 0;
 }
 
-int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder* pCoder, const SVGetTsmaExpWndsRsp* pReq) {
+int32_t tEncodeSVGetTSmaExpWndsRsp(SEncoder *pCoder, const SVGetTsmaExpWndsRsp *pReq) {
   if (tStartEncode(pCoder) < 0) return -1;
 
   if (tEncodeI64(pCoder, pReq->indexUid) < 0) return -1;
@@ -3814,7 +3823,7 @@ int32_t tDecodeSVGetTsmaExpWndsRsp(SDecoder *pCoder, SVGetTsmaExpWndsRsp *pReq)
   return 0;
 }
 
-int32_t tEncodeSVDeleteReq(SEncoder* pCoder, const SVDeleteReq* pReq) {
+int32_t tEncodeSVDeleteReq(SEncoder *pCoder, const SVDeleteReq *pReq) {
   if (tStartEncode(pCoder) < 0) return -1;
 
   if (tEncodeI64(pCoder, pReq->delUid) < 0) return -1;
@@ -3832,7 +3841,7 @@ int32_t tEncodeSVDeleteReq(SEncoder* pCoder, const SVDeleteReq* pReq) {
   return 0;
 }
 
-int32_t tDecodeSVDeleteReq(SDecoder* pCoder, SVDeleteReq* pReq) {
+int32_t tDecodeSVDeleteReq(SDecoder *pCoder, SVDeleteReq *pReq) {
   if (tStartDecode(pCoder) < 0) return -1;
 
   if (tDecodeI64(pCoder, &pReq->delUid) < 0) return -1;
@@ -3850,7 +3859,7 @@ int32_t tDecodeSVDeleteReq(SDecoder* pCoder, SVDeleteReq* pReq) {
   return 0;
 }
 
-int32_t tEncodeSVDeleteRsp(SEncoder* pCoder, const SVDeleteRsp* pReq) {
+int32_t tEncodeSVDeleteRsp(SEncoder *pCoder, const SVDeleteRsp *pReq) {
   if (tStartEncode(pCoder) < 0) return -1;
 
   if (tEncodeI32(pCoder, pReq->code) < 0) return -1;
@@ -3860,7 +3869,7 @@ int32_t tEncodeSVDeleteRsp(SEncoder* pCoder, const SVDeleteRsp* pReq) {
   return 0;
 }
 
-int32_t tDecodeSVDeleteRsp(SDecoder* pCoder, SVDeleteRsp* pReq) {
+int32_t tDecodeSVDeleteRsp(SDecoder *pCoder, SVDeleteRsp *pReq) {
   if (tStartDecode(pCoder) < 0) return -1;
 
   if (tDecodeI32(pCoder, &pReq->code) < 0) return -1;
@@ -4502,7 +4511,7 @@ int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) {
 }
 
 int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) {
-  int32_t meta = 0;
+  int32_t  meta = 0;
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
 
@@ -4543,7 +4552,7 @@ int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp) {
 }
 
 int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) {
-  int32_t meta = 0;
+  int32_t  meta = 0;
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
 
@@ -4559,7 +4568,7 @@ int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp
   return 0;
 }
 
-void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp) {
+void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) {
   if (NULL == pRsp) {
     return;
   }
@@ -4569,6 +4578,3 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp) {
     taosMemoryFree(pRsp->pMeta);
   }
 }
-
-
-

source/dnode/mnode/impl/inc/mndDef.h

@@ -298,28 +298,30 @@ typedef struct {
 } SVgObj;
 
 typedef struct {
-  char    name[TSDB_TABLE_FNAME_LEN];
+  char      name[TSDB_TABLE_FNAME_LEN];
-  char    stb[TSDB_TABLE_FNAME_LEN];
+  char      stb[TSDB_TABLE_FNAME_LEN];
-  char    db[TSDB_DB_FNAME_LEN];
+  char      db[TSDB_DB_FNAME_LEN];
-  int64_t createdTime;
+  int64_t   createdTime;
-  int64_t uid;
+  int64_t   uid;
-  int64_t stbUid;
+  int64_t   stbUid;
-  int64_t dbUid;
+  int64_t   dbUid;
-  int8_t  intervalUnit;
+  int8_t    intervalUnit;
-  int8_t  slidingUnit;
+  int8_t    slidingUnit;
-  int8_t  timezone;
+  int8_t    timezone;
-  int32_t dstVgId;  // for stream
+  int32_t   dstVgId;  // for stream
-  int64_t interval;
+  int64_t   interval;
-  int64_t offset;
+  int64_t   offset;
-  int64_t sliding;
+  int64_t   sliding;
-  int32_t exprLen;  // strlen + 1
+  int32_t   exprLen;  // strlen + 1
-  int32_t tagsFilterLen;
+  int32_t   tagsFilterLen;
-  int32_t sqlLen;
+  int32_t   sqlLen;
-  int32_t astLen;
+  int32_t   astLen;
-  char*   expr;
+  int32_t   numOfVgroups;
-  char*   tagsFilter;
+  char*     expr;
-  char*   sql;
+  char*     tagsFilter;
-  char*   ast;
+  char*     sql;
+  char*     ast;
+  SVgEpSet* pVgEpSet;
 } SSmaObj;
 
 typedef struct {

source/dnode/mnode/impl/inc/mndDnode.h

@@ -28,7 +28,7 @@ SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId);
 void       mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode);
 SEpSet     mndGetDnodeEpset(SDnodeObj *pDnode);
 int32_t    mndGetDnodeSize(SMnode *pMnode);
-bool       mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs);
+bool       mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndMnode.h

@@ -28,6 +28,7 @@ SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId);
 void       mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj);
 bool       mndIsMnode(SMnode *pMnode, int32_t dnodeId);
 void       mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet);
+int32_t    mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndTopic.h

@@ -37,7 +37,7 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]);
 
 int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic);
 
-bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *colIds);
+int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/inc/mndVgroup.h

@@ -30,15 +30,21 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
 SEpSet   mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup);
 int32_t  mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId);
 
-int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup);
+SArray *mndBuildDnodesArray(SMnode *, int32_t exceptDnodeId);
-int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
+int32_t mndAllocSmaVgroup(SMnode *, SDbObj *pDb, SVgObj *pVgroup);
-SArray *mndBuildDnodesArray(SMnode *pMnode);
+int32_t mndAllocVgroup(SMnode *, SDbObj *pDb, SVgObj **ppVgroups);
-int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray);
+int32_t mndAddVnodeToVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray);
-int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pVgId);
+int32_t mndRemoveVnodeFromVgroup(SMnode *, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid);
+int32_t mndAddCreateVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool standby);
+int32_t mndAddAlterVnodeConfirmAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup);
+int32_t mndAddAlterVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType);
+int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo);
+int32_t mndSetMoveVgroupInfoToTrans(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vn, SArray *pArray);
+int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId);
 
-void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen, bool standby);
+void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *cntlen, bool standby);
-void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
+void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
-void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
+void *mndBuildAlterVnodeReq(SMnode *, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndBnode.c

@@ -299,7 +299,7 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckNodeAuth(pUser)) {
+  if (mndCheckNodeAuth(pUser) != 0) {
     goto _OVER;
   }
 
@@ -409,7 +409,7 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckNodeAuth(pUser)) {
+  if (mndCheckNodeAuth(pUser) != 0) {
     goto _OVER;
   }
 

source/dnode/mnode/impl/src/mndDb.c

@@ -263,111 +263,6 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) {
   sdbRelease(pSdb, pDb);
 }
 
-static int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid,
-                                       bool standby) {
-  STransAction action = {0};
-
-  SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
-  if (pDnode == NULL) return -1;
-  action.epSet = mndGetDnodeEpset(pDnode);
-  mndReleaseDnode(pMnode, pDnode);
-
-  int32_t contLen = 0;
-  void   *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen, standby);
-  if (pReq == NULL) return -1;
-
-  action.pCont = pReq;
-  action.contLen = contLen;
-  action.msgType = TDMT_DND_CREATE_VNODE;
-  action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
-
-  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-    taosMemoryFree(pReq);
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) {
-  STransAction action = {0};
-  action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
-
-  int32_t   contLen = sizeof(SMsgHead);
-  SMsgHead *pHead = taosMemoryMalloc(contLen);
-  if (pHead == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  pHead->contLen = htonl(contLen);
-  pHead->vgId = htonl(pVgroup->vgId);
-
-  action.pCont = pHead;
-  action.contLen = contLen;
-  action.msgType = TDMT_VND_ALTER_CONFIRM;
-
-  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-    taosMemoryFree(pHead);
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType) {
-  STransAction action = {0};
-  action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
-
-  int32_t contLen = 0;
-  void   *pReq = mndBuildAlterVnodeReq(pMnode, pDb, pVgroup, &contLen);
-  if (pReq == NULL) return -1;
-
-  action.pCont = pReq;
-  action.contLen = contLen;
-  action.msgType = msgType;
-
-  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-    taosMemoryFree(pReq);
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid,
-                                     bool isRedo) {
-  STransAction action = {0};
-
-  SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
-  if (pDnode == NULL) return -1;
-  action.epSet = mndGetDnodeEpset(pDnode);
-  mndReleaseDnode(pMnode, pDnode);
-
-  int32_t contLen = 0;
-  void   *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
-  if (pReq == NULL) return -1;
-
-  action.pCont = pReq;
-  action.contLen = contLen;
-  action.msgType = TDMT_DND_DROP_VNODE;
-  action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
-
-  if (isRedo) {
-    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
-      taosMemoryFree(pReq);
-      return -1;
-    }
-  } else {
-    if (mndTransAppendUndoAction(pTrans, &action) != 0) {
-      taosMemoryFree(pReq);
-      return -1;
-    }
-  }
-
-  return 0;
-}
-
 static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) {
   char *pos = strstr(dbName, TS_PATH_DELIMITER);
   if (pos == NULL) {
@@ -795,7 +690,7 @@ static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj
 static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOld, SDbObj *pNew) {
   SSdb   *pSdb = pMnode->pSdb;
   void   *pIter = NULL;
-  SArray *pArray = mndBuildDnodesArray(pMnode);
+  SArray *pArray = mndBuildDnodesArray(pMnode, 0);
 
   while (1) {
     SVgObj *pVgroup = NULL;
@@ -1497,20 +1392,26 @@ char *buildRetension(SArray *pRetension) {
 
   int64_t v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
   int64_t v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
-  len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c,", v1, p->freqUnit, v2, p->keepUnit);
-
-  p = taosArrayGet(pRetension, 1);
-
-  v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
-  v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
-  len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c,", v1, p->freqUnit, v2, p->keepUnit);
-
-  p = taosArrayGet(pRetension, 2);
-
-  v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
-  v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
   len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
 
+  if (size > 1) {
+    len += sprintf(p1 + len, ",");
+    p = taosArrayGet(pRetension, 1);
+
+    v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
+    v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
+    len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
+  }
+
+  if (size > 2) {
+    len += sprintf(p1 + len, ",");
+    p = taosArrayGet(pRetension, 2);
+
+    v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
+    v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
+    len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
+  }
+
   varDataSetLen(p1, len);
   return p1;
 }
@@ -1742,3 +1643,4 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) {
   SSdb *pSdb = pMnode->pSdb;
   sdbCancelFetch(pSdb, pIter);
 }
+

source/dnode/mnode/impl/src/mndDnode.c

@@ -254,7 +254,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) {
   return sdbGetSize(pSdb, SDB_DNODE);
 }
 
-bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) {
+bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs) {
   int64_t interval = TABS(pDnode->lastAccessTime - curMs);
   if (interval > 5000 * tsStatusInterval) {
     if (pDnode->rebootTime > 0) {
@@ -393,7 +393,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
 
   int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE);
   int64_t curMs = taosGetTimestampMs();
-  bool    online = mndIsDnodeOnline(pMnode, pDnode, curMs);
+  bool    online = mndIsDnodeOnline(pDnode, curMs);
   bool    dnodeChanged = (statusReq.dnodeVer != dnodeVer);
   bool    reboot = (pDnode->rebootTime != statusReq.rebootTime);
   bool    needCheck = !online || dnodeChanged || reboot;
@@ -542,7 +542,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
     goto CREATE_DNODE_OVER;
   }
 
-  if (mndCheckNodeAuth(pUser)) {
+  if (mndCheckNodeAuth(pUser) != 0) {
     goto CREATE_DNODE_OVER;
   }
 
@@ -559,30 +559,36 @@ CREATE_DNODE_OVER:
   return code;
 }
 
-static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode) {
+static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMnodeObj *pMObj) {
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq);
+  int32_t  code = -1;
-  if (pTrans == NULL) {
+  SSdbRaw *pRaw = NULL;
-    mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr());
+  STrans  *pTrans = NULL;
-    return -1;
+
-  }
+  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq);
+  if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
   mDebug("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id);
 
-  SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode);
+  pRaw = mndDnodeActionEncode(pDnode);
-  if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
+  if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER;
-    mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
+  sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING);
-    mndTransDrop(pTrans);
+  pRaw = NULL;
-    return -1;
-  }
-  sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
 
-  if (mndTransPrepare(pMnode, pTrans) != 0) {
+  pRaw = mndDnodeActionEncode(pDnode);
-    mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
+  if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
-    mndTransDrop(pTrans);
+  sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
-    return -1;
+  pRaw = NULL;
-  }
 
+  if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj) != 0) goto _OVER;
+  if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
+
+  code = 0;
+
+_OVER:
   mndTransDrop(pTrans);
-  return 0;
+  sdbFreeRaw(pRaw);
+  return code;
 }
 
 static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
@@ -595,42 +601,53 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
 
   if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto DROP_DNODE_OVER;
+    goto _OVER;
   }
 
   mDebug("dnode:%d, start to drop", dropReq.dnodeId);
 
   if (dropReq.dnodeId <= 0) {
     terrno = TSDB_CODE_MND_INVALID_DNODE_ID;
-    goto DROP_DNODE_OVER;
+    goto _OVER;
   }
 
   pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId);
   if (pDnode == NULL) {
     terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
-    goto DROP_DNODE_OVER;
+    goto _OVER;
+  }
+
+  if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
+    terrno = TSDB_CODE_NODE_OFFLINE;
+    goto _OVER;
   }
 
   pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId);
   if (pMObj != NULL) {
-    terrno = TSDB_CODE_MND_MNODE_NOT_EXIST;
+    if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) {
-    goto DROP_DNODE_OVER;
+      terrno = TSDB_CODE_MND_TOO_FEW_MNODES;
+      goto _OVER;
+    }
+    if (pMnode->selfDnodeId == dropReq.dnodeId) {
+      terrno = TSDB_CODE_MND_CANT_DROP_MASTER;
+      goto _OVER;
+    }
   }
 
   pUser = mndAcquireUser(pMnode, pReq->conn.user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
-    goto DROP_DNODE_OVER;
+    goto _OVER;
   }
 
-  if (mndCheckNodeAuth(pUser)) {
+  if (mndCheckNodeAuth(pUser) != 0) {
-    goto DROP_DNODE_OVER;
+    goto _OVER;
   }
 
-  code = mndDropDnode(pMnode, pReq, pDnode);
+  code = mndDropDnode(pMnode, pReq, pDnode, pMObj);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
-DROP_DNODE_OVER:
+_OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
     mError("dnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
   }
@@ -638,7 +655,6 @@ DROP_DNODE_OVER:
   mndReleaseDnode(pMnode, pDnode);
   mndReleaseUser(pMnode, pUser);
   mndReleaseMnode(pMnode, pMObj);
-
   return code;
 }
 
@@ -736,7 +752,7 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
   while (numOfRows < rows) {
     pShow->pIter = sdbFetch(pSdb, SDB_DNODE, pShow->pIter, (void **)&pDnode);
     if (pShow->pIter == NULL) break;
-    bool online = mndIsDnodeOnline(pMnode, pDnode, curMs);
+    bool online = mndIsDnodeOnline(pDnode, curMs);
 
     cols = 0;
 

source/dnode/mnode/impl/src/mndMain.c

@@ -529,7 +529,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
     SMonDnodeDesc desc = {0};
     desc.dnode_id = pObj->id;
     tstrncpy(desc.dnode_ep, pObj->ep, sizeof(desc.dnode_ep));
-    if (mndIsDnodeOnline(pMnode, pObj, ms)) {
+    if (mndIsDnodeOnline(pObj, ms)) {
       tstrncpy(desc.status, "ready", sizeof(desc.status));
     } else {
       tstrncpy(desc.status, "offline", sizeof(desc.status));

source/dnode/mnode/impl/src/mndMnode.c

@@ -358,9 +358,9 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode,
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq);
   if (pTrans == NULL) goto _OVER;
-
-  mDebug("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
   mndTransSetSerial(pTrans);
+  mDebug("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
+
   if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
   if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
   if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
@@ -408,7 +408,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (!mndIsDnodeOnline(pMnode, pDnode, taosGetTimestampMs())) {
+  if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) {
     terrno = TSDB_CODE_NODE_OFFLINE;
     goto _OVER;
   }
@@ -419,7 +419,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckNodeAuth(pUser)) {
+  if (mndCheckNodeAuth(pUser) != 0) {
     goto _OVER;
   }
 
@@ -535,18 +535,25 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode
   return 0;
 }
 
+int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
+  if (pObj == NULL) return 0;
+  if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) return -1;
+  if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1;
+  if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1;
+  if (mndTransAppendNullLog(pTrans) != 0) return -1;
+  return 0;
+}
+
 static int32_t mndDropMnode(SMnode *pMnode, SRpcMsg *pReq, SMnodeObj *pObj) {
   int32_t code = -1;
+  STrans *pTrans = NULL;
 
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq);
+  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq);
   if (pTrans == NULL) goto _OVER;
-
-  mDebug("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id);
   mndTransSetSerial(pTrans);
-  if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) goto _OVER;
+  mDebug("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id);
-  if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) goto _OVER;
+
-  if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) goto _OVER;
+  if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER;
-  if (mndTransAppendNullLog(pTrans) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;
@@ -596,7 +603,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckNodeAuth(pUser)) {
+  if (mndCheckNodeAuth(pUser) != 0) {
     goto _OVER;
   }
 
@@ -642,7 +649,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
     if (pObj->id == pMnode->selfDnodeId) {
       roles = syncStr(TAOS_SYNC_STATE_LEADER);
     }
-    if (pObj->pDnode && mndIsDnodeOnline(pMnode, pObj->pDnode, curMs)) {
+    if (pObj->pDnode && mndIsDnodeOnline(pObj->pDnode, curMs)) {
       roles = syncStr(pObj->state);
     }
     char b2[12 + VARSTR_HEADER_SIZE] = {0};

source/dnode/mnode/impl/src/mndQnode.c

@@ -301,7 +301,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckNodeAuth(pUser)) {
+  if (mndCheckNodeAuth(pUser) != 0) {
     goto _OVER;
   }
 
@@ -411,7 +411,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckNodeAuth(pUser)) {
+  if (mndCheckNodeAuth(pUser) != 0) {
     goto _OVER;
   }
 

source/dnode/mnode/impl/src/mndScheduler.c

@@ -190,7 +190,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p
       sdbRelease(pSdb, pVgroup);
       continue;
     }
-    SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+    SStreamTask* pTask = tNewSStreamTask(pStream->uid, 0);
     if (pTask == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       return -1;
@@ -230,7 +230,7 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p
 int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
   ASSERT(pStream->fixedSinkVgId != 0);
   SArray*      tasks = taosArrayGetP(pStream->tasks, 0);
-  SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+  SStreamTask* pTask = tNewSStreamTask(pStream->uid, 0);
   if (pTask == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
@@ -322,7 +322,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
           sdbRelease(pSdb, pVgroup);
           continue;
         }
-        SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+        SStreamTask* pTask = tNewSStreamTask(pStream->uid, 0);
         // source part
         pTask->sourceType = TASK_SOURCE__SCAN;
         pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK;
@@ -387,7 +387,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
 
       // else, assign to vnode
       ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE);
-      SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+      SStreamTask* pTask = tNewSStreamTask(pStream->uid, 0);
 
       // source part, currently only support multi source
       pTask->sourceType = TASK_SOURCE__PIPE;
@@ -477,7 +477,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
         sdbRelease(pSdb, pVgroup);
         continue;
       }
-      SStreamTask* pTask = tNewSStreamTask(pStream->uid);
+      SStreamTask* pTask = tNewSStreamTask(pStream->uid, 0);
 
       // source part
       pTask->sourceType = TASK_SOURCE__MERGE;

source/dnode/mnode/impl/src/mndSma.c

@@ -36,6 +36,7 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw);
 static int32_t  mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma);
 static int32_t  mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSpSmatb);
 static int32_t  mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew);
+static int32_t  mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups);
 static int32_t  mndProcessMCreateSmaReq(SRpcMsg *pReq);
 static int32_t  mndProcessMDropSmaReq(SRpcMsg *pReq);
 static int32_t  mndProcessGetSmaReq(SRpcMsg *pReq);
@@ -262,7 +263,9 @@ static void *mndBuildVCreateSmaReq(SMnode *pMnode, SVgObj *pVgroup, SSmaObj *pSm
   req.sliding = pSma->sliding;
   req.expr = pSma->expr;
   req.tagsFilter = pSma->tagsFilter;
-
+  req.numOfVgroups = pSma->numOfVgroups;
+  req.pVgEpSet = pSma->pVgEpSet;
+  
   // get length
   int32_t ret = 0;
   tEncodeSize(tEncodeSVCreateTSmaReq, &req, contLen, ret);
@@ -420,6 +423,15 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans,
   mndReleaseDnode(pMnode, pDnode);
 
   // todo add sma info here
+  SVgEpSet *pVgEpSet = NULL;
+  int32_t   numOfVgroups = 0;
+  if (mndSmaGetVgEpSet(pMnode, pDb, &pVgEpSet, &numOfVgroups) != 0) {
+    return -1;
+  }
+
+  pSma->pVgEpSet = pVgEpSet;
+  pSma->numOfVgroups = numOfVgroups;
+
   int32_t smaContLen = 0;
   void   *pSmaReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &smaContLen);
   if (pSmaReq == NULL) return -1;
@@ -510,10 +522,9 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
   int32_t code = -1;
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq);
   if (pTrans == NULL) goto _OVER;
-
-  mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name);
   mndTransSetDbName(pTrans, pDb->name);
   mndTransSetSerial(pTrans);
+  mDebug("trans:%d, used to create sma:%s", pTrans->id, pCreate->name);
 
   if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
   if (mndSetCreateSmaVgroupRedoLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER;
@@ -964,3 +975,52 @@ static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) {
   SSdb *pSdb = pMnode->pSdb;
   sdbCancelFetch(pSdb, pIter);
 }
+
+static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups) {
+  SSdb     *pSdb = pMnode->pSdb;
+  SVgObj   *pVgroup = NULL;
+  void     *pIter = NULL;
+  SVgEpSet *pVgEpSet = NULL;
+  int32_t   nAllocVgs = 16;
+  int32_t   nVgs = 0;
+
+  pVgEpSet = taosMemoryCalloc(nAllocVgs, sizeof(SVgEpSet));
+  if (!pVgEpSet) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  while (1) {
+    pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
+    if (pIter == NULL) break;
+    if (pVgroup->dbUid != pDb->uid) {
+      sdbRelease(pSdb, pVgroup);
+      continue;
+    }
+
+    if (nVgs >= nAllocVgs) {
+      void *p = taosMemoryRealloc(pVgEpSet, nAllocVgs * 2 * sizeof(SVgEpSet));
+      if (!p) {
+        taosMemoryFree(pVgEpSet);
+        sdbCancelFetch(pSdb, pIter);
+        sdbRelease(pSdb, pVgroup);
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return -1;
+      }
+      pVgEpSet = (SVgEpSet *)p;
+      nAllocVgs *= 2;
+    }
+
+    (pVgEpSet + nVgs)->vgId = pVgroup->vgId;
+    (pVgEpSet + nVgs)->epSet = mndGetVgroupEpset(pMnode, pVgroup);
+
+    ++nVgs;
+
+    sdbRelease(pSdb, pVgroup);
+  }
+
+  *ppVgEpSet = pVgEpSet;
+  *numOfVgroups = nVgs;
+
+  return 0;
+}

source/dnode/mnode/impl/src/mndSnode.c

@@ -307,7 +307,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckNodeAuth(pUser)) {
+  if (mndCheckNodeAuth(pUser) != 0) {
     goto _OVER;
   }
 
@@ -419,7 +419,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (mndCheckNodeAuth(pUser)) {
+  if (mndCheckNodeAuth(pUser) != 0) {
     goto _OVER;
   }
 

source/dnode/mnode/impl/src/mndStb.c

@@ -23,6 +23,7 @@
 #include "mndPerfSchema.h"
 #include "mndScheduler.h"
 #include "mndShow.h"
+#include "mndTopic.h"
 #include "mndTrans.h"
 #include "mndUser.h"
 #include "mndVgroup.h"
@@ -394,14 +395,14 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
     req.pRSmaParam.xFilesFactor = pStb->xFilesFactor;
     req.pRSmaParam.delay = pStb->delay;
     if (pStb->ast1Len > 0) {
-      if (mndConvertRSmaTask(pStb->pAst1, pStb->uid, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, req.pRSmaParam.xFilesFactor) !=
+      if (mndConvertRSmaTask(pStb->pAst1, pStb->uid, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len,
-          TSDB_CODE_SUCCESS) {
+                             req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) {
         return NULL;
       }
     }
     if (pStb->ast2Len > 0) {
-      if (mndConvertRSmaTask(pStb->pAst2, pStb->uid, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, req.pRSmaParam.xFilesFactor) !=
+      if (mndConvertRSmaTask(pStb->pAst2, pStb->uid, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len,
-          TSDB_CODE_SUCCESS) {
+                             req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) {
         return NULL;
       }
     }
@@ -949,13 +950,18 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p
   return 0;
 }
 
-static int32_t mndDropSuperTableTag(const SStbObj *pOld, SStbObj *pNew, const char *tagName) {
+static int32_t mndDropSuperTableTag(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const char *tagName) {
   int32_t tag = mndFindSuperTableTagIndex(pOld, tagName);
   if (tag < 0) {
     terrno = TSDB_CODE_MND_TAG_NOT_EXIST;
     return -1;
   }
 
+  col_id_t colId = pOld->pTags[tag].colId;
+  if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) {
+    return -1;
+  }
+
   if (mndAllocStbSchemas(pOld, pNew) != 0) {
     return -1;
   }
@@ -968,7 +974,7 @@ static int32_t mndDropSuperTableTag(const SStbObj *pOld, SStbObj *pNew, const ch
   return 0;
 }
 
-static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pFields) {
+static int32_t mndAlterStbTagName(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, SArray *pFields) {
   if ((int32_t)taosArrayGetSize(pFields) != 2) {
     terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
     return -1;
@@ -986,6 +992,11 @@ static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pF
     return -1;
   }
 
+  col_id_t colId = pOld->pTags[tag].colId;
+  if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) {
+    return -1;
+  }
+
   if (mndFindSuperTableTagIndex(pOld, newTagName) >= 0) {
     terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST;
     return -1;
@@ -1008,13 +1019,18 @@ static int32_t mndAlterStbTagName(const SStbObj *pOld, SStbObj *pNew, SArray *pF
   return 0;
 }
 
-static int32_t mndAlterStbTagBytes(const SStbObj *pOld, SStbObj *pNew, const SField *pField) {
+static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const SField *pField) {
   int32_t tag = mndFindSuperTableTagIndex(pOld, pField->name);
   if (tag < 0) {
     terrno = TSDB_CODE_MND_TAG_NOT_EXIST;
     return -1;
   }
 
+  col_id_t colId = pOld->pTags[tag].colId;
+  if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) {
+    return -1;
+  }
+
   if (mndAllocStbSchemas(pOld, pNew) != 0) {
     return -1;
   }
@@ -1075,7 +1091,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
   return 0;
 }
 
-static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const char *colName) {
+static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const char *colName) {
   int32_t col = mndFindSuperTableColumnIndex(pOld, colName);
   if (col < 0) {
     terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST;
@@ -1092,6 +1108,11 @@ static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const
     return -1;
   }
 
+  col_id_t colId = pOld->pColumns[col].colId;
+  if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) {
+    return -1;
+  }
+
   if (mndAllocStbSchemas(pOld, pNew) != 0) {
     return -1;
   }
@@ -1104,7 +1125,7 @@ static int32_t mndDropSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, const
   return 0;
 }
 
-static int32_t mndAlterStbColumnBytes(const SStbObj *pOld, SStbObj *pNew, const SField *pField) {
+static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, const SField *pField) {
   int32_t col = mndFindSuperTableColumnIndex(pOld, pField->name);
   if (col < 0) {
     terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST;
@@ -1121,6 +1142,11 @@ static int32_t mndAlterStbColumnBytes(const SStbObj *pOld, SStbObj *pNew, const
     return -1;
   }
 
+  col_id_t colId = pOld->pColumns[col].colId;
+  if (mndCheckColAndTagModifiable(pMnode, pOld->uid, colId) != 0) {
+    return -1;
+  }
+
   if (mndAllocStbSchemas(pOld, pNew) != 0) {
     return -1;
   }
@@ -1199,7 +1225,6 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
   return 0;
 }
 
-
 static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableMetaRsp *pRsp) {
   taosRLockLatch(&pStb->lock);
 
@@ -1269,13 +1294,13 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char
   return code;
 }
 
-
+static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont,
-static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, int32_t *pLen) {
+                                     int32_t *pLen) {
   int           ret;
   SEncoder      ec = {0};
-  uint32_t      contLen = 0; 
+  uint32_t      contLen = 0;
   SMAlterStbRsp alterRsp = {0};
-  SName name = {0};
+  SName         name = {0};
   tNameFromString(&name, pAlter->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
 
   alterRsp.pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp));
@@ -1283,20 +1308,20 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
-  
+
   ret = mndBuildStbSchemaImp(pDb, pObj, name.tname, alterRsp.pMeta);
   if (ret) {
     tFreeSMAlterStbRsp(&alterRsp);
     return ret;
   }
-  
+
   tEncodeSize(tEncodeSMAlterStbRsp, &alterRsp, contLen, ret);
   if (ret) {
     tFreeSMAlterStbRsp(&alterRsp);
     return ret;
   }
 
-  void* cont = taosMemoryMalloc(contLen);
+  void *cont = taosMemoryMalloc(contLen);
   tEncoderInit(&ec, cont, contLen);
   tEncodeSMAlterStbRsp(&ec, &alterRsp);
   tEncoderClear(&ec);
@@ -1305,24 +1330,24 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S
 
   *pCont = cont;
   *pLen = contLen;
-  
+
   return 0;
 }
 
 static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) {
-  bool needRsp = true;
+  bool    needRsp = true;
+  int32_t code = -1;
+  STrans *pTrans = NULL;
+  SField *pField0 = NULL;
+
   SStbObj stbObj = {0};
   taosRLockLatch(&pOld->lock);
   memcpy(&stbObj, pOld, sizeof(SStbObj));
+  taosRUnLockLatch(&pOld->lock);
   stbObj.pColumns = NULL;
   stbObj.pTags = NULL;
   stbObj.updateTime = taosGetTimestampMs();
   stbObj.lock = 0;
-  taosRUnLockLatch(&pOld->lock);
-
-  int32_t code = -1;
-  STrans *pTrans = NULL;
-  SField *pField0 = NULL;
 
   switch (pAlter->alterType) {
     case TSDB_ALTER_TABLE_ADD_TAG:
@@ -1330,25 +1355,25 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
       break;
     case TSDB_ALTER_TABLE_DROP_TAG:
       pField0 = taosArrayGet(pAlter->pFields, 0);
-      code = mndDropSuperTableTag(pOld, &stbObj, pField0->name);
+      code = mndDropSuperTableTag(pMnode, pOld, &stbObj, pField0->name);
       break;
     case TSDB_ALTER_TABLE_UPDATE_TAG_NAME:
-      code = mndAlterStbTagName(pOld, &stbObj, pAlter->pFields);
+      code = mndAlterStbTagName(pMnode, pOld, &stbObj, pAlter->pFields);
       break;
     case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES:
       pField0 = taosArrayGet(pAlter->pFields, 0);
-      code = mndAlterStbTagBytes(pOld, &stbObj, pField0);
+      code = mndAlterStbTagBytes(pMnode, pOld, &stbObj, pField0);
       break;
     case TSDB_ALTER_TABLE_ADD_COLUMN:
       code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields);
       break;
     case TSDB_ALTER_TABLE_DROP_COLUMN:
       pField0 = taosArrayGet(pAlter->pFields, 0);
-      code = mndDropSuperTableColumn(pOld, &stbObj, pField0->name);
+      code = mndDropSuperTableColumn(pMnode, pOld, &stbObj, pField0->name);
       break;
     case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
       pField0 = taosArrayGet(pAlter->pFields, 0);
-      code = mndAlterStbColumnBytes(pOld, &stbObj, pField0);
+      code = mndAlterStbColumnBytes(pMnode, pOld, &stbObj, pField0);
       break;
     case TSDB_ALTER_TABLE_UPDATE_OPTIONS:
       needRsp = false;
@@ -1370,12 +1395,12 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
   mndTransSetDbName(pTrans, pDb->name);
 
   if (needRsp) {
-    void* pCont = NULL;
+    void   *pCont = NULL;
     int32_t contLen = 0;
-    if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen)) goto _OVER;
+    if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen) != 0) goto _OVER;
     mndTransSetRpcRsp(pTrans, pCont, contLen);
   }
-  
+
   if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
   if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;
   if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER;

source/dnode/mnode/impl/src/mndTopic.c

@@ -71,7 +71,7 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) {
   return strchr(topic, '.') + 1;
 }
 
-bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *colAndTagIds) {
+int32_t mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, col_id_t colId) {
   SSdb *pSdb = pMnode->pSdb;
   void *pIter = NULL;
   bool  found = false;
@@ -91,7 +91,7 @@ bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *col
     }
 
     SHashObj  *pColHash = NULL;
-    SNodeList *pNodeList;
+    SNodeList *pNodeList = NULL;
     nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList);
     SNode *pNode = NULL;
     FOREACH(pNode, pNodeList) {
@@ -103,22 +103,24 @@ bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *col
       if (pCol->colId > 0) {
         taosHashPut(pColHash, &pCol->colId, sizeof(int16_t), NULL, 0);
       }
+      mTrace("topic:%s, colId:%d is used", pTopic->name, pCol->colId);
     }
 
-    for (int32_t i = 0; i < taosArrayGetSize(colAndTagIds); i++) {
+    if (taosHashGet(pColHash, &colId, sizeof(int16_t)) != NULL) {
-      int16_t *pColId = taosArrayGet(colAndTagIds, i);
+      found = true;
-      if (taosHashGet(pColHash, pColId, sizeof(int16_t)) != NULL) {
+      goto NEXT;
-        found = true;
-        goto NEXT;
-      }
     }
 
   NEXT:
     sdbRelease(pSdb, pTopic);
     nodesDestroyNode(pAst);
-    if (found) return false;
+    if (found) {
+      terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC;
+      return -1;
+    }
   }
-  return true;
+
+  return 0;
 }
 
 SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {

source/dnode/mnode/impl/src/mndVgroup.c

@@ -15,11 +15,13 @@
 
 #define _DEFAULT_SOURCE
 #include "mndVgroup.h"
+#include "mndAuth.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
 #include "mndShow.h"
 #include "mndTrans.h"
+#include "mndUser.h"
 
 #define VGROUP_VER_NUMBER   1
 #define VGROUP_RESERVE_SIZE 64
@@ -34,6 +36,10 @@ static void    mndCancelGetNextVgroup(SMnode *pMnode, void *pIter);
 static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextVnode(SMnode *pMnode, void *pIter);
 
+static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq);
+static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq);
+static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq);
+
 int32_t mndInitVgroup(SMnode *pMnode) {
   SSdbTable table = {
       .sdbType = SDB_VGROUP,
@@ -344,9 +350,14 @@ static bool mndResetDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2
 static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) {
   SDnodeObj *pDnode = pObj;
   SArray    *pArray = p1;
+  int32_t    exceptDnodeId = *(int32_t *)p2;
+
+  if (exceptDnodeId == pDnode->id) {
+    return true;
+  }
 
   int64_t curMs = taosGetTimestampMs();
-  bool    online = mndIsDnodeOnline(pMnode, pDnode, curMs);
+  bool    online = mndIsDnodeOnline(pDnode, curMs);
   bool    isMnode = mndIsMnode(pMnode, pDnode->id);
   pDnode->numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id);
 
@@ -363,7 +374,7 @@ static bool mndBuildDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2
   return true;
 }
 
-SArray *mndBuildDnodesArray(SMnode *pMnode) {
+SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) {
   SSdb   *pSdb = pMnode->pSdb;
   int32_t numOfDnodes = mndGetDnodeSize(pMnode);
 
@@ -374,7 +385,7 @@ SArray *mndBuildDnodesArray(SMnode *pMnode) {
   }
 
   sdbTraverse(pSdb, SDB_DNODE, mndResetDnodesArrayFp, NULL, NULL, NULL);
-  sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, NULL, NULL);
+  sdbTraverse(pSdb, SDB_DNODE, mndBuildDnodesArrayFp, pArray, &exceptDnodeId, NULL);
   return pArray;
 }
 
@@ -422,7 +433,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr
 }
 
 int32_t mndAllocSmaVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup) {
-  SArray *pArray = mndBuildDnodesArray(pMnode);
+  SArray *pArray = mndBuildDnodesArray(pMnode, 0);
   if (pArray == NULL) return -1;
 
   pVgroup->vgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
@@ -451,7 +462,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
     goto _OVER;
   }
 
-  pArray = mndBuildDnodesArray(pMnode);
+  pArray = mndBuildDnodesArray(pMnode, 0);
   if (pArray == NULL) goto _OVER;
 
   mInfo("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray),
@@ -501,86 +512,6 @@ _OVER:
   return code;
 }
 
-int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
-  taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
-  for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
-    SDnodeObj *pDnode = taosArrayGet(pArray, i);
-    mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes);
-  }
-
-  SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica];
-  for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) {
-    SDnodeObj *pDnode = taosArrayGet(pArray, d);
-
-    bool used = false;
-    for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
-      if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) {
-        used = true;
-        break;
-      }
-    }
-    if (used) continue;
-
-    if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) {
-      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
-      return -1;
-    }
-
-    pVgid->dnodeId = pDnode->id;
-    pVgid->role = TAOS_SYNC_STATE_ERROR;
-    mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is added", pVgroup->dbName, pVgroup->vgId, pVgroup->replica,
-          pVgid->dnodeId);
-
-    pVgroup->replica++;
-    pDnode->numOfVnodes++;
-    return 0;
-  }
-
-  terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
-  mError("db:%s, failed to add vnode to vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr());
-  return -1;
-}
-
-int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid) {
-  taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
-  for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
-    SDnodeObj *pDnode = taosArrayGet(pArray, i);
-    mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes);
-  }
-
-  int32_t code = -1;
-  for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) {
-    SDnodeObj *pDnode = taosArrayGet(pArray, d);
-
-    for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
-      SVnodeGid *pVgid = &pVgroup->vnodeGid[vn];
-      if (pVgid->dnodeId == pDnode->id) {
-        mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId);
-        pDnode->numOfVnodes--;
-        pVgroup->replica--;
-        *pDelVgid = *pVgid;
-        *pVgid = pVgroup->vnodeGid[pVgroup->replica];
-        memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid));
-        code = 0;
-        goto _OVER;
-      }
-    }
-  }
-
-_OVER:
-  if (code != 0) {
-    terrno = TSDB_CODE_APP_ERROR;
-    mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr());
-    return -1;
-  }
-
-  for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
-    SVnodeGid *pVgid = &pVgroup->vnodeGid[vn];
-    mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId);
-  }
-  return 0;
-}
-
 SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup) {
   SEpSet epset = {0};
 
@@ -678,7 +609,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
         bool       online = false;
         SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgroup->vnodeGid[i].dnodeId);
         if (pDnode != NULL) {
-          online = mndIsDnodeOnline(pMnode, pDnode, curMs);
+          online = mndIsDnodeOnline(pDnode, curMs);
           mndReleaseDnode(pMnode, pDnode);
         }
 
@@ -797,3 +728,597 @@ static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter) {
   SSdb *pSdb = pMnode->pSdb;
   sdbCancelFetch(pSdb, pIter);
 }
+
+int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) {
+  taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
+  for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
+    SDnodeObj *pDnode = taosArrayGet(pArray, i);
+    mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes);
+  }
+
+  SVnodeGid *pVgid = &pVgroup->vnodeGid[pVgroup->replica];
+  for (int32_t d = 0; d < taosArrayGetSize(pArray); ++d) {
+    SDnodeObj *pDnode = taosArrayGet(pArray, d);
+
+    bool used = false;
+    for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
+      if (pDnode->id == pVgroup->vnodeGid[vn].dnodeId) {
+        used = true;
+        break;
+      }
+    }
+    if (used) continue;
+
+    if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) {
+      terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+      return -1;
+    }
+
+    pVgid->dnodeId = pDnode->id;
+    pVgid->role = TAOS_SYNC_STATE_ERROR;
+    mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is added", pVgroup->dbName, pVgroup->vgId, pVgroup->replica, pVgid->dnodeId);
+
+    pVgroup->replica++;
+    pDnode->numOfVnodes++;
+    return 0;
+  }
+
+  terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
+  mError("db:%s, failed to add vnode to vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr());
+  return -1;
+}
+
+int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *pDelVgid) {
+  taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
+  for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
+    SDnodeObj *pDnode = taosArrayGet(pArray, i);
+    mDebug("dnode:%d, equivalent vnodes:%d", pDnode->id, pDnode->numOfVnodes);
+  }
+
+  int32_t code = -1;
+  for (int32_t d = taosArrayGetSize(pArray) - 1; d >= 0; --d) {
+    SDnodeObj *pDnode = taosArrayGet(pArray, d);
+
+    for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
+      SVnodeGid *pVgid = &pVgroup->vnodeGid[vn];
+      if (pVgid->dnodeId == pDnode->id) {
+        mInfo("db:%s, vgId:%d, vn:%d dnode:%d, is removed", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId);
+        pDnode->numOfVnodes--;
+        pVgroup->replica--;
+        *pDelVgid = *pVgid;
+        *pVgid = pVgroup->vnodeGid[pVgroup->replica];
+        memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid));
+        code = 0;
+        goto _OVER;
+      }
+    }
+  }
+
+_OVER:
+  if (code != 0) {
+    terrno = TSDB_CODE_APP_ERROR;
+    mError("db:%s, failed to remove vnode from vgId:%d since %s", pVgroup->dbName, pVgroup->vgId, terrstr());
+    return -1;
+  }
+
+  for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
+    SVnodeGid *pVgid = &pVgroup->vnodeGid[vn];
+    mInfo("db:%s, vgId:%d, vn:%d dnode:%d is reserved", pVgroup->dbName, pVgroup->vgId, vn, pVgid->dnodeId);
+  }
+  return 0;
+}
+
+int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid,
+                                bool standby) {
+  STransAction action = {0};
+
+  SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
+  if (pDnode == NULL) return -1;
+  action.epSet = mndGetDnodeEpset(pDnode);
+  mndReleaseDnode(pMnode, pDnode);
+
+  int32_t contLen = 0;
+  void   *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen, standby);
+  if (pReq == NULL) return -1;
+
+  action.pCont = pReq;
+  action.contLen = contLen;
+  action.msgType = TDMT_DND_CREATE_VNODE;
+  action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED;
+
+  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+    taosMemoryFree(pReq);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) {
+  STransAction action = {0};
+  action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
+
+  int32_t   contLen = sizeof(SMsgHead);
+  SMsgHead *pHead = taosMemoryMalloc(contLen);
+  if (pHead == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pHead->contLen = htonl(contLen);
+  pHead->vgId = htonl(pVgroup->vgId);
+
+  action.pCont = pHead;
+  action.contLen = contLen;
+  action.msgType = TDMT_VND_ALTER_CONFIRM;
+
+  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+    taosMemoryFree(pHead);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, tmsg_t msgType) {
+  STransAction action = {0};
+  action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
+
+  int32_t contLen = 0;
+  void   *pReq = mndBuildAlterVnodeReq(pMnode, pDb, pVgroup, &contLen);
+  if (pReq == NULL) return -1;
+
+  action.pCont = pReq;
+  action.contLen = contLen;
+  action.msgType = msgType;
+
+  if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+    taosMemoryFree(pReq);
+    return -1;
+  }
+
+  return 0;
+}
+
+int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid,
+                              bool isRedo) {
+  STransAction action = {0};
+
+  SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
+  if (pDnode == NULL) return -1;
+  action.epSet = mndGetDnodeEpset(pDnode);
+  mndReleaseDnode(pMnode, pDnode);
+
+  int32_t contLen = 0;
+  void   *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen);
+  if (pReq == NULL) return -1;
+
+  action.pCont = pReq;
+  action.contLen = contLen;
+  action.msgType = TDMT_DND_DROP_VNODE;
+  action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
+
+  if (isRedo) {
+    if (mndTransAppendRedoAction(pTrans, &action) != 0) {
+      taosMemoryFree(pReq);
+      return -1;
+    }
+  } else {
+    if (mndTransAppendUndoAction(pTrans, &action) != 0) {
+      taosMemoryFree(pReq);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t vnIndex,
+                                    SArray *pArray) {
+  SVgObj newVg = {0};
+  memcpy(&newVg, pVgroup, sizeof(SVgObj));
+
+  mInfo("vgId:%d, vgroup info before move, replica:%d", newVg.vgId, newVg.replica);
+  for (int32_t i = 0; i < newVg.replica; ++i) {
+    mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
+  }
+
+  mInfo("vgId:%d, will add 1 vnodes", pVgroup->vgId);
+  if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1;
+  if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[1], true) != 0) return -1;
+  if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+  if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
+
+  mInfo("vgId:%d, will remove 1 vnodes", pVgroup->vgId);
+  newVg.replica--;
+  SVnodeGid del = newVg.vnodeGid[vnIndex];
+  newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica];
+  memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid));
+  if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+  if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1;
+  if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
+
+  mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica);
+  for (int32_t i = 0; i < newVg.replica; ++i) {
+    mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
+  }
+  return 0;
+}
+
+int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId) {
+  SArray *pArray = mndBuildDnodesArray(pMnode, delDnodeId);
+  if (pArray == NULL) return -1;
+
+  void *pIter = NULL;
+  while (1) {
+    SVgObj *pVgroup = NULL;
+    pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
+    if (pIter == NULL) break;
+
+    int32_t vnIndex = -1;
+    for (int32_t i = 0; i < pVgroup->replica; ++i) {
+      if (pVgroup->vnodeGid[i].dnodeId == delDnodeId) {
+        vnIndex = i;
+        break;
+      }
+    }
+
+    if (vnIndex != -1) {
+      mInfo("vgId:%d, vnode:%d will be removed from dnode:%d", pVgroup->vgId, vnIndex, delDnodeId);
+      SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
+      mndSetMoveVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, vnIndex, pArray);
+      mndReleaseDb(pMnode, pDb);
+    }
+
+    sdbRelease(pMnode->pSdb, pVgroup);
+  }
+
+  taosArrayDestroy(pArray);
+  return 0;
+}
+
+static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
+                                             int32_t newDnodeId) {
+  mDebug("vgId:%d, will add 1 vnode, replica:%d, dnode:%d", pVgroup->vgId, pVgroup->replica, newDnodeId);
+
+  SVnodeGid *pGid = &pVgroup->vnodeGid[pVgroup->replica];
+  pVgroup->replica++;
+  pGid->dnodeId = newDnodeId;
+  pGid->role = TAOS_SYNC_STATE_ERROR;
+
+  if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, pVgroup, pGid, true) != 0) return -1;
+  if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+  if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1;
+
+  return 0;
+}
+
+static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
+                                               int32_t delDnodeId) {
+  mDebug("vgId:%d, will remove 1 vnode, replica:%d, dnode:%d", pVgroup->vgId, pVgroup->replica, delDnodeId);
+
+  SVnodeGid *pGid = NULL;
+  SVnodeGid  delGid = {0};
+  for (int32_t i = 0; i < pVgroup->replica; ++i) {
+    if (pVgroup->vnodeGid[i].dnodeId == delDnodeId) {
+      pGid = &pVgroup->vnodeGid[i];
+      break;
+    }
+  }
+
+  if (pGid == NULL) return 0;
+
+  memcpy(&delGid, pGid, sizeof(SVnodeGid));
+  memcpy(pGid, &pVgroup->vnodeGid[pVgroup->replica], sizeof(SVnodeGid));
+  memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid));
+  pVgroup->replica--;
+
+  if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1;
+  if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1;
+  if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1;
+
+  return 0;
+}
+
+static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup, SDnodeObj *pNew1,
+                                     SDnodeObj *pOld1, SDnodeObj *pNew2, SDnodeObj *pOld2, SDnodeObj *pNew3,
+                                     SDnodeObj *pOld3) {
+  int32_t  code = -1;
+  SSdbRaw *pRaw = NULL;
+  STrans  *pTrans = NULL;
+
+  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq);
+  if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
+  mDebug("trans:%d, used to drop redistribute vgId:%d", pTrans->id, pVgroup->vgId);
+
+  SVgObj newVg = {0};
+  memcpy(&newVg, pVgroup, sizeof(SVgObj));
+  mInfo("vgId:%d, vgroup info before redistribute, replica:%d", newVg.vgId, newVg.replica);
+  for (int32_t i = 0; i < newVg.replica; ++i) {
+    mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
+  }
+
+  if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew1->id) != 0) goto _OVER;
+  if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld1->id) != 0) goto _OVER;
+  if (pNew2 != NULL) {
+    if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew2->id) != 0) goto _OVER;
+    if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld2->id) != 0) goto _OVER;
+    if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pNew3->id) != 0) goto _OVER;
+    if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld3->id) != 0) goto _OVER;
+  }
+
+  pRaw = mndVgroupActionEncode(&newVg);
+  if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
+  sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+  pRaw = NULL;
+
+  mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica);
+  for (int32_t i = 0; i < newVg.replica; ++i) {
+    mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
+  }
+
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
+  code = 0;
+
+_OVER:
+  mndTransDrop(pTrans);
+  sdbFreeRaw(pRaw);
+  mndReleaseDb(pMnode, pDb);
+  return code;
+}
+
+static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
+  SMnode        *pMnode = pReq->info.node;
+  SUserObj      *pUser = NULL;
+  SDnodeObj     *pNew1 = NULL;
+  SDnodeObj     *pNew2 = NULL;
+  SDnodeObj     *pNew3 = NULL;
+  SDnodeObj     *pOld1 = NULL;
+  SDnodeObj     *pOld2 = NULL;
+  SDnodeObj     *pOld3 = NULL;
+  SVgObj        *pVgroup = NULL;
+  SDbObj        *pDb = NULL;
+  int32_t        code = -1;
+  int64_t        curMs = taosGetTimestampMs();
+  SMDropMnodeReq redReq = {0};
+
+#if 0
+  if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto _OVER;
+  }
+#endif
+
+  mDebug("vgId:%d, start to redistribute", 2);
+  pUser = mndAcquireUser(pMnode, pReq->conn.user);
+  if (pUser == NULL) {
+    terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
+    goto _OVER;
+  }
+
+  if (mndCheckNodeAuth(pUser) != 0) {
+    goto _OVER;
+  }
+
+  pVgroup = mndAcquireVgroup(pMnode, 2);
+  if (pVgroup == NULL) goto _OVER;
+
+  pDb = mndAcquireDb(pMnode, pVgroup->dbName);
+  if (pDb == NULL) goto _OVER;
+
+  if (pVgroup->replica == 1) {
+    pNew1 = mndAcquireDnode(pMnode, 1);
+    pOld1 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[0].dnodeId);
+    if (pNew1 == NULL || pOld1 == NULL) goto _OVER;
+    if (!mndIsDnodeOnline(pNew1, curMs) || !mndIsDnodeOnline(pOld1, curMs)) {
+      terrno = TSDB_CODE_NODE_OFFLINE;
+      goto _OVER;
+    }
+    if (pNew1 == pOld1) {
+      terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
+      goto _OVER;
+    }
+    if (mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, NULL, NULL, NULL, NULL) != 0) goto _OVER;
+  }
+
+  if (pVgroup->replica == 3) {
+    pNew1 = mndAcquireDnode(pMnode, 1);
+    pNew2 = mndAcquireDnode(pMnode, 2);
+    pNew3 = mndAcquireDnode(pMnode, 3);
+    pOld1 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[0].dnodeId);
+    pOld2 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[1].dnodeId);
+    pOld3 = mndAcquireDnode(pMnode, pVgroup->vnodeGid[2].dnodeId);
+    if (pNew1 == NULL || pOld1 == NULL || pNew2 == NULL || pOld2 == NULL || pNew3 == NULL || pOld3 == NULL) goto _OVER;
+    if (!mndIsDnodeOnline(pNew1, curMs) || !mndIsDnodeOnline(pOld1, curMs) || !mndIsDnodeOnline(pNew2, curMs) ||
+        !mndIsDnodeOnline(pOld2, curMs) || !mndIsDnodeOnline(pNew3, curMs) || !mndIsDnodeOnline(pOld3, curMs)) {
+      terrno = TSDB_CODE_NODE_OFFLINE;
+      goto _OVER;
+    }
+    bool changed = true;
+    if (pNew1 != pOld1 || pNew1 != pOld2 || pNew1 != pOld3) changed = true;
+    if (pNew2 != pOld1 || pNew2 != pOld2 || pNew2 != pOld3) changed = true;
+    if (pNew3 != pOld1 || pNew3 != pOld2 || pNew3 != pOld3) changed = true;
+    if (!changed) {
+      terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
+      goto _OVER;
+    }
+    if (mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, pNew2, pOld2, pNew3, pOld3) != 0) goto _OVER;
+  }
+
+  if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
+
+_OVER:
+  if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
+    mDebug("vgId:%d, failed to redistribute since %s", 1, terrstr());
+  }
+
+  mndReleaseDnode(pMnode, pNew1);
+  mndReleaseDnode(pMnode, pNew2);
+  mndReleaseDnode(pMnode, pNew3);
+  mndReleaseDnode(pMnode, pOld1);
+  mndReleaseDnode(pMnode, pOld2);
+  mndReleaseDnode(pMnode, pOld3);
+  mndReleaseUser(pMnode, pUser);
+  mndReleaseVgroup(pMnode, pVgroup);
+  mndReleaseDb(pMnode, pDb);
+
+  return code;
+}
+
+static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { return 0; }
+
+static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
+                                              SDnodeObj *pSrc, SDnodeObj *pDst) {
+  SVgObj newVg = {0};
+  memcpy(&newVg, pVgroup, sizeof(SVgObj));
+  mInfo("vgId:%d, vgroup info before balance, replica:%d", newVg.vgId, newVg.replica);
+  for (int32_t i = 0; i < newVg.replica; ++i) {
+    mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
+  }
+
+  if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pDst->id) != 0) return -1;
+  if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pSrc->id) != 0) return -1;
+
+  SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
+  if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) {
+    sdbFreeRaw(pRaw);
+    return -1;
+  }
+  sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+
+  mInfo("vgId:%d, vgroup info after balance, replica:%d", newVg.vgId, newVg.replica);
+  for (int32_t i = 0; i < newVg.replica; ++i) {
+    mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId);
+  }
+  return 0;
+}
+
+static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst) {
+  void   *pIter = NULL;
+  int32_t code = -1;
+
+  while (1) {
+    SVgObj *pVgroup = NULL;
+    pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
+    if (pIter == NULL) break;
+
+    bool existInSrc = false;
+    bool existInDst = false;
+    for (int32_t i = 0; i < pVgroup->replica; ++i) {
+      SVnodeGid *pGid = &pVgroup->vnodeGid[i];
+      if (pGid->dnodeId == pSrc->id) existInSrc = true;
+      if (pGid->dnodeId == pDst->id) existInDst = true;
+    }
+
+    if (!existInSrc || existInDst) {
+      sdbRelease(pMnode->pSdb, pVgroup);
+    }
+
+    SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
+    code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst);
+    mndReleaseDb(pMnode, pDb);
+    sdbRelease(pMnode->pSdb, pVgroup);
+    break;
+  }
+
+  return code;
+}
+
+static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
+  int32_t code = -1;
+  int32_t numOfVgroups = 0;
+  STrans *pTrans = NULL;
+
+  pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq);
+  if (pTrans == NULL) goto _OVER;
+  mndTransSetSerial(pTrans);
+  mDebug("trans:%d, used to balance vgroup", pTrans->id);
+
+  while (1) {
+    taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes);
+    SDnodeObj *pSrc = taosArrayGet(pArray, 0);
+    SDnodeObj *pDst = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1);
+
+    float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes;
+    float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes;
+    if (srcScore + 0.0001 < dstScore) {
+      mDebug("trans:%d, balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id);
+      code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst);
+      if (code == 0) {
+        numOfVgroups++;
+        continue;
+      } else {
+        mError("trans:%d, failed to balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id);
+        return -1;
+      }
+    } else {
+      mDebug("trans:%d, no vgroup need to balance vgroup any more", pTrans->id);
+      break;
+    }
+  }
+
+  if (numOfVgroups <= 0) {
+    mDebug("no need to balance vgroup");
+    code = 0;
+  } else {
+    mDebug("start to balance vgroup, numOfVgroups:%d", numOfVgroups);
+    if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
+    code = TSDB_CODE_ACTION_IN_PROGRESS;
+  }
+
+_OVER:
+  mndTransDrop(pTrans);
+  return code;
+}
+
+static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
+  SMnode   *pMnode = pReq->info.node;
+  int32_t   code = -1;
+  SUserObj *pUser = NULL;
+  SArray   *pArray = NULL;
+  void     *pIter = NULL;
+  int64_t   curMs = taosGetTimestampMs();
+
+  mDebug("start to balance vgroup");
+  pUser = mndAcquireUser(pMnode, pReq->conn.user);
+  if (pUser == NULL) {
+    terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
+    goto _OVER;
+  }
+
+  if (mndCheckNodeAuth(pUser) != 0) goto _OVER;
+
+  while (1) {
+    SDnodeObj *pDnode = NULL;
+    pIter = sdbFetch(pMnode->pSdb, SDB_DNODE, pIter, (void **)&pDnode);
+    if (pIter == NULL) break;
+    if (!mndIsDnodeOnline(pDnode, curMs)) {
+      terrno = TSDB_CODE_MND_HAS_OFFLINE_DNODE;
+      mError("failed to balance vgroup since %s, dnode:%d", terrstr(), pDnode->id);
+      sdbRelease(pMnode->pSdb, pDnode);
+      goto _OVER;
+    }
+
+    sdbRelease(pMnode->pSdb, pDnode);
+  }
+
+  pArray = mndBuildDnodesArray(pMnode, 0);
+  if (pArray == NULL) goto _OVER;
+
+  if (taosArrayGetSize(pArray) < 2) {
+    mDebug("no need to balance vgroup since dnode num less than 2");
+    code = 0;
+  } else {
+    code = mndBalanceVgroup(pMnode, pReq, pArray);
+  }
+
+_OVER:
+  if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
+    mError("failed to balance vgroup since %s", terrstr());
+  }
+
+  mndReleaseUser(pMnode, pUser);
+  taosArrayDestroy(pArray);
+  return code;
+}

source/dnode/mnode/sdb/src/sdbRaw.c

@@ -42,8 +42,10 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) {
 }
 
 void sdbFreeRaw(SSdbRaw *pRaw) {
-  mTrace("raw:%p, is freed", pRaw);
+  if (pRaw != NULL) {
-  taosMemoryFree(pRaw);
+    mTrace("raw:%p, is freed", pRaw);
+    taosMemoryFree(pRaw);
+  }
 }
 
 int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) {

source/dnode/vnode/src/inc/tq.h

@@ -78,10 +78,10 @@ typedef struct {
   tmr_h          timerId;
   int8_t         tmrStopped;
   // exec
-  int8_t   inputStatus;
+  int8_t       inputStatus;
-  int8_t   execStatus;
+  int8_t       execStatus;
-  SStreamQ inputQ;
+  SStreamQueue inputQ;
-  SRWLatch lock;
+  SRWLatch     lock;
 } STqPushHandle;
 
 // tqExec
@@ -107,7 +107,7 @@ typedef struct {
     STqExecCol execCol;
     STqExecTb  execTb;
     STqExecDb  execDb;
-  } exec;
+  };
 } STqExecHandle;
 
 typedef struct {

source/dnode/vnode/src/meta/metaTable.c

@@ -103,7 +103,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
   tIndexJsonPut(pMeta->pTagIvtIdx, terms, tuid);
   indexMultiTermDestroy(terms);
 #endif
-  return -1;
+  return 0;
 }
 
 int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {

source/dnode/vnode/src/tq/tq.c

@@ -266,7 +266,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
       pHandle->execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
     }
     if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-      pHandle->execHandle.exec.execCol.qmsg = req.qmsg;
+      pHandle->execHandle.execCol.qmsg = req.qmsg;
       req.qmsg = NULL;
       for (int32_t i = 0; i < 5; i++) {
         SReadHandle handle = {
@@ -274,15 +274,14 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
             .meta = pTq->pVnode->pMeta,
             .pMsgCb = &pTq->pVnode->msgCb,
         };
-        pHandle->execHandle.exec.execCol.task[i] =
+        pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle);
-            qCreateStreamExecTaskInfo(pHandle->execHandle.exec.execCol.qmsg, &handle);
+        ASSERT(pHandle->execHandle.execCol.task[i]);
-        ASSERT(pHandle->execHandle.exec.execCol.task[i]);
       }
     } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
-      pHandle->execHandle.exec.execDb.pFilterOutTbUid =
+      pHandle->execHandle.execDb.pFilterOutTbUid =
           taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
     } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) {
-      pHandle->execHandle.exec.execTb.suid = req.suid;
+      pHandle->execHandle.execTb.suid = req.suid;
       SArray* tbUidList = taosArrayInit(0, sizeof(int64_t));
       tsdbGetCtbIdList(pTq->pVnode->pMeta, req.suid, tbUidList);
       tqDebug("vg %d, tq try get suid: %ld", pTq->pVnode->config.vgId, req.suid);
@@ -296,17 +295,20 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
       taosArrayDestroy(tbUidList);
     }
     taosHashPut(pTq->handles, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle));
+    if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
+      // TODO
+    }
   } else {
     /*ASSERT(pExec->consumerId == req.oldConsumerId);*/
     // TODO handle qmsg and exec modification
     atomic_store_32(&pHandle->epoch, -1);
     atomic_store_64(&pHandle->consumerId, req.newConsumerId);
     atomic_add_fetch_32(&pHandle->epoch, 1);
+    if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
+      // TODO
+    }
   }
 
-  if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
-    // TODO
-  }
   return 0;
 }
 
@@ -323,16 +325,13 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
   tDecoderClear(&decoder);
 
   pTask->status = TASK_STATUS__IDLE;
+
+  pTask->inputQueue = streamQueueOpen();
+  pTask->outputQueue = streamQueueOpen();
   pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
   pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
 
-  pTask->inputQ = taosOpenQueue();
+  if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) goto FAIL;
-  pTask->outputQ = taosOpenQueue();
-  pTask->inputQAll = taosAllocateQall();
-  pTask->outputQAll = taosAllocateQall();
-
-  if (pTask->inputQ == NULL || pTask->outputQ == NULL || pTask->inputQAll == NULL || pTask->outputQAll == NULL)
-    goto FAIL;
 
   // exec
   if (pTask->execType != TASK_EXEC__NONE) {
@@ -369,10 +368,8 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) {
 
   return 0;
 FAIL:
-  if (pTask->inputQ) taosCloseQueue(pTask->inputQ);
+  if (pTask->inputQueue) streamQueueClose(pTask->inputQueue);
-  if (pTask->outputQ) taosCloseQueue(pTask->outputQ);
+  if (pTask->outputQueue) streamQueueClose(pTask->outputQueue);
-  if (pTask->inputQAll) taosFreeQall(pTask->inputQAll);
-  if (pTask->outputQAll) taosFreeQall(pTask->outputQAll);
   if (pTask) taosMemoryFree(pTask);
   return -1;
 }
@@ -393,38 +390,16 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) {
     SStreamTask* pTask = (SStreamTask*)pIter;
     if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue;
 
-    int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
+    if (!failed) {
-    if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
+      if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) {
-      if (failed) {
-        atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
         continue;
       }
 
-      SStreamDataSubmit* pSubmitClone = streamSubmitRefClone(pSubmit);
+      if (streamTriggerByWrite(pTask, pTq->pVnode->config.vgId, &pTq->pVnode->msgCb) < 0) {
-      if (pSubmitClone == NULL) {
-        atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
         continue;
       }
-      taosWriteQitem(pTask->inputQ, pSubmitClone);
-
-      int8_t execStatus = atomic_load_8(&pTask->status);
-      if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) {
-        SStreamTaskRunReq* pRunReq = taosMemoryMalloc(sizeof(SStreamTaskRunReq));
-        if (pRunReq == NULL) continue;
-        // TODO: do we need htonl?
-        pRunReq->head.vgId = pTq->pVnode->config.vgId;
-        pRunReq->streamId = pTask->streamId;
-        pRunReq->taskId = pTask->taskId;
-        SRpcMsg msg = {
-            .msgType = TDMT_VND_TASK_RUN,
-            .pCont = pRunReq,
-            .contLen = sizeof(SStreamTaskRunReq),
-        };
-        tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &msg);
-      }
-
     } else {
-      // blocked or stopped, do nothing
+      streamTaskInputFail(pTask);
     }
   }
 

source/dnode/vnode/src/tq/tqExec.c

@@ -59,7 +59,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqD
 
 int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId) {
   if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) {
-    qTaskInfo_t task = pExec->exec.execCol.task[workerId];
+    qTaskInfo_t task = pExec->execCol.task[workerId];
     ASSERT(task);
     qSetStreamInput(task, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
     while (1) {
@@ -101,7 +101,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR
     pRsp->withSchema = 1;
     STqReadHandle* pReader = pExec->pExecReader[workerId];
     tqReadHandleSetMsg(pReader, pReq, 0);
-    while (tqNextDataBlockFilterOut(pReader, pExec->exec.execDb.pFilterOutTbUid)) {
+    while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) {
       SSDataBlock block = {0};
       if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows,
                               &block.info.numOfCols) < 0) {

source/dnode/vnode/src/tq/tqMeta.c

@@ -22,7 +22,7 @@ static int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) {
   if (tEncodeI32(pEncoder, pHandle->epoch) < 0) return -1;
   if (tEncodeI8(pEncoder, pHandle->execHandle.subType) < 0) return -1;
   if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-    if (tEncodeCStr(pEncoder, pHandle->execHandle.exec.execCol.qmsg) < 0) return -1;
+    if (tEncodeCStr(pEncoder, pHandle->execHandle.execCol.qmsg) < 0) return -1;
   }
   tEndEncode(pEncoder);
   return pEncoder->pos;
@@ -35,7 +35,7 @@ static int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
   if (tDecodeI32(pDecoder, &pHandle->epoch) < 0) return -1;
   if (tDecodeI8(pDecoder, &pHandle->execHandle.subType) < 0) return -1;
   if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
-    if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.exec.execCol.qmsg) < 0) return -1;
+    if (tDecodeCStrAlloc(pDecoder, &pHandle->execHandle.execCol.qmsg) < 0) return -1;
   }
   tEndDecode(pDecoder);
   return 0;
@@ -88,12 +88,11 @@ int32_t tqMetaOpen(STQ* pTq) {
             .meta = pTq->pVnode->pMeta,
             .pMsgCb = &pTq->pVnode->msgCb,
         };
-        handle.execHandle.exec.execCol.task[i] =
+        handle.execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(handle.execHandle.execCol.qmsg, &reader);
-            qCreateStreamExecTaskInfo(handle.execHandle.exec.execCol.qmsg, &reader);
+        ASSERT(handle.execHandle.execCol.task[i]);
-        ASSERT(handle.execHandle.exec.execCol.task[i]);
       }
     } else {
-      handle.execHandle.exec.execDb.pFilterOutTbUid =
+      handle.execHandle.execDb.pFilterOutTbUid =
           taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
     }
     taosHashPut(pTq->handles, pKey, kLen, &handle, sizeof(STqHandle));

source/dnode/vnode/src/tq/tqPush.c

@@ -29,13 +29,13 @@ static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubm
     }
     // update processed
     atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver);
-    streamQSetSuccess(&pHandle->pushHandle.inputQ);
+    streamQueueProcessSuccess(&pHandle->pushHandle.inputQ);
     streamDataSubmitRefDec(pSubmit);
     if (pRsp->blockNum > 0) {
       *ppSubmit = pSubmit;
       return 0;
     } else {
-      pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ);
+      pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ);
     }
   }
   *ppSubmit = pSubmit;
@@ -52,14 +52,14 @@ int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) {
     // 2. check processedVer
     // 2.1. if not missed, get msg from queue
     // 2.2. if missed, scan wal
-    pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ);
+    pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ);
     while (pHandle->pushHandle.processedVer <= pSubmit->ver) {
       // read from wal
     }
     while (pHandle->pushHandle.processedVer > pSubmit->ver + 1) {
-      streamQSetSuccess(&pHandle->pushHandle.inputQ);
+      streamQueueProcessSuccess(&pHandle->pushHandle.inputQ);
       streamDataSubmitRefDec(pSubmit);
-      pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ);
+      pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ);
       if (pSubmit == NULL) break;
     }
     // 3. exec, after each success, update processed ver

source/dnode/vnode/src/tq/tqRead.c

@@ -307,7 +307,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
     STqHandle* pExec = (STqHandle*)pIter;
     if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
       for (int32_t i = 0; i < 5; i++) {
-        int32_t code = qUpdateQualifiedTableId(pExec->execHandle.exec.execCol.task[i], tbUidList, isAdd);
+        int32_t code = qUpdateQualifiedTableId(pExec->execHandle.execCol.task[i], tbUidList, isAdd);
         ASSERT(code == 0);
       }
     } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) {
@@ -315,7 +315,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
         int32_t sz = taosArrayGetSize(tbUidList);
         for (int32_t i = 0; i < sz; i++) {
           int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i);
-          taosHashPut(pExec->execHandle.exec.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0);
+          taosHashPut(pExec->execHandle.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0);
         }
       }
     } else {

source/libs/catalog/inc/catalogInt.h

@@ -515,6 +515,7 @@ int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2);
 int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2);
 void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput);
 int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target);
+char *ctgTaskTypeStr(CTG_TASK_TYPE type);
 
 
 extern SCatalogMgmt gCtgMgmt;

source/libs/catalog/src/ctgAsync.c

@@ -44,7 +44,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, SName *name) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname);
+  qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -67,7 +67,7 @@ int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, task.type, dbFName);
+  qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -90,7 +90,7 @@ int32_t ctgInitGetDbCfgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, task.type, dbFName);
+  qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -113,7 +113,7 @@ int32_t ctgInitGetDbInfoTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:%" PRIx64 " task %d type %d initialized, dbFName:%s", pJob->queryId, taskIdx, task.type, dbFName);
+  qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -143,7 +143,7 @@ int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, SName *name) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:0x%" PRIx64 " task %d type %d initialized, tableName:%s", pJob->queryId, taskIdx, task.type, name->tname);
+  qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -158,7 +158,7 @@ int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:%" PRIx64 " task %d type %d initialized", pJob->queryId, taskIdx, task.type);
+  qDebug("QID:%" PRIx64 " the %d task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type));
 
   return TSDB_CODE_SUCCESS;
 }
@@ -181,7 +181,7 @@ int32_t ctgInitGetIndexTask(SCtgJob *pJob, int32_t taskIdx, char *name) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:%" PRIx64 " task %d type %d initialized, indexFName:%s", pJob->queryId, taskIdx, task.type, name);
+  qDebug("QID:%" PRIx64 " the %d task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -204,7 +204,7 @@ int32_t ctgInitGetUdfTask(SCtgJob *pJob, int32_t taskIdx, char *name) {
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:%" PRIx64 " task %d type %d initialized, udfName:%s", pJob->queryId, taskIdx, task.type, name);
+  qDebug("QID:%" PRIx64 " the %d task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name);
 
   return TSDB_CODE_SUCCESS;
 }
@@ -227,11 +227,96 @@ int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, SUserAuthInfo *user)
 
   taosArrayPush(pJob->pTasks, &task);
 
-  qDebug("QID:%" PRIx64 " task %d type %d initialized, user:%s", pJob->queryId, taskIdx, task.type, user->user);
+  qDebug("QID:%" PRIx64 " the %d task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user);
 
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t ctgHandleForceUpdate(SCatalog* pCtg, SCtgJob *pJob, const SCatalogReq* pReq) {
+  int32_t dbNum = pJob->dbCfgNum + pJob->dbVgNum + pJob->dbInfoNum;
+  if (dbNum > 0) {
+    if (dbNum > pJob->dbCfgNum && dbNum > pJob->dbVgNum && dbNum > pJob->dbInfoNum) {
+      SHashObj* pDb = taosHashInit(dbNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+      if (NULL == pDb) {
+        CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
+      }
+      
+      for (int32_t i = 0; i < pJob->dbVgNum; ++i) {
+        char* dbFName = taosArrayGet(pReq->pDbVgroup, i);
+        taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN);
+      }
+
+      for (int32_t i = 0; i < pJob->dbCfgNum; ++i) {
+        char* dbFName = taosArrayGet(pReq->pDbCfg, i);
+        taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN);
+      }
+
+      for (int32_t i = 0; i < pJob->dbInfoNum; ++i) {
+        char* dbFName = taosArrayGet(pReq->pDbInfo, i);
+        taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN);
+      }
+
+      char* dbFName = taosHashIterate(pDb, NULL);
+      while (dbFName) {
+        ctgDropDbVgroupEnqueue(pCtg, dbFName, true);
+        dbFName = taosHashIterate(pDb, dbFName);
+      }
+
+      taosHashCleanup(pDb);      
+    } else {
+      for (int32_t i = 0; i < pJob->dbVgNum; ++i) {
+        char* dbFName = taosArrayGet(pReq->pDbVgroup, i);
+        CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true));
+      }
+      
+      for (int32_t i = 0; i < pJob->dbCfgNum; ++i) {
+        char* dbFName = taosArrayGet(pReq->pDbCfg, i);
+        CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true));
+      }
+      
+      for (int32_t i = 0; i < pJob->dbInfoNum; ++i) {
+        char* dbFName = taosArrayGet(pReq->pDbInfo, i);
+        CTG_ERR_RET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true));
+      }
+    }
+  }
+
+  int32_t tbNum = pJob->tbMetaNum + pJob->tbHashNum;
+  if (tbNum > 0) {
+    if (tbNum > pJob->tbMetaNum && tbNum > pJob->tbHashNum) {
+      SHashObj* pTb = taosHashInit(tbNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+      for (int32_t i = 0; i < pJob->tbMetaNum; ++i) {
+        SName* name = taosArrayGet(pReq->pTableMeta, i);
+        taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName));
+      }
+      
+      for (int32_t i = 0; i < pJob->tbHashNum; ++i) {
+        SName* name = taosArrayGet(pReq->pTableHash, i);
+        taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName));
+      }
+
+      SName* name = taosHashIterate(pTb, NULL);
+      while (name) {
+        catalogRemoveTableMeta(pCtg, name);
+        name = taosHashIterate(pTb, name);
+      }
+
+      taosHashCleanup(pTb);
+    } else {
+      for (int32_t i = 0; i < pJob->tbMetaNum; ++i) {
+        SName* name = taosArrayGet(pReq->pTableMeta, i);
+        catalogRemoveTableMeta(pCtg, name);
+      }
+      
+      for (int32_t i = 0; i < pJob->tbHashNum; ++i) {
+        SName* name = taosArrayGet(pReq->pTableHash, i);
+        catalogRemoveTableMeta(pCtg, name);
+      }
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
 
 int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum) {
   int32_t code = 0;
@@ -283,12 +368,13 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq*
     CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
   }
 
+  if (pReq->forceUpdate) {
+    CTG_ERR_JRET(ctgHandleForceUpdate(pCtg, pJob, pReq));
+  }
+
   int32_t taskIdx = 0;
   for (int32_t i = 0; i < dbVgNum; ++i) {
     char* dbFName = taosArrayGet(pReq->pDbVgroup, i);
-    if (pReq->forceUpdate) {
-      ctgDropDbVgroupEnqueue(pCtg, dbFName, true);
-    }
     CTG_ERR_JRET(ctgInitGetDbVgTask(pJob, taskIdx++, dbFName));
   }
 
@@ -304,9 +390,6 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq*
 
   for (int32_t i = 0; i < tbMetaNum; ++i) {
     SName* name = taosArrayGet(pReq->pTableMeta, i);
-    if (pReq->forceUpdate) {
-      catalogRemoveTableMeta(pCtg, name);
-    }
     CTG_ERR_JRET(ctgInitGetTbMetaTask(pJob, taskIdx++, name));
   }
 
@@ -342,7 +425,7 @@ int32_t ctgInitJob(CTG_PARAMS, SCtgJob** job, uint64_t reqId, const SCatalogReq*
 
   taosAcquireRef(gCtgMgmt.jobPool, pJob->refId);
 
-  qDebug("QID:%" PRIx64 ", job %" PRIx64 " initialized, task num %d", pJob->queryId, pJob->refId, *taskNum);
+  qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d", pJob->queryId, pJob->refId, *taskNum, pReq->forceUpdate);
   return TSDB_CODE_SUCCESS;
 
 
@@ -1108,7 +1191,7 @@ int32_t ctgLaunchJob(SCtgJob *pJob) {
   for (int32_t i = 0; i < taskNum; ++i) {
     SCtgTask *pTask = taosArrayGet(pJob->pTasks, i);
 
-    qDebug("QID:%" PRIx64 " start to launch task %d", pJob->queryId, pTask->taskId); 
+    qDebug("QID:0x%" PRIx64 " start to launch task %d", pJob->queryId, pTask->taskId);
     CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask));
   }
 

source/libs/catalog/src/ctgRemote.c

@@ -177,7 +177,7 @@ int32_t ctgHandleMsgCallback(void *param, const SDataBuf *pMsg, int32_t rspCode)
 
   SCtgTask *pTask = taosArrayGet(pJob->pTasks, cbParam->taskId);
 
-  qDebug("QID:%" PRIx64 " task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1));
+  qDebug("QID:0x%" PRIx64 " task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1));
   
   CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode));
  
@@ -244,7 +244,7 @@ int32_t ctgAsyncSendMsg(CTG_PARAMS, SCtgTask* pTask, int32_t msgType, void *msg,
     CTG_ERR_JRET(code);
   }
 
-  ctgDebug("req msg sent, reqId:%" PRIx64 ", msg type:%d, %s", pTask->pJob->queryId, msgType, TMSG_INFO(msgType));
+  ctgDebug("req msg sent, reqId:0x%" PRIx64 ", msg type:%d, %s", pTask->pJob->queryId, msgType, TMSG_INFO(msgType));
   return TSDB_CODE_SUCCESS;
 
 _return:
@@ -565,7 +565,7 @@ int32_t ctgGetTbMetaFromVnode(CTG_PARAMS, const SName* pTableName, SVgroupInfo *
     }
     CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName));
     
-    CTG_RET(ctgAsyncSendMsg(CTG_PARAMS_LIST(), pTask, reqType, msg, msgLen));
+    CTG_RET(ctgAsyncSendMsg(pCtg, pTrans, &vgroupInfo->epSet, pTask, reqType, msg, msgLen));
   }
 
   SRpcMsg rpcMsg = {

source/libs/catalog/src/ctgUtil.c

@@ -19,6 +19,31 @@
 #include "catalogInt.h"
 #include "systable.h"
 
+char *ctgTaskTypeStr(CTG_TASK_TYPE type) {
+  switch (type) {
+    case CTG_TASK_GET_QNODE:
+      return "[get qnode list]";
+    case CTG_TASK_GET_DB_VGROUP:
+      return "[get db vgroup]";
+    case CTG_TASK_GET_DB_CFG:
+      return "[get db cfg]";
+    case CTG_TASK_GET_DB_INFO:
+      return "[get db info]";
+    case CTG_TASK_GET_TB_META:
+      return "[get table meta]";
+    case CTG_TASK_GET_TB_HASH:
+      return "[get table hash]";
+    case CTG_TASK_GET_INDEX:
+      return "[get index]";
+    case CTG_TASK_GET_UDF:
+      return "[get udf]";
+    case CTG_TASK_GET_USER:
+      return "[get user]";
+    default:
+      return "unknown";
+  }
+}
+
 void ctgFreeSMetaData(SMetaData* pData) {
   taosArrayDestroy(pData->pTableMeta);
   pData->pTableMeta = NULL;
@@ -477,6 +502,9 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName
 
   *pVgroup = *vgInfo;
 
+  ctgDebug("Got tb %s hash vgroup, vgId %d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, vgInfo->epSet.numOfEps,
+    vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port);
+
   CTG_RET(code);
 }
 

source/libs/executor/inc/executorimpl.h

@@ -524,6 +524,17 @@ typedef struct SProjectOperatorInfo {
   int64_t            curOutput;
 } SProjectOperatorInfo;
 
+typedef struct SIndefOperatorInfo {
+  SOptrBasicInfo     binfo;
+  SAggSupporter      aggSup;
+  SArray*            pPseudoColInfo;
+
+  SExprInfo*         pScalarExpr;
+  int32_t            numOfScalarExpr;
+  SqlFunctionCtx*    pScalarCtx;
+  int32_t*           rowCellInfoOffset;
+} SIndefOperatorInfo;
+
 typedef struct SFillOperatorInfo {
   struct SFillInfo* pFillInfo;
   SSDataBlock*      pRes;
@@ -770,10 +781,13 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
 SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo,
                                            int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo);
 
+SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SNode* pCondition, SExecTaskInfo* pTaskInfo);
 SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols,
                                       SArray* pIndexMap, SExecTaskInfo* pTaskInfo);
-
+SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams,
+                                                   SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo,
+                                                   SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,

source/libs/executor/src/executorimpl.c

@@ -108,8 +108,6 @@ static SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutpu
 
 static void releaseQueryBuf(size_t numOfTables);
 
-static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr);
-
 static void destroySFillOperatorInfo(void* param, int32_t numOfOutput);
 static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput);
 static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput);
@@ -2522,8 +2520,8 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray
 }
 
 int32_t setDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData,
-                                  int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total,
+                                 int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total,
-                                  SArray* pColList) {
+                                 SArray* pColList) {
   if (pColList == NULL) {  // data from other sources
     blockCompressDecode(pRes, numOfOutput, numOfRows, pData);
   } else {  // extract data according to pColList
@@ -2677,7 +2675,7 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
 
       SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp;
       code = setDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data,
-                                       pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL);
+                                      pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL);
       if (code != 0) {
         taosMemoryFreeClear(pDataInfo->pRsp);
         goto _error;
@@ -2790,7 +2788,7 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
     SRetrieveTableRsp* pTableRsp = pDataInfo->pRsp;
     int32_t            code =
         setDataBlockFromFetchRsp(pExchangeInfo->pResult, pLoadInfo, pTableRsp->numOfRows, pTableRsp->data,
-                                  pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL);
+                                 pTableRsp->compLen, pTableRsp->numOfCols, startTs, &pDataInfo->totalRows, NULL);
 
     if (pRsp->completed == 1) {
       qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64
@@ -2856,7 +2854,7 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) {
     return seqLoadRemoteData(pOperator);
   } else {
     return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo);
-//    return concurrentlyLoadRemoteData(pOperator);
+    //    return concurrentlyLoadRemoteData(pOperator);
   }
 }
 
@@ -2911,18 +2909,18 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
     goto _error;
   }
 
-  pInfo->seqLoadData  = false;
+  pInfo->seqLoadData = false;
   pInfo->pTransporter = pTransporter;
-  pInfo->pResult      = createResDataBlock(pExNode->node.pOutputDataBlockDesc);
+  pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc);
   tsem_init(&pInfo->ready, 0, 0);
 
-  pOperator->name         = "ExchangeOperator";
+  pOperator->name = "ExchangeOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE;
-  pOperator->blocking     = false;
+  pOperator->blocking = false;
-  pOperator->status       = OP_NOT_OPENED;
+  pOperator->status = OP_NOT_OPENED;
-  pOperator->info         = pInfo;
+  pOperator->info = pInfo;
-  pOperator->numOfExprs   = pInfo->pResult->info.numOfCols;
+  pOperator->numOfExprs = pInfo->pResult->info.numOfCols;
-  pOperator->pTaskInfo    = pTaskInfo;
+  pOperator->pTaskInfo = pTaskInfo;
 
   pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL,
                                          destroyExchangeOperatorInfo, NULL, NULL, NULL);
@@ -3636,18 +3634,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
       break;
     }
 
-#if 0
-    // Return result of the previous group in the firstly.
-    if (false) {
-      if (pRes->info.rows > 0) {
-        pProjectInfo->existDataBlock = pBlock;
-        break;
-      } else {  // init output buffer for a new group data
-        initCtxOutputBuffer(pInfo->pCtx, pOperator->numOfExprs);
-      }
-    }
-#endif
-
     // the pDataBlock are always the same one, no need to call this again
     int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);
     if (code != TSDB_CODE_SUCCESS) {
@@ -3793,6 +3779,17 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
   }
 }
 
+static void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) {
+  for (int32_t i = 0; i < numOfExprs; ++i) {
+    SExprInfo* pExprInfo = &pExpr[i];
+    if (pExprInfo->pExpr->nodeType == QUERY_NODE_COLUMN) {
+      taosMemoryFree(pExprInfo->base.pParam[0].pCol);
+    }
+    taosMemoryFree(pExprInfo->base.pParam);
+    taosMemoryFree(pExprInfo->pExpr);
+  }
+}
+
 static void destroyOperatorInfo(SOperatorInfo* pOperator) {
   if (pOperator == NULL) {
     return;
@@ -3812,14 +3809,7 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) {
   }
 
   if (pOperator->pExpr != NULL) {
-    for (int32_t i = 0; i < pOperator->numOfExprs; ++i) {
+    destroyExprInfo(pOperator->pExpr, pOperator->numOfExprs);
-      SExprInfo* pExprInfo = &pOperator->pExpr[i];
-      if (pExprInfo->pExpr->nodeType == QUERY_NODE_COLUMN) {
-        taosMemoryFree(pExprInfo->base.pParam[0].pCol);
-      }
-      taosMemoryFree(pExprInfo->base.pParam);
-      taosMemoryFree(pExprInfo->pExpr);
-    }
   }
 
   taosMemoryFreeClear(pOperator->pExpr);
@@ -4008,6 +3998,19 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
   taosArrayDestroy(pInfo->pPseudoColInfo);
 }
 
+static void destroyIndefinitOperatorInfo(void* param, int32_t numOfOutput) {
+  SIndefOperatorInfo* pInfo = (SIndefOperatorInfo*)param;
+  doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
+
+  taosArrayDestroy(pInfo->pPseudoColInfo);
+  cleanupAggSup(&pInfo->aggSup);
+
+  destroySqlFunctionCtx(pInfo->pScalarCtx, numOfOutput);
+  destroyExprInfo(pInfo->pScalarExpr, pInfo->numOfScalarExpr);
+
+  taosMemoryFree(pInfo->rowCellInfoOffset);
+}
+
 void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) {
   SExchangeInfo* pExInfo = (SExchangeInfo*)param;
   taosArrayDestroy(pExInfo->pSources);
@@ -4085,6 +4088,136 @@ _error:
   return NULL;
 }
 
+static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
+  SIndefOperatorInfo* pIndefInfo = pOperator->info;
+  SOptrBasicInfo*     pInfo = &pIndefInfo->binfo;
+
+  SSDataBlock* pRes = pInfo->pRes;
+  blockDataCleanup(pRes);
+
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  if (pOperator->status == OP_EXEC_DONE) {
+    return NULL;
+  }
+
+  int64_t st = 0;
+  int32_t order = 0;
+  int32_t scanFlag = 0;
+
+  if (pOperator->cost.openCost == 0) {
+    st = taosGetTimestampUs();
+  }
+
+  SOperatorInfo* downstream = pOperator->pDownstream[0];
+
+  while (1) {
+    // The downstream exec may change the value of the newgroup, so use a local variable instead.
+    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
+    if (pBlock == NULL) {
+      doSetOperatorCompleted(pOperator);
+      break;
+    }
+
+    // the pDataBlock are always the same one, no need to call this again
+    int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);
+    if (code != TSDB_CODE_SUCCESS) {
+      longjmp(pTaskInfo->env, code);
+    }
+
+    // there is an scalar expression that needs to be calculated before apply the group aggregation.
+    if (pIndefInfo->pScalarExpr != NULL) {
+      code = projectApplyFunctions(pIndefInfo->pScalarExpr, pBlock, pBlock, pIndefInfo->pScalarCtx,
+                                   pIndefInfo->numOfScalarExpr, pIndefInfo->pPseudoColInfo);
+      if (code != TSDB_CODE_SUCCESS) {
+        longjmp(pTaskInfo->env, code);
+      }
+    }
+
+    setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, scanFlag, false);
+    blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
+
+    code = projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfExprs,
+                                 pIndefInfo->pPseudoColInfo);
+    if (code != TSDB_CODE_SUCCESS) {
+      longjmp(pTaskInfo->env, code);
+    }
+  }
+
+  size_t rows = pInfo->pRes->info.rows;
+  pOperator->resultInfo.totalRows += rows;
+
+  if (pOperator->cost.openCost == 0) {
+    pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
+  }
+
+  return (rows > 0) ? pInfo->pRes : NULL;
+}
+
+SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode,
+                                                 SExecTaskInfo* pTaskInfo) {
+  SIndefOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIndefOperatorInfo));
+  SOperatorInfo*      pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  if (pInfo == NULL || pOperator == NULL) {
+    goto _error;
+  }
+
+  SIndefRowsFuncPhysiNode* pPhyNode = (SIndefRowsFuncPhysiNode*)pNode;
+
+  int32_t    numOfExpr = 0;
+  SExprInfo* pExprInfo = createExprInfo(pPhyNode->pVectorFuncs, NULL, &numOfExpr);
+
+  int32_t numOfScalarExpr = 0;
+  if (pPhyNode->pExprs != NULL) {
+    pInfo->pScalarExpr = createExprInfo(pPhyNode->pExprs, NULL, &numOfScalarExpr);
+    pInfo->pScalarCtx = createSqlFunctionCtx(pInfo->pScalarExpr, numOfScalarExpr, &pInfo->rowCellInfoOffset);
+  }
+
+  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->node.pOutputDataBlockDesc);
+  ;
+
+  int32_t numOfRows = 4096;
+  size_t  keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
+
+  // Make sure the size of SSDataBlock will never exceed the size of 2MB.
+  int32_t TWOMB = 2 * 1024 * 1024;
+  if (numOfRows * pResBlock->info.rowSize > TWOMB) {
+    numOfRows = TWOMB / pResBlock->info.rowSize;
+  }
+  initResultSizeInfo(pOperator, numOfRows);
+
+  initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfExpr, pResBlock, keyBufSize, pTaskInfo->id.str);
+  setFunctionResultOutput(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr, pTaskInfo);
+
+  pInfo->binfo.pRes = pResBlock;
+  pInfo->numOfScalarExpr = numOfScalarExpr;
+  pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pInfo->binfo.pCtx, numOfExpr);
+
+  pOperator->name = "IndefinitOperator";
+  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT;
+  pOperator->blocking = false;
+  pOperator->status = OP_NOT_OPENED;
+  pOperator->info = pInfo;
+  pOperator->pExpr = pExprInfo;
+  pOperator->numOfExprs = numOfExpr;
+  pOperator->pTaskInfo = pTaskInfo;
+
+  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, NULL,
+                                         destroyIndefinitOperatorInfo, NULL, NULL, NULL);
+
+  int32_t code = appendDownstream(pOperator, &downstream, 1);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  return pOperator;
+
+_error:
+  taosMemoryFree(pInfo);
+  taosMemoryFree(pOperator);
+  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+  return NULL;
+}
+
 static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t numOfCols, SNodeListNode* pValNode,
                             STimeWindow win, int32_t capacity, const char* id, SInterval* pInterval, int32_t fillType) {
   SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfCols, pValNode);
@@ -4389,7 +4522,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       SScanPhysiNode*      pScanPhyNode = (SScanPhysiNode*)pPhyNode;  // simple child table.
       STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode;
       STimeWindowAggSupp   twSup = {
-          .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN};
+            .waterMark = pTableScanNode->watermark, .calTrigger = pTableScanNode->triggerType, .maxTs = INT64_MIN};
       tsdbReaderT pDataReader = NULL;
       if (pHandle->vnode) {
         pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond);
@@ -4517,6 +4650,18 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
         extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID);
 
     pOptr = createSortOperatorInfo(ops[0], pResBlock, info, pExprInfo, numOfCols, pColList, pTaskInfo);
+  } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) {
+    SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode;
+
+    SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc;
+    SSDataBlock*        pResBlock = createResDataBlock(pDescNode);
+
+    SArray* sortInfo = createSortInfo(pMergePhyNode->pMergeKeys);
+    int32_t numOfOutputCols = 0;
+    SArray* pColList =
+        extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, pTaskInfo, COL_MATCH_FROM_SLOT_ID);
+
+    pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pResBlock, sortInfo, pColList, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) {
     SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
 
@@ -4577,6 +4722,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     SInterval* pInterval = &((SIntervalAggOperatorInfo*)ops[0]->info)->interval;
     pOptr = createFillOperatorInfo(ops[0], pExprInfo, num, pInterval, &pFillNode->timeRange, pResBlock, pFillNode->mode,
                                    (SNodeListNode*)pFillNode->pValues, false, pTaskInfo);
+  } else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) {
+    pOptr = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo);
   } else {
     ASSERT(0);
   }

source/libs/executor/src/sortoperator.c

@@ -13,17 +13,18 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tdatablock.h"
 #include "executorimpl.h"
+#include "tdatablock.h"
 
 static SSDataBlock* doSort(SOperatorInfo* pOperator);
-static int32_t doOpenSortOperator(SOperatorInfo* pOperator);
+static int32_t      doOpenSortOperator(SOperatorInfo* pOperator);
-static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len);
+static int32_t      getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len);
 
 static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput);
 
-SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExprInfo* pExprInfo, int32_t numOfCols,
+SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo,
-                                      SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) {
+                                      SExprInfo* pExprInfo, int32_t numOfCols, SArray* pColMatchColInfo,
+                                      SExecTaskInfo* pTaskInfo) {
   SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo));
   SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   int32_t            rowSize = pResBlock->info.rowSize;
@@ -32,33 +33,33 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
     goto _error;
   }
 
-  pOperator->pExpr    = pExprInfo;
+  pOperator->pExpr = pExprInfo;
   pOperator->numOfExprs = numOfCols;
-  pInfo->binfo.pCtx   = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset);
+  pInfo->binfo.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset);
-  pInfo->binfo.pRes   = pResBlock;
+  pInfo->binfo.pRes = pResBlock;
 
   initResultSizeInfo(pOperator, 1024);
 
-  pInfo->pSortInfo    = pSortInfo;
+  pInfo->pSortInfo = pSortInfo;
-  pInfo->pColMatchInfo= pColMatchColInfo;
+  pInfo->pColMatchInfo = pColMatchColInfo;
-  pOperator->name     = "SortOperator";
+  pOperator->name = "SortOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT;
   pOperator->blocking = true;
-  pOperator->status   = OP_NOT_OPENED;
+  pOperator->status = OP_NOT_OPENED;
-  pOperator->info     = pInfo;
+  pOperator->info = pInfo;
 
   // lazy evaluation for the following parameter since the input datablock is not known till now.
-//  pInfo->bufPageSize  = rowSize < 1024 ? 1024 * 2 : rowSize * 2;  // there are headers, so pageSize = rowSize + header
+  //  pInfo->bufPageSize  = rowSize < 1024 ? 1024 * 2 : rowSize * 2;  // there are headers, so pageSize = rowSize +
-//  pInfo->sortBufSize  = pInfo->bufPageSize * 16;  // TODO dynamic set the available sort buffer
+  //  header pInfo->sortBufSize  = pInfo->bufPageSize * 16;  // TODO dynamic set the available sort buffer
 
   pOperator->pTaskInfo = pTaskInfo;
-  pOperator->fpSet =
+  pOperator->fpSet = createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL,
-      createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo, NULL, NULL, getExplainExecInfo);
+                                         getExplainExecInfo);
 
   int32_t code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
 
-  _error:
+_error:
   pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
   taosMemoryFree(pInfo);
   taosMemoryFree(pOperator);
@@ -68,7 +69,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR
 void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
   for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
     SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
-    bool isNull = tsortIsNullVal(pTupleHandle, i);
+    bool             isNull = tsortIsNullVal(pTupleHandle, i);
     if (isNull) {
       colDataAppendNULL(pColInfo, pBlock->info.rows);
     } else {
@@ -80,7 +81,8 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
   pBlock->info.rows += 1;
 }
 
-SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo) {
+SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
+                                SArray* pColMatchInfo) {
   blockDataCleanup(pDataBlock);
   ASSERT(taosArrayGetSize(pColMatchInfo) == pDataBlock->info.numOfCols);
 
@@ -129,10 +131,11 @@ SSDataBlock* loadNextDataBlock(void* param) {
 
 // todo refactor: merged with fetch fp
 void applyScalarFunction(SSDataBlock* pBlock, void* param) {
-  SOperatorInfo* pOperator = param;
+  SOperatorInfo*     pOperator = param;
   SSortOperatorInfo* pSort = pOperator->info;
   if (pOperator->pExpr != NULL) {
-    int32_t code = projectApplyFunctions(pOperator->pExpr, pBlock, pBlock, pSort->binfo.pCtx, pOperator->numOfExprs, NULL);
+    int32_t code =
+        projectApplyFunctions(pOperator->pExpr, pBlock, pBlock, pSort->binfo.pCtx, pOperator->numOfExprs, NULL);
     if (code != TSDB_CODE_SUCCESS) {
       longjmp(pOperator->pTaskInfo->env, code);
     }
@@ -141,7 +144,7 @@ void applyScalarFunction(SSDataBlock* pBlock, void* param) {
 
 int32_t doOpenSortOperator(SOperatorInfo* pOperator) {
   SSortOperatorInfo* pInfo = pOperator->info;
-  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  SExecTaskInfo*     pTaskInfo = pOperator->pTaskInfo;
 
   if (OPTR_IS_OPENED(pOperator)) {
     return TSDB_CODE_SUCCESS;
@@ -150,8 +153,8 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) {
   pInfo->startTs = taosGetTimestampUs();
 
   //  pInfo->binfo.pRes is not equalled to the input datablock.
-  pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT,
+  pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_SINGLESOURCE_SORT, -1, -1,
-                                             -1, -1, NULL, pTaskInfo->id.str);
+                                             NULL, pTaskInfo->id.str);
 
   tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator);
 
@@ -166,7 +169,7 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) {
     longjmp(pTaskInfo->env, terrno);
   }
 
-  pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs)/1000.0;
+  pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs) / 1000.0;
   pOperator->status = OP_RES_TO_RETURN;
 
   OPTR_SET_OPENED(pOperator);
@@ -186,7 +189,8 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
     longjmp(pTaskInfo->env, code);
   }
 
-  SSDataBlock* pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
+  SSDataBlock* pBlock =
+      getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
 
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
@@ -208,10 +212,147 @@ int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t*
   ASSERT(pOptr != NULL);
   SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo));
 
-  SSortOperatorInfo *pOperatorInfo = (SSortOperatorInfo*)pOptr->info;
+  SSortOperatorInfo* pOperatorInfo = (SSortOperatorInfo*)pOptr->info;
 
   *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle);
   *pOptrExplain = pInfo;
   *len = sizeof(SSortExecInfo);
   return TSDB_CODE_SUCCESS;
 }
+
+typedef struct SMultiwaySortMergeOperatorInfo {
+  SOptrBasicInfo binfo;
+
+  int32_t  bufPageSize;
+  uint32_t sortBufSize;  // max buffer size for in-memory sort
+
+  SArray*      pSortInfo;
+  SSortHandle* pSortHandle;
+  SArray*      pColMatchInfo;  // for index map from table scan output
+
+  int64_t startTs;  // sort start time
+} SMultiwaySortMergeOperatorInfo;
+
+int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
+  SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
+  SExecTaskInfo*                  pTaskInfo = pOperator->pTaskInfo;
+
+  if (OPTR_IS_OPENED(pOperator)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  pInfo->startTs = taosGetTimestampUs();
+
+  int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
+
+  pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->pColMatchInfo, SORT_MULTISOURCE_MERGE,
+                                             pInfo->bufPageSize, numOfBufPage, NULL, pTaskInfo->id.str);
+
+  tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL);
+
+  for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
+    SSortSource ps = {0};
+    ps.param = pOperator->pDownstream[i];
+    tsortAddSource(pInfo->pSortHandle, &ps);
+  }
+
+  int32_t code = tsortOpen(pInfo->pSortHandle);
+
+  if (code != TSDB_CODE_SUCCESS) {
+    longjmp(pTaskInfo->env, terrno);
+  }
+
+  pOperator->cost.openCost = (taosGetTimestampUs() - pInfo->startTs) / 1000.0;
+  pOperator->status = OP_RES_TO_RETURN;
+
+  OPTR_SET_OPENED(pOperator);
+  return TSDB_CODE_SUCCESS;
+}
+
+SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
+  if (pOperator->status == OP_EXEC_DONE) {
+    return NULL;
+  }
+
+  SExecTaskInfo*                  pTaskInfo = pOperator->pTaskInfo;
+  SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
+
+  int32_t code = pOperator->fpSet._openFn(pOperator);
+  if (code != TSDB_CODE_SUCCESS) {
+    longjmp(pTaskInfo->env, code);
+  }
+
+  SSDataBlock* pBlock =
+      getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, pInfo->pColMatchInfo);
+
+  if (pBlock != NULL) {
+    pOperator->resultInfo.totalRows += pBlock->info.rows;
+  } else {
+    doSetOperatorCompleted(pOperator);
+  }
+  return pBlock;
+}
+
+void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) {
+  SMultiwaySortMergeOperatorInfo * pInfo = (SMultiwaySortMergeOperatorInfo*)param;
+  pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
+
+  taosArrayDestroy(pInfo->pSortInfo);
+  taosArrayDestroy(pInfo->pColMatchInfo);
+}
+
+int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
+  ASSERT(pOptr != NULL);
+  SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo));
+
+  SMultiwaySortMergeOperatorInfo* pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info;
+
+  *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle);
+  *pOptrExplain = pInfo;
+  *len = sizeof(SSortExecInfo);
+  return TSDB_CODE_SUCCESS;
+}
+
+SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams,
+                                                   SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo,
+                                                   SExecTaskInfo* pTaskInfo) {
+  SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo));
+  SOperatorInfo*                  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  int32_t                         rowSize = pResBlock->info.rowSize;
+
+  if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) {
+    goto _error;
+  }
+
+  pInfo->binfo.pRes = pResBlock;
+
+  initResultSizeInfo(pOperator, 1024);
+
+  pInfo->pSortInfo = pSortInfo;
+  pInfo->pColMatchInfo = pColMatchColInfo;
+  pOperator->name = "MultiwaySortMerge";
+  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE;
+  pOperator->blocking = true;
+  pOperator->status = OP_NOT_OPENED;
+  pOperator->info = pInfo;
+
+  pInfo->bufPageSize = rowSize < 1024 ? 1024 : rowSize * 2;
+  pInfo->sortBufSize = pInfo->bufPageSize * 16;
+
+  pOperator->pTaskInfo = pTaskInfo;
+  pOperator->fpSet =
+      createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL,
+                          destroyMultiwaySortMergeOperatorInfo, NULL, NULL, getMultiwaySortMergeExplainExecInfo);
+
+  int32_t code = appendDownstream(pOperator, downStreams, numStreams);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+  return pOperator;
+
+_error:
+  pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
+  taosMemoryFree(pInfo);
+  taosMemoryFree(pOperator);
+  return NULL;
+}

source/libs/function/src/builtins.c

@@ -177,13 +177,6 @@ static int32_t translatePercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // param0
-  SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0);
-  if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The first parameter of PERCENTILE function can only be column");
-  }
-
   // param1
   SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);
 
@@ -218,13 +211,6 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // param0
-  SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0);
-  if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The first parameter of APERCENTILE function can only be column");
-  }
-
   // param1
   SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 1);
   if (nodeType(pParamNode1) != QUERY_NODE_VALUE) {
@@ -284,13 +270,6 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // param0
-  SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0);
-  if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The first parameter of TOP/BOTTOM function can only be column");
-  }
-
   // param1
   SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 1);
   if (nodeType(pParamNode1) != QUERY_NODE_VALUE) {
@@ -338,13 +317,6 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // param0
-  SNode* pParaNode0 = nodesListGetNode(pFunc->pParameterList, 0);
-  if (QUERY_NODE_COLUMN != nodeType(pParaNode0)) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The first parameter of ELAPSED function can only be column");
-  }
-
   uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
   if (TSDB_DATA_TYPE_TIMESTAMP != paraType) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
@@ -410,13 +382,6 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // param0
-  SNode* pParaNode0 = nodesListGetNode(pFunc->pParameterList, 0);
-  if (QUERY_NODE_COLUMN != nodeType(pParaNode0)) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The first parameter of HISTOGRAM function can only be column");
-  }
-
   uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
   if (!IS_NUMERIC_TYPE(colType)) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
@@ -449,12 +414,6 @@ static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
-  if (QUERY_NODE_COLUMN != nodeType(pPara)) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The input parameter of HYPERLOGLOG function can only be column");
-  }
-
   pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
   return TSDB_CODE_SUCCESS;
 }
@@ -474,12 +433,6 @@ static int32_t translateStateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // param0
-  SNode* pParaNode0 = nodesListGetNode(pFunc->pParameterList, 0);
-  if (QUERY_NODE_COLUMN != nodeType(pParaNode0)) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The input parameter of STATECOUNT function can only be column");
-  }
   uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
   if (!IS_NUMERIC_TYPE(colType)) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
@@ -520,12 +473,6 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // param0
-  SNode* pParaNode0 = nodesListGetNode(pFunc->pParameterList, 0);
-  if (QUERY_NODE_COLUMN != nodeType(pParaNode0)) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The input parameter of STATEDURATION function can only be column");
-  }
   uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
   if (!IS_NUMERIC_TYPE(colType)) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
@@ -573,12 +520,6 @@ static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
-  if (QUERY_NODE_COLUMN != nodeType(pPara)) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The input parameter of CSUM function can only be column");
-  }
-
   uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
   uint8_t resType;
   if (!IS_NUMERIC_TYPE(colType)) {
@@ -604,13 +545,6 @@ static int32_t translateMavg(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // param0
-  SNode* pParaNode0 = nodesListGetNode(pFunc->pParameterList, 0);
-  if (QUERY_NODE_COLUMN != nodeType(pParaNode0)) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The first parameter of MAVG function can only be column");
-  }
-
   uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
 
   // param1
@@ -640,13 +574,6 @@ static int32_t translateSample(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // param0
-  SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0);
-  if (QUERY_NODE_COLUMN != nodeType(pParamNode0)) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The first parameter of SAMPLE function can only be column");
-  }
-
   SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0);
   uint8_t    colType = pCol->resType.type;
 
@@ -684,12 +611,6 @@ static int32_t translateTail(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // param0
-  SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
-  if (QUERY_NODE_COLUMN != nodeType(pPara)) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The first parameter of TAIL function can only be column");
-  }
   SExprNode* pCol = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0);
   uint8_t    colType = pCol->resType.type;
 
@@ -766,13 +687,6 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
     return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
   }
 
-  // param0
-  SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0);
-  if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) {
-    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
-                           "The first parameter of DIFF function can only be column");
-  }
-
   uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
   if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType) {
     return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);

source/libs/index/src/indexFilter.c

@@ -38,7 +38,7 @@ typedef struct SIFParam {
   col_id_t      colId;
   int64_t       suid;  // add later
   char          dbName[TSDB_DB_NAME_LEN];
-  char          colName[TSDB_COL_NAME_LEN];
+  char          colName[TSDB_COL_NAME_LEN * 2 + 4];
 
   SIndexMetaArg arg;
 } SIFParam;
@@ -171,7 +171,10 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
   param->colId = l->colId;
   param->colValType = l->node.resType.type;
   memcpy(param->dbName, l->dbName, sizeof(l->dbName));
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-overflow"
   sprintf(param->colName, "%s_%s", l->colName, r->literal);
+#pragma GCC diagnostic pop
   param->colValType = r->typeData;
   return 0;
   // memcpy(param->colName, l->colName, sizeof(l->colName));

source/libs/nodes/src/nodesCloneFuncs.c

@@ -318,6 +318,7 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
   CLONE_NODE_LIST_FIELD(pScanPseudoCols);
   COPY_SCALAR_FIELD(tableType);
   COPY_SCALAR_FIELD(tableId);
+  COPY_SCALAR_FIELD(stableId);
   CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone);
   COPY_SCALAR_FIELD(scanType);
   COPY_OBJECT_FIELD(scanSeq[0], sizeof(uint8_t) * 2);
@@ -370,7 +371,7 @@ static SNode* logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModif
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
   COPY_SCALAR_FIELD(modifyType);
   COPY_SCALAR_FIELD(msgType);
-  CLONE_NODE_FIELD(pModifyRows);
+  CLONE_NODE_FIELD(pAffectedRows);
   COPY_SCALAR_FIELD(tableId);
   COPY_SCALAR_FIELD(tableType);
   COPY_CHAR_ARRAY_FIELD(tableFName);
@@ -387,6 +388,7 @@ static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNo
 static SNode* logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
   CLONE_NODE_LIST_FIELD(pMergeKeys);
+  CLONE_NODE_LIST_FIELD(pInputs);
   COPY_SCALAR_FIELD(numOfChannels);
   COPY_SCALAR_FIELD(srcGroupId);
   return (SNode*)pDst;
@@ -432,6 +434,12 @@ static SNode* logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLogi
   return (SNode*)pDst;
 }
 
+static SNode* logicIndefRowsFuncCopy(const SIndefRowsFuncLogicNode* pSrc, SIndefRowsFuncLogicNode* pDst) {
+  COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
+  CLONE_NODE_LIST_FIELD(pVectorFuncs);
+  return (SNode*)pDst;
+}
+
 static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) {
   COPY_OBJECT_FIELD(id, sizeof(SSubplanId));
   CLONE_NODE_FIELD(pNode);
@@ -563,6 +571,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) {
       return logicSortCopy((const SSortLogicNode*)pNode, (SSortLogicNode*)pDst);
     case QUERY_NODE_LOGIC_PLAN_PARTITION:
       return logicPartitionCopy((const SPartitionLogicNode*)pNode, (SPartitionLogicNode*)pDst);
+    case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
+      return logicIndefRowsFuncCopy((const SIndefRowsFuncLogicNode*)pNode, (SIndefRowsFuncLogicNode*)pDst);
     case QUERY_NODE_LOGIC_SUBPLAN:
       return logicSubplanCopy((const SLogicSubplan*)pNode, (SLogicSubplan*)pDst);
     default:

source/libs/nodes/src/nodesCodeFuncs.c

@@ -202,6 +202,8 @@ const char* nodesNodeName(ENodeType type) {
       return "LogicSort";
     case QUERY_NODE_LOGIC_PLAN_PARTITION:
       return "LogicPartition";
+    case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
+      return "LogicIndefRowsFunc";
     case QUERY_NODE_LOGIC_SUBPLAN:
       return "LogicSubplan";
     case QUERY_NODE_LOGIC_PLAN:
@@ -230,6 +232,8 @@ const char* nodesNodeName(ENodeType type) {
       return "PhysiSort";
     case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
       return "PhysiHashInterval";
+    case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL:
+      return "PhysiMergeInterval";
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
       return "PhysiStreamInterval";
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
@@ -248,6 +252,8 @@ const char* nodesNodeName(ENodeType type) {
       return "PhysiStreamStateWindow";
     case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
       return "PhysiPartition";
+    case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
+      return "PhysiIndefRowsFunc";
     case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
       return "PhysiDispatch";
     case QUERY_NODE_PHYSICAL_PLAN_INSERT:
@@ -611,7 +617,7 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) {
 
 static const char* jkVnodeModifyLogicPlanModifyType = "ModifyType";
 static const char* jkVnodeModifyLogicPlanMsgType = "MsgType";
-static const char* jkVnodeModifyLogicPlanModifyRows = "ModifyRows";
+static const char* jkVnodeModifyLogicPlanAffectedRows = "AffectedRows";
 
 static int32_t logicVnodeModifyNodeToJson(const void* pObj, SJson* pJson) {
   const SVnodeModifyLogicNode* pNode = (const SVnodeModifyLogicNode*)pObj;
@@ -624,7 +630,7 @@ static int32_t logicVnodeModifyNodeToJson(const void* pObj, SJson* pJson) {
     code = tjsonAddIntegerToObject(pJson, jkVnodeModifyLogicPlanMsgType, pNode->msgType);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = tjsonAddObject(pJson, jkVnodeModifyLogicPlanModifyRows, nodeToJson, pNode->pModifyRows);
+    code = tjsonAddObject(pJson, jkVnodeModifyLogicPlanAffectedRows, nodeToJson, pNode->pAffectedRows);
   }
 
   return code;
@@ -641,7 +647,7 @@ static int32_t jsonToLogicVnodeModifyNode(const SJson* pJson, void* pObj) {
     code = tjsonGetIntValue(pJson, jkVnodeModifyLogicPlanMsgType, &pNode->msgType);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = jsonToNodeObject(pJson, jkVnodeModifyLogicPlanModifyRows, &pNode->pModifyRows);
+    code = jsonToNodeObject(pJson, jkVnodeModifyLogicPlanAffectedRows, &pNode->pAffectedRows);
   }
 
   return code;
@@ -911,6 +917,30 @@ static int32_t jsonToLogicPartitionNode(const SJson* pJson, void* pObj) {
   return code;
 }
 
+static const char* jkIndefRowsFuncLogicPlanVectorFuncs = "VectorFuncs";
+
+static int32_t logicIndefRowsFuncNodeToJson(const void* pObj, SJson* pJson) {
+  const SIndefRowsFuncLogicNode* pNode = (const SIndefRowsFuncLogicNode*)pObj;
+
+  int32_t code = logicPlanNodeToJson(pObj, pJson);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodeListToJson(pJson, jkIndefRowsFuncLogicPlanVectorFuncs, pNode->pVectorFuncs);
+  }
+
+  return code;
+}
+
+static int32_t jsonToLogicIndefRowsFuncNode(const SJson* pJson, void* pObj) {
+  SIndefRowsFuncLogicNode* pNode = (SIndefRowsFuncLogicNode*)pObj;
+
+  int32_t code = jsonToLogicPlanNode(pJson, pObj);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeList(pJson, jkIndefRowsFuncLogicPlanVectorFuncs, &pNode->pVectorFuncs);
+  }
+
+  return code;
+}
+
 static const char* jkSubplanIdQueryId = "QueryId";
 static const char* jkSubplanIdGroupId = "GroupId";
 static const char* jkSubplanIdSubplanId = "SubplanId";
@@ -1249,6 +1279,7 @@ static int32_t jsonToName(const SJson* pJson, void* pObj) {
 static const char* jkScanPhysiPlanScanCols = "ScanCols";
 static const char* jkScanPhysiPlanScanPseudoCols = "ScanPseudoCols";
 static const char* jkScanPhysiPlanTableId = "TableId";
+static const char* jkScanPhysiPlanSTableId = "STableId";
 static const char* jkScanPhysiPlanTableType = "TableType";
 static const char* jkScanPhysiPlanTableName = "TableName";
 
@@ -1265,6 +1296,9 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableId, pNode->uid);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanSTableId, pNode->suid);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableType, pNode->tableType);
   }
@@ -1288,6 +1322,9 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetUBigIntValue(pJson, jkScanPhysiPlanTableId, &pNode->uid);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetUBigIntValue(pJson, jkScanPhysiPlanSTableId, &pNode->suid);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetTinyIntValue(pJson, jkScanPhysiPlanTableType, &pNode->tableType);
   }
@@ -1644,6 +1681,7 @@ static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) {
 }
 
 static const char* jkMergePhysiPlanMergeKeys = "MergeKeys";
+static const char* jkMergePhysiPlanTargets = "Targets";
 static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels";
 static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId";
 
@@ -1654,6 +1692,9 @@ static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = nodeListToJson(pJson, jkMergePhysiPlanMergeKeys, pNode->pMergeKeys);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodeListToJson(pJson, jkMergePhysiPlanTargets, pNode->pTargets);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkMergePhysiPlanNumOfChannels, pNode->numOfChannels);
   }
@@ -1671,6 +1712,9 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = jsonToNodeList(pJson, jkMergePhysiPlanMergeKeys, &pNode->pMergeKeys);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeList(pJson, jkMergePhysiPlanTargets, &pNode->pTargets);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetIntValue(pJson, jkMergePhysiPlanNumOfChannels, &pNode->numOfChannels);
   }
@@ -1979,6 +2023,37 @@ static int32_t jsonToPhysiPartitionNode(const SJson* pJson, void* pObj) {
   return code;
 }
 
+static const char* jkIndefRowsFuncPhysiPlanExprs = "Exprs";
+static const char* jkIndefRowsFuncPhysiPlanVectorFuncs = "VectorFuncs";
+
+static int32_t physiIndefRowsFuncNodeToJson(const void* pObj, SJson* pJson) {
+  const SIndefRowsFuncPhysiNode* pNode = (const SIndefRowsFuncPhysiNode*)pObj;
+
+  int32_t code = physicPlanNodeToJson(pObj, pJson);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodeListToJson(pJson, jkIndefRowsFuncPhysiPlanExprs, pNode->pExprs);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = nodeListToJson(pJson, jkIndefRowsFuncPhysiPlanVectorFuncs, pNode->pVectorFuncs);
+  }
+
+  return code;
+}
+
+static int32_t jsonToPhysiIndefRowsFuncNode(const SJson* pJson, void* pObj) {
+  SIndefRowsFuncPhysiNode* pNode = (SIndefRowsFuncPhysiNode*)pObj;
+
+  int32_t code = jsonToPhysicPlanNode(pJson, pObj);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeList(pJson, jkIndefRowsFuncPhysiPlanExprs, &pNode->pExprs);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeList(pJson, jkIndefRowsFuncPhysiPlanVectorFuncs, &pNode->pVectorFuncs);
+  }
+
+  return code;
+}
+
 static const char* jkDataSinkInputDataBlockDesc = "InputDataBlockDesc";
 
 static int32_t physicDataSinkNodeToJson(const void* pObj, SJson* pJson) {
@@ -2000,6 +2075,7 @@ static const char* jkDeletePhysiPlanTableType = "TableType";
 static const char* jkDeletePhysiPlanTableFName = "TableFName";
 static const char* jkDeletePhysiPlanDeleteTimeRangeStartKey = "DeleteTimeRangeStartKey";
 static const char* jkDeletePhysiPlanDeleteTimeRangeEndKey = "DeleteTimeRangeEndKey";
+static const char* jkDeletePhysiPlanAffectedRows = "AffectedRows";
 
 static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) {
   const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj;
@@ -2020,6 +2096,9 @@ static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, pNode->deleteTimeRange.ekey);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddObject(pJson, jkDeletePhysiPlanAffectedRows, nodeToJson, pNode->pAffectedRows);
+  }
 
   return code;
 }
@@ -2043,6 +2122,9 @@ static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetBigIntValue(pJson, jkDeletePhysiPlanDeleteTimeRangeEndKey, &pNode->deleteTimeRange.ekey);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = jsonToNodeObject(pJson, jkDeletePhysiPlanAffectedRows, &pNode->pAffectedRows);
+  }
 
   return code;
 }
@@ -3777,6 +3859,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       return logicSortNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN_PARTITION:
       return logicPartitionNodeToJson(pObj, pJson);
+    case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
+      return logicIndefRowsFuncNodeToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_SUBPLAN:
       return logicSubplanToJson(pObj, pJson);
     case QUERY_NODE_LOGIC_PLAN:
@@ -3802,6 +3886,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
     case QUERY_NODE_PHYSICAL_PLAN_SORT:
       return physiSortNodeToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
@@ -3816,6 +3901,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       return physiStateWindowNodeToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
       return physiPartitionNodeToJson(pObj, pJson);
+    case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
+      return physiIndefRowsFuncNodeToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
       return physiDispatchNodeToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_PLAN_INSERT:
@@ -3905,6 +3992,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
       return jsonToLogicSortNode(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN_PARTITION:
       return jsonToLogicPartitionNode(pJson, pObj);
+    case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
+      return jsonToLogicIndefRowsFuncNode(pJson, pObj);
     case QUERY_NODE_LOGIC_SUBPLAN:
       return jsonToLogicSubplan(pJson, pObj);
     case QUERY_NODE_LOGIC_PLAN:
@@ -3930,6 +4019,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
     case QUERY_NODE_PHYSICAL_PLAN_SORT:
       return jsonToPhysiSortNode(pJson, pObj);
     case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL:
@@ -3944,6 +4034,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
       return jsonToPhysiStateWindowNode(pJson, pObj);
     case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
       return jsonToPhysiPartitionNode(pJson, pObj);
+    case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
+      return jsonToPhysiIndefRowsFuncNode(pJson, pObj);
     case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
       return jsonToPhysiDispatchNode(pJson, pObj);
     case QUERY_NODE_PHYSICAL_PLAN_DELETE:

source/libs/nodes/src/nodesTraverseFuncs.c

@@ -350,6 +350,7 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa
     case SQL_CLAUSE_GROUP_BY:
       nodesWalkExpr(pSelect->pHaving, walker, pContext);
     case SQL_CLAUSE_HAVING:
+    case SQL_CLAUSE_SELECT:
     case SQL_CLAUSE_DISTINCT:
       nodesWalkExprs(pSelect->pOrderByList, walker, pContext);
     case SQL_CLAUSE_ORDER_BY:
@@ -382,6 +383,7 @@ void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewrit
     case SQL_CLAUSE_GROUP_BY:
       nodesRewriteExpr(&(pSelect->pHaving), rewriter, pContext);
     case SQL_CLAUSE_HAVING:
+    case SQL_CLAUSE_SELECT:
     case SQL_CLAUSE_DISTINCT:
       nodesRewriteExprs(pSelect->pOrderByList, rewriter, pContext);
     case SQL_CLAUSE_ORDER_BY:

source/libs/nodes/src/nodesUtilFuncs.c

@@ -232,6 +232,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SSortLogicNode));
     case QUERY_NODE_LOGIC_PLAN_PARTITION:
       return makeNode(type, sizeof(SPartitionLogicNode));
+    case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
+      return makeNode(type, sizeof(SIndefRowsFuncLogicNode));
     case QUERY_NODE_LOGIC_SUBPLAN:
       return makeNode(type, sizeof(SLogicSubplan));
     case QUERY_NODE_LOGIC_PLAN:
@@ -260,8 +262,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SSortPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
       return makeNode(type, sizeof(SIntervalPhysiNode));
-    case QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL:
+    case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL:
-      return makeNode(type, sizeof(SSortMergeIntervalPhysiNode));
+      return makeNode(type, sizeof(SMergeIntervalPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL:
       return makeNode(type, sizeof(SStreamIntervalPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL:
@@ -280,6 +282,8 @@ SNodeptr nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SStreamStateWinodwPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
       return makeNode(type, sizeof(SPartitionPhysiNode));
+    case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC:
+      return makeNode(type, sizeof(SIndefRowsFuncPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
       return makeNode(type, sizeof(SDataDispatcherNode));
     case QUERY_NODE_PHYSICAL_PLAN_INSERT:

source/libs/parser/src/parAstParser.c

@@ -258,6 +258,17 @@ static int32_t collectMetaKeyFromExplain(SCollectMetaKeyCxt* pCxt, SExplainStmt*
   return collectMetaKeyFromQuery(pCxt, pStmt->pQuery);
 }
 
+static int32_t collectMetaKeyFromDescribe(SCollectMetaKeyCxt* pCxt, SDescribeStmt* pStmt) {
+  SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId};
+  strcpy(name.dbname, pStmt->dbName);
+  strcpy(name.tname, pStmt->tableName);
+  int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
+  }
+  return code;
+}
+
 static int32_t collectMetaKeyFromCreateStream(SCollectMetaKeyCxt* pCxt, SCreateStreamStmt* pStmt) {
   return collectMetaKeyFromQuery(pCxt, pStmt->pQuery);
 }
@@ -381,6 +392,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
       return collectMetaKeyFromCreateTopic(pCxt, (SCreateTopicStmt*)pStmt);
     case QUERY_NODE_EXPLAIN_STMT:
       return collectMetaKeyFromExplain(pCxt, (SExplainStmt*)pStmt);
+    case QUERY_NODE_DESCRIBE_STMT:
+      return collectMetaKeyFromDescribe(pCxt, (SDescribeStmt*)pStmt);
     case QUERY_NODE_CREATE_STREAM_STMT:
       return collectMetaKeyFromCreateStream(pCxt, (SCreateStreamStmt*)pStmt);
     case QUERY_NODE_SHOW_DNODES_STMT:

source/libs/parser/src/parInsert.c

@@ -1006,7 +1006,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
     }
 
     SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]];
-    char*    tmpTokenBuf = taosMemoryCalloc(1, sToken.n);  // this can be optimize with parse column
+    char*    tmpTokenBuf = taosMemoryCalloc(1, sToken.n);  //todo this can be optimize with parse column
     code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg);
     if (code != TSDB_CODE_SUCCESS) {
       taosMemoryFree(tmpTokenBuf);
@@ -1018,7 +1018,11 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint
         taosMemoryFree(tmpTokenBuf);
         goto end;
       }
-      code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg);
+      if(isNullStr(&sToken)) {
+        code = tTagNew(pTagVals, 1, true, &pTag);
+      } else {
+        code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg);
+      }
       taosMemoryFree(tmpTokenBuf);
       if (code != TSDB_CODE_SUCCESS) {
         goto end;

source/libs/parser/src/parTokenizer.c

@@ -64,6 +64,7 @@ static SKeyword keywordTable[] = {
     {"CONSUMER",      TK_CONSUMER},
     {"COUNT",         TK_COUNT},
     {"CREATE",        TK_CREATE},
+    {"CONTAINS",      TK_CONTAINS},
     {"DATABASE",      TK_DATABASE},
     {"DATABASES",     TK_DATABASES},
     {"DAYS",          TK_DAYS},

source/libs/parser/src/parTranslater.c

@@ -847,8 +847,12 @@ static EDealRes translateJsonOperator(STranslateContext* pCxt, SOperatorNode* pO
   if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
     return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
   }
-  pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
+  if(pOp->opType == OP_TYPE_JSON_GET_VALUE){
-  pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes;
+    pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
+  }else if(pOp->opType == OP_TYPE_JSON_CONTAINS){
+    pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
+  }
+  pOp->node.resType.bytes = tDataTypes[pOp->node.resType.type].bytes;
   return DEAL_RES_CONTINUE;
 }
 
@@ -2903,7 +2907,6 @@ static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt,
     }
   }
 
-  taosArrayDestroy(dbCfg.pRetensions);
   return code;
 }
 

source/libs/parser/test/mockCatalog.cpp

@@ -207,6 +207,13 @@ int32_t __catalogGetUdfInfo(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps
   return g_mockCatalogService->catalogGetUdfInfo(funcName, pInfo);
 }
 
+int32_t __catalogRefreshGetTableMeta(SCatalog* pCatalog, void* pTransporter, const SEpSet* pMgmtEps,
+                                     const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) {
+  return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta);
+}
+
+int32_t __catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { return 0; }
+
 void initMetaDataEnv() {
   g_mockCatalogService.reset(new MockCatalogService());
 
@@ -221,6 +228,8 @@ void initMetaDataEnv() {
   stub.set(catalogGetDBCfg, __catalogGetDBCfg);
   stub.set(catalogChkAuth, __catalogChkAuth);
   stub.set(catalogGetUdfInfo, __catalogGetUdfInfo);
+  stub.set(catalogRefreshGetTableMeta, __catalogRefreshGetTableMeta);
+  stub.set(catalogRemoveTableMeta, __catalogRemoveTableMeta);
   // {
   //   AddrAny any("libcatalog.so");
   //   std::map<std::string,void*> result;

source/libs/parser/test/parInitialDTest.cpp

@@ -21,7 +21,7 @@ namespace ParserTest {
 
 class ParserInitialDTest : public ParserDdlTest {};
 
-// DELETE FROM tb_name [WHERE condition]
+// DELETE FROM table_name [WHERE condition]
 TEST_F(ParserInitialDTest, delete) {
   useDb("root", "test");
 
@@ -40,7 +40,15 @@ TEST_F(ParserInitialDTest, deleteSemanticCheck) {
   run("DELETE FROM t1 WHERE c1 > 10", TSDB_CODE_PAR_INVALID_DELETE_WHERE, PARSER_STAGE_TRANSLATE);
 }
 
-// todo desc
+// DESC table_name
+TEST_F(ParserInitialDTest, describe) {
+  useDb("root", "test");
+
+  run("DESC t1");
+
+  run("DESCRIBE st1");
+}
+
 // todo describe
 // todo DROP account
 
@@ -51,7 +59,7 @@ TEST_F(ParserInitialDTest, dropBnode) {
 }
 
 // DROP CONSUMER GROUP [ IF EXISTS ] cgroup_name ON topic_name
-TEST_F(ParserInitialDTest, dropCGroup) {
+TEST_F(ParserInitialDTest, dropConsumerGroup) {
   useDb("root", "test");
 
   SMDropCgroupReq expect = {0};

source/libs/parser/test/parTestUtil.h

@@ -50,11 +50,13 @@ class ParserDdlTest : public ParserTestBase {
 
   virtual void checkDdl(const SQuery* pQuery, ParserStage stage) {
     ASSERT_NE(pQuery, nullptr);
-    ASSERT_EQ(pQuery->haveResultSet, false);
     ASSERT_NE(pQuery->pRoot, nullptr);
-    ASSERT_EQ(pQuery->numOfResCols, 0);
+    if (QUERY_EXEC_MODE_RPC == pQuery->execMode) {
-    ASSERT_EQ(pQuery->pResSchema, nullptr);
+      ASSERT_EQ(pQuery->haveResultSet, false);
-    ASSERT_EQ(pQuery->precision, 0);
+      ASSERT_EQ(pQuery->numOfResCols, 0);
+      ASSERT_EQ(pQuery->pResSchema, nullptr);
+      ASSERT_EQ(pQuery->precision, 0);
+    }
     if (nullptr != checkDdl_) {
       checkDdl_(pQuery, stage);
     }

source/libs/planner/src/planLogicCreater.c

@@ -219,9 +219,9 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  // TSWAP(pScan->pMeta, pRealTable->pMeta);
   TSWAP(pScan->pVgroupList, pRealTable->pVgroupList);
   pScan->tableId = pRealTable->pMeta->uid;
+  pScan->stableId = pRealTable->pMeta->suid;
   pScan->tableType = pRealTable->pMeta->tableType;
   pScan->scanSeq[0] = hasRepeatScanFuncs ? 2 : 1;
   pScan->scanSeq[1] = 0;
@@ -456,6 +456,37 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
   return code;
 }
 
+static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
+  // top/bottom are both an aggregate function and a indefinite rows function
+  if (!pSelect->hasIndefiniteRowsFunc || pSelect->hasAggFuncs || NULL != pSelect->pWindow) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SIndefRowsFuncLogicNode* pIdfRowsFunc =
+      (SIndefRowsFuncLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC);
+  if (NULL == pIdfRowsFunc) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pVectorFuncs);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = rewriteExprForSelect(pIdfRowsFunc->pVectorFuncs, pSelect, SQL_CLAUSE_SELECT);
+  }
+
+  // set the output
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createColumnByRewriteExps(pIdfRowsFunc->pVectorFuncs, &pIdfRowsFunc->node.pTargets);
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    *pLogicNode = (SLogicNode*)pIdfRowsFunc;
+  } else {
+    nodesDestroyNode(pIdfRowsFunc);
+  }
+
+  return code;
+}
+
 static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow,
                                              SLogicNode** pLogicNode) {
   int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_WINDOW, fmIsWindowClauseFunc, &pWindow->pFuncs);
@@ -772,6 +803,9 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele
   if (TSDB_CODE_SUCCESS == code) {
     code = createSelectRootLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = createSelectRootLogicNode(pCxt, pSelect, createIndefRowsFuncLogicNode, &pRoot);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = createSelectRootLogicNode(pCxt, pSelect, createDistinctLogicNode, &pRoot);
   }
@@ -1057,8 +1091,8 @@ static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDelet
   snprintf(pModify->tableFName, sizeof(pModify->tableFName), "%d.%s.%s", pCxt->pPlanCxt->acctId,
            pRealTable->table.dbName, pRealTable->table.tableName);
   pModify->deleteTimeRange = pDelete->timeRange;
-  pModify->pModifyRows = nodesCloneNode(pDelete->pCountFunc);
+  pModify->pAffectedRows = nodesCloneNode(pDelete->pCountFunc);
-  if (NULL == pModify->pModifyRows) {
+  if (NULL == pModify->pAffectedRows) {
     nodesDestroyNode(pModify);
     return TSDB_CODE_OUT_OF_MEMORY;
   }

source/libs/planner/src/planPhysiCreater.c

@@ -425,6 +425,7 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS
 
   if (TSDB_CODE_SUCCESS == code) {
     pScanPhysiNode->uid = pScanLogicNode->tableId;
+    pScanPhysiNode->suid = pScanLogicNode->stableId;
     pScanPhysiNode->tableType = pScanLogicNode->tableType;
     memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName));
     if (NULL != pScanLogicNode->pTagCond) {
@@ -790,6 +791,43 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
   return code;
 }
 
+static int32_t createIndefRowsFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
+                                            SIndefRowsFuncLogicNode* pFuncLogicNode, SPhysiNode** pPhyNode) {
+  SIndefRowsFuncPhysiNode* pIdfRowsFunc = (SIndefRowsFuncPhysiNode*)makePhysiNode(
+      pCxt, (SLogicNode*)pFuncLogicNode, QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC);
+  if (NULL == pIdfRowsFunc) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  SNodeList* pPrecalcExprs = NULL;
+  SNodeList* pVectorFuncs = NULL;
+  int32_t    code = rewritePrecalcExprs(pCxt, pFuncLogicNode->pVectorFuncs, &pPrecalcExprs, &pVectorFuncs);
+
+  SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc);
+  // push down expression to pOutputDataBlockDesc of child node
+  if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) {
+    code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pIdfRowsFunc->pExprs);
+    if (TSDB_CODE_SUCCESS == code) {
+      code = pushdownDataBlockSlots(pCxt, pIdfRowsFunc->pExprs, pChildTupe);
+    }
+  }
+
+  if (TSDB_CODE_SUCCESS == code && NULL != pVectorFuncs) {
+    code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pVectorFuncs, &pIdfRowsFunc->pVectorFuncs);
+    if (TSDB_CODE_SUCCESS == code) {
+      code = addDataBlockSlots(pCxt, pIdfRowsFunc->pVectorFuncs, pIdfRowsFunc->node.pOutputDataBlockDesc);
+    }
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    *pPhyNode = (SPhysiNode*)pIdfRowsFunc;
+  } else {
+    nodesDestroyNode(pIdfRowsFunc);
+  }
+
+  return code;
+}
+
 static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
                                       SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) {
   SProjectPhysiNode* pProject =
@@ -923,8 +961,8 @@ static ENodeType getIntervalOperatorType(EIntervalAlgorithm intervalAlgo) {
   switch (intervalAlgo) {
     case INTERVAL_ALGO_HASH:
       return QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL;
-    case INTERVAL_ALGO_SORT_MERGE:
+    case INTERVAL_ALGO_MERGE:
-      return QUERY_NODE_PHYSICAL_PLAN_SORT_MERGE_INTERVAL;
+      return QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL;
     case INTERVAL_ALGO_STREAM_FINAL:
       return QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL;
     case INTERVAL_ALGO_STREAM_SEMI:
@@ -1155,6 +1193,8 @@ static int32_t createExchangePhysiNodeByMerge(SMergePhysiNode* pMerge) {
     nodesDestroyNode(pExchange);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  SNode* pSlot = NULL;
+  FOREACH(pSlot, pExchange->node.pOutputDataBlockDesc->pSlots) { ((SSlotDescNode*)pSlot)->output = true; }
   return nodesListMakeStrictAppend(&pMerge->node.pChildren, pExchange);
 }
 
@@ -1168,12 +1208,14 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM
   pMerge->numOfChannels = pMergeLogicNode->numOfChannels;
   pMerge->srcGroupId = pMergeLogicNode->srcGroupId;
 
-  int32_t code = TSDB_CODE_SUCCESS;
+  int32_t code = addDataBlockSlots(pCxt, pMergeLogicNode->pInputs, pMerge->node.pOutputDataBlockDesc);
 
-  for (int32_t i = 0; i < pMerge->numOfChannels; ++i) {
+  if (TSDB_CODE_SUCCESS == code) {
-    code = createExchangePhysiNodeByMerge(pMerge);
+    for (int32_t i = 0; i < pMerge->numOfChannels; ++i) {
-    if (TSDB_CODE_SUCCESS != code) {
+      code = createExchangePhysiNodeByMerge(pMerge);
-      break;
+      if (TSDB_CODE_SUCCESS != code) {
+        break;
+      }
     }
   }
 
@@ -1182,6 +1224,14 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM
                          &pMerge->pMergeKeys);
   }
 
+  if (TSDB_CODE_SUCCESS == code) {
+    code = setListSlotId(pCxt, pMerge->node.pOutputDataBlockDesc->dataBlockId, -1, pMergeLogicNode->node.pTargets,
+                         &pMerge->pTargets);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = addDataBlockSlots(pCxt, pMerge->pTargets, pMerge->node.pOutputDataBlockDesc);
+  }
+
   if (TSDB_CODE_SUCCESS == code) {
     *pPhyNode = (SPhysiNode*)pMerge;
   } else {
@@ -1212,6 +1262,8 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode
       return createPartitionPhysiNode(pCxt, pChildren, (SPartitionLogicNode*)pLogicNode, pPhyNode);
     case QUERY_NODE_LOGIC_PLAN_FILL:
       return createFillPhysiNode(pCxt, pChildren, (SFillLogicNode*)pLogicNode, pPhyNode);
+    case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC:
+      return createIndefRowsFuncPhysiNode(pCxt, pChildren, (SIndefRowsFuncLogicNode*)pLogicNode, pPhyNode);
     case QUERY_NODE_LOGIC_PLAN_MERGE:
       return createMergePhysiNode(pCxt, (SMergeLogicNode*)pLogicNode, pPhyNode);
     default:
@@ -1319,13 +1371,21 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode*
   strcpy(pDeleter->tableFName, pModify->tableFName);
   pDeleter->deleteTimeRange = pModify->deleteTimeRange;
 
-  pDeleter->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc);
+  int32_t code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pAffectedRows,
-  if (NULL == pDeleter->sink.pInputDataBlockDesc) {
+                               &pDeleter->pAffectedRows);
-    nodesDestroyNode(pDeleter);
+  if (TSDB_CODE_SUCCESS == code) {
-    return TSDB_CODE_OUT_OF_MEMORY;
+    pDeleter->sink.pInputDataBlockDesc = nodesCloneNode(pRoot->pOutputDataBlockDesc);
+    if (NULL == pDeleter->sink.pInputDataBlockDesc) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
+
+  if (TSDB_CODE_SUCCESS == code) {
+    *pSink = (SDataSinkNode*)pDeleter;
+  } else {
+    nodesDestroyNode(pDeleter);
   }
 
-  *pSink = (SDataSinkNode*)pDeleter;
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/planner/src/planSpliter.c

@@ -80,30 +80,36 @@ static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pChild, SE
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t splCreateExchangeNodeForSubplan(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode,
+static int32_t splReplaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew) {
-                                               ESubplanType subplanType) {
+  if (NULL == pOld->pParent) {
-  SExchangeLogicNode* pExchange = NULL;
+    pSubplan->pNode = (SLogicNode*)pNew;
-  if (TSDB_CODE_SUCCESS != splCreateExchangeNode(pCxt, pSplitNode, &pExchange)) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-
-  pSubplan->subplanType = subplanType;
-
-  if (NULL == pSplitNode->pParent) {
-    pSubplan->pNode = (SLogicNode*)pExchange;
     return TSDB_CODE_SUCCESS;
   }
 
   SNode* pNode;
-  FOREACH(pNode, pSplitNode->pParent->pChildren) {
+  FOREACH(pNode, pOld->pParent->pChildren) {
-    if (nodesEqualNode(pNode, pSplitNode)) {
+    if (nodesEqualNode(pNode, pOld)) {
-      REPLACE_NODE(pExchange);
+      REPLACE_NODE(pNew);
-      pExchange->node.pParent = pSplitNode->pParent;
+      pNew->pParent = pOld->pParent;
       return TSDB_CODE_SUCCESS;
     }
   }
-  nodesDestroyNode(pExchange);
+  return TSDB_CODE_PLAN_INTERNAL_ERROR;
-  return TSDB_CODE_FAILED;
+}
+
+static int32_t splCreateExchangeNodeForSubplan(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode,
+                                               ESubplanType subplanType) {
+  SExchangeLogicNode* pExchange = NULL;
+  int32_t             code = splCreateExchangeNode(pCxt, pSplitNode, &pExchange);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pExchange);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    pSubplan->subplanType = subplanType;
+  } else {
+    nodesDestroyNode(pExchange);
+  }
+  return code;
 }
 
 static bool splMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan, int32_t flag, FSplFindSplitNode func, void* pInfo) {
@@ -295,24 +301,34 @@ static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogic
   return code;
 }
 
-static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicNode* pParent, SNodeList* pMergeKeys,
+static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode,
-                                     SLogicNode* pPartChild) {
+                                     SNodeList* pMergeKeys, SLogicNode* pPartChild) {
   SMergeLogicNode* pMerge = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE);
   if (NULL == pMerge) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   pMerge->numOfChannels = ((SScanLogicNode*)nodesListGetNode(pPartChild->pChildren, 0))->pVgroupList->numOfVgroups;
   pMerge->srcGroupId = pCxt->groupId;
-  pMerge->node.pParent = pParent;
   pMerge->node.precision = pPartChild->precision;
   pMerge->pMergeKeys = pMergeKeys;
-  pMerge->node.pTargets = nodesCloneList(pPartChild->pTargets);
-  if (NULL == pMerge->node.pTargets) {
-    nodesDestroyNode(pMerge);
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
 
-  return nodesListMakeAppend(&pParent->pChildren, pMerge);
+  int32_t code = TSDB_CODE_SUCCESS;
+  pMerge->pInputs = nodesCloneList(pPartChild->pTargets);
+  pMerge->node.pTargets = nodesCloneList(pSplitNode->pTargets);
+  if (NULL == pMerge->node.pTargets || NULL == pMerge->pInputs) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    if (NULL == pSubplan) {
+      code = nodesListMakeAppend(&pSplitNode->pChildren, pMerge);
+    } else {
+      code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge);
+    }
+  }
+  if (TSDB_CODE_SUCCESS != code) {
+    nodesDestroyNode(pMerge);
+  }
+  return code;
 }
 
 static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent, SLogicNode* pPartChild) {
@@ -329,8 +345,15 @@ static int32_t stbSplSplitWindowNodeForBatch(SSplitContext* pCxt, SStableSplitIn
   int32_t     code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow);
   if (TSDB_CODE_SUCCESS == code) {
     ((SWindowLogicNode*)pPartWindow)->intervalAlgo = INTERVAL_ALGO_HASH;
-    ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_SORT_MERGE;
+    ((SWindowLogicNode*)pInfo->pSplitNode)->intervalAlgo = INTERVAL_ALGO_MERGE;
-    code = stbSplCreateExchangeNode(pCxt, pInfo->pSplitNode, pPartWindow);
+    SNodeList* pMergeKeys = NULL;
+    code = nodesListMakeStrictAppend(&pMergeKeys, nodesCloneNode(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk));
+    if (TSDB_CODE_SUCCESS == code) {
+      code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow);
+    }
+    if (TSDB_CODE_SUCCESS != code) {
+      nodesDestroyList(pMergeKeys);
+    }
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
@@ -424,37 +447,99 @@ static int32_t stbSplSplitAggNode(SSplitContext* pCxt, SStableSplitInfo* pInfo)
   return code;
 }
 
-static int32_t stbSplCreatePartSortNode(SSortLogicNode* pMergeSort, SLogicNode** pOutput) {
+static SNode* stbSplCreateColumnNode(SExprNode* pExpr) {
-  SNodeList* pSortKeys = pMergeSort->pSortKeys;
+  SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
-  pMergeSort->pSortKeys = NULL;
+  if (NULL == pCol) {
-  SNodeList* pTargets = pMergeSort->node.pTargets;
+    return NULL;
-  pMergeSort->node.pTargets = NULL;
+  }
-  SNodeList* pChildren = pMergeSort->node.pChildren;
+  if (QUERY_NODE_COLUMN == nodeType(pExpr)) {
-  pMergeSort->node.pChildren = NULL;
+    strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias);
+  }
+  strcpy(pCol->colName, pExpr->aliasName);
+  strcpy(pCol->node.aliasName, pExpr->aliasName);
+  pCol->node.resType = pExpr->resType;
+  return (SNode*)pCol;
+}
+
+static SNode* stbSplCreateOrderByExpr(SOrderByExprNode* pSortKey, SNode* pCol) {
+  SOrderByExprNode* pOutput = nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
+  if (NULL == pOutput) {
+    return NULL;
+  }
+  pOutput->pExpr = nodesCloneNode(pCol);
+  if (NULL == pOutput->pExpr) {
+    nodesDestroyNode(pOutput);
+    return NULL;
+  }
+  pOutput->order = pSortKey->order;
+  pOutput->nullOrder = pSortKey->nullOrder;
+  return (SNode*)pOutput;
+}
+
+static int32_t stbSplCreateMergeKeys(SNodeList* pSortKeys, SNodeList* pTargets, SNodeList** pOutput) {
+  int32_t    code = TSDB_CODE_SUCCESS;
+  SNodeList* pMergeKeys = NULL;
+  SNode*     pNode = NULL;
+  FOREACH(pNode, pSortKeys) {
+    SOrderByExprNode* pSortKey = (SOrderByExprNode*)pNode;
+    SNode*            pTarget = NULL;
+    bool              found = false;
+    FOREACH(pTarget, pTargets) {
+      if (0 == strcmp(((SExprNode*)pSortKey->pExpr)->aliasName, ((SColumnNode*)pTarget)->colName)) {
+        code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pTarget));
+        if (TSDB_CODE_SUCCESS != code) {
+          break;
+        }
+        found = true;
+      }
+    }
+    if (TSDB_CODE_SUCCESS == code && !found) {
+      SNode* pCol = stbSplCreateColumnNode((SExprNode*)pSortKey->pExpr);
+      code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pCol));
+      if (TSDB_CODE_SUCCESS == code) {
+        code = nodesListStrictAppend(pTargets, pCol);
+      } else {
+        nodesDestroyNode(pCol);
+      }
+    }
+    if (TSDB_CODE_SUCCESS != code) {
+      break;
+    }
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    *pOutput = pMergeKeys;
+  } else {
+    nodesDestroyList(pMergeKeys);
+  }
+  return code;
+}
+
+static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOutputPartSort,
+                                        SNodeList** pOutputMergeKeys) {
+  SNodeList* pSortKeys = pSort->pSortKeys;
+  pSort->pSortKeys = NULL;
+  SNodeList* pChildren = pSort->node.pChildren;
+  pSort->node.pChildren = NULL;
 
   int32_t         code = TSDB_CODE_SUCCESS;
-  SSortLogicNode* pPartSort = nodesCloneNode(pMergeSort);
+  SSortLogicNode* pPartSort = nodesCloneNode(pSort);
   if (NULL == pPartSort) {
     code = TSDB_CODE_OUT_OF_MEMORY;
   }
 
-  pMergeSort->node.pTargets = pTargets;
+  SNodeList* pMergeKeys = NULL;
-  pPartSort->node.pChildren = pChildren;
   if (TSDB_CODE_SUCCESS == code) {
+    pPartSort->node.pChildren = pChildren;
     pPartSort->pSortKeys = pSortKeys;
-    code = createColumnByRewriteExps(pPartSort->pSortKeys, &pPartSort->node.pTargets);
+    code = stbSplCreateMergeKeys(pPartSort->pSortKeys, pPartSort->node.pTargets, &pMergeKeys);
-  }
-  if (TSDB_CODE_SUCCESS == code) {
-    pMergeSort->pSortKeys = nodesCloneList(pPartSort->node.pTargets);
-    if (NULL == pMergeSort->pSortKeys) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-    }
   }
 
   if (TSDB_CODE_SUCCESS == code) {
-    *pOutput = (SLogicNode*)pPartSort;
+    *pOutputPartSort = (SLogicNode*)pPartSort;
+    *pOutputMergeKeys = pMergeKeys;
   } else {
     nodesDestroyNode(pPartSort);
+    nodesDestroyList(pMergeKeys);
   }
 
   return code;
@@ -462,17 +547,10 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pMergeSort, SLogicNode**
 
 static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
   SLogicNode* pPartSort = NULL;
-  int32_t     code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort);
+  SNodeList*  pMergeKeys = NULL;
+  int32_t     code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort, &pMergeKeys);
   if (TSDB_CODE_SUCCESS == code) {
-    SNodeList* pMergeKeys = nodesCloneList(((SSortLogicNode*)pInfo->pSplitNode)->pSortKeys);
+    code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort);
-    if (NULL != pMergeKeys) {
-      code = stbSplCreateMergeNode(pCxt, pInfo->pSplitNode, pMergeKeys, pPartSort);
-      if (TSDB_CODE_SUCCESS != code) {
-        nodesDestroyList(pMergeKeys);
-      }
-    } else {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-    }
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,

source/libs/planner/test/planIntervalTest.cpp

@@ -50,6 +50,8 @@ TEST_F(PlanIntervalTest, selectFunc) {
   run("SELECT MAX(c1), MIN(c1) FROM t1 INTERVAL(10s)");
   // select function along with the columns of select row, and with INTERVAL clause
   run("SELECT MAX(c1), c2 FROM t1 INTERVAL(10s)");
+
+  run("SELECT TOP(c1, 1) FROM t1 INTERVAL(10s) ORDER BY c1");
 }
 
 TEST_F(PlanIntervalTest, stable) {

source/libs/planner/test/planOrderByTest.cpp

@@ -46,4 +46,7 @@ TEST_F(PlanOrderByTest, stable) {
 
   // ORDER BY key is in the projection list
   run("SELECT c1 FROM st1 ORDER BY c1");
+
+  // ORDER BY key is not in the projection list
+  run("SELECT c2 FROM st1 ORDER BY c1");
 }

source/libs/planner/test/planProjectTest.cpp

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "planTestUtil.h"
+
+using namespace std;
+
+class PlanProjectTest : public PlannerTestBase {};
+
+TEST_F(PlanProjectTest, basic) {
+  useDb("root", "test");
+
+  run("SELECT CEIL(c1) FROM t1");
+}
+
+TEST_F(PlanProjectTest, indefiniteRowsFunc) {
+  useDb("root", "test");
+
+  run("SELECT MAVG(c1, 10) FROM t1");
+
+  run("SELECT MAVG(CEIL(c1), 20) + 2 FROM t1");
+}

source/libs/planner/test/planTestUtil.cpp

@@ -81,6 +81,8 @@ int32_t getLogLevel() { return g_logLevel; }
 
 class PlannerTestBaseImpl {
  public:
+  PlannerTestBaseImpl() : sqlNo_(0) {}
+
   void useDb(const string& acctId, const string& db) {
     caseEnv_.acctId_ = acctId;
     caseEnv_.db_ = db;
@@ -88,6 +90,7 @@ class PlannerTestBaseImpl {
   }
 
   void run(const string& sql) {
+    ++sqlNo_;
     if (caseEnv_.nsql_ > 0) {
       --(caseEnv_.nsql_);
       return;
@@ -187,6 +190,8 @@ class PlannerTestBaseImpl {
     string  acctId_;
     string  db_;
     int32_t nsql_;
+
+    caseEnv() : nsql_(0) {}
   };
 
   struct stmtEnv {
@@ -194,6 +199,7 @@ class PlannerTestBaseImpl {
     array<char, 1024> msgBuf_;
     SQuery*           pQuery_;
 
+    stmtEnv() : pQuery_(nullptr) {}
     ~stmtEnv() { qDestroyQuery(pQuery_); }
   };
 
@@ -229,7 +235,7 @@ class PlannerTestBaseImpl {
       return;
     }
 
-    cout << "==========================================sql : [" << stmtEnv_.sql_ << "]" << endl;
+    cout << "========================================== " << sqlNo_ << " sql : [" << stmtEnv_.sql_ << "]" << endl;
 
     if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) {
       if (res_.prepareAst_.empty()) {
@@ -382,6 +388,7 @@ class PlannerTestBaseImpl {
   caseEnv caseEnv_;
   stmtEnv stmtEnv_;
   stmtRes res_;
+  int32_t sqlNo_;
 };
 
 PlannerTestBase::PlannerTestBase() : impl_(new PlannerTestBaseImpl()) {}

source/libs/qcom/src/querymsg.c

@@ -37,6 +37,8 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) {
   pOut->dbVgroup->vgVersion = usedbRsp->vgVersion;
   pOut->dbVgroup->hashMethod = usedbRsp->hashMethod;
 
+  qDebug("Got %d vgroup for db %s", usedbRsp->vgNum, usedbRsp->db);
+
   if (usedbRsp->vgNum <= 0) {
     return TSDB_CODE_SUCCESS;
   }
@@ -50,6 +52,8 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) {
   for (int32_t i = 0; i < usedbRsp->vgNum; ++i) {
     SVgroupInfo *pVgInfo = taosArrayGet(usedbRsp->pVgroupInfos, i);
     pOut->dbVgroup->numOfTable += pVgInfo->numOfTable;
+    qDebug("the %dth vgroup, id %d, epNum %d, current %s port %d", i, pVgInfo->vgId, pVgInfo->epSet.numOfEps,
+      pVgInfo->epSet.eps[pVgInfo->epSet.inUse].fqdn, pVgInfo->epSet.eps[pVgInfo->epSet.inUse].port);
     if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }

source/libs/scalar/src/filter.c

@@ -168,7 +168,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val
   compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
   setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, 
   compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8,
-  compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey
+  compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch
 };
 
 int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {

source/libs/scalar/src/sclvector.c

@@ -88,7 +88,7 @@ void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int
   }
 }
 
-void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){
+void convertNcharToDouble(const void *inData, void *outData){
   char *tmp = taosMemoryMalloc(varDataTLen(inData));
   int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp);
   if (len < 0) {
@@ -97,13 +97,24 @@ void convertStringToDouble(const void *inData, void *outData, int8_t inType, int
 
   tmp[len] = 0;
 
-  ASSERT(outType == TSDB_DATA_TYPE_DOUBLE);
   double value = taosStr2Double(tmp, NULL);
 
   *((double *)outData) = value;
   taosMemoryFreeClear(tmp);
 }
 
+void convertBinaryToDouble(const void *inData, void *outData){
+  char *tmp = taosMemoryCalloc(1, varDataTLen(inData));
+  if(tmp == NULL){
+    *((double *)outData) = 0.;
+    return;
+  }
+  memcpy(tmp, varDataVal(inData), varDataLen(inData));
+  double ret = taosStr2Double(tmp, NULL);
+  taosMemoryFree(tmp);
+  *((double *)outData) = ret;
+}
+
 typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index);
 
 int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) {
@@ -147,7 +158,7 @@ int64_t getVectorBigintValue_JSON(void *src, int32_t index){
   if (*data == TSDB_DATA_TYPE_NULL){
     return 0;
   } else if(*data == TSDB_DATA_TYPE_NCHAR) {   // json inner type can not be BINARY
-    convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
+    convertNcharToDouble(data+CHAR_BYTES, &out);
   } else {
     convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
   }
@@ -445,14 +456,30 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){
   if (*data == TSDB_DATA_TYPE_NULL){
     return out;
   } else if(*data == TSDB_DATA_TYPE_NCHAR) {   // json inner type can not be BINARY
-    convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
+    convertNcharToDouble(data+CHAR_BYTES, &out);
   } else {
     convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE);
   }
   return out;
 }
 
-bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){
+void* ncharTobinary(void *buf){            // todo need to remove , if tobinary is nchar
+  int32_t inputLen = varDataLen(buf);
+
+  void* t = taosMemoryCalloc(1, inputLen);
+  int32_t len  = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t));
+  if (len < 0) {
+    sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
+             (char*)varDataVal(buf));
+    taosMemoryFree(t);
+    return NULL;
+  }
+  varDataSetLen(t, len);
+  return t;
+}
+
+bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData,
+                      void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft, bool *freeRight){
   if(optr == OP_TYPE_JSON_CONTAINS) {
     return true;
   }
@@ -489,21 +516,41 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t
 
   *fp = filterGetCompFunc(type, optr);
 
-  if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
+  if(IS_NUMERIC_TYPE(type) || IS_FLOAT_TYPE(type)){
-    convertStringToDouble(*pLeftData, pLeftOut, typeLeft, type);
+    if(typeLeft == TSDB_DATA_TYPE_NCHAR) {
-    *pLeftData = pLeftOut;
+      convertNcharToDouble(*pLeftData, pLeftOut);
-  } else if(typeLeft != type) {
+      *pLeftData = pLeftOut;
-    convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
+    } else if(typeLeft == TSDB_DATA_TYPE_BINARY) {
-    *pLeftData = pLeftOut;
+      convertBinaryToDouble(*pLeftData, pLeftOut);
+      *pLeftData = pLeftOut;
+    } else if(typeLeft != type) {
+      convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type);
+      *pLeftData = pLeftOut;
+    }
+
+    if(typeRight == TSDB_DATA_TYPE_NCHAR) {
+      convertNcharToDouble(*pRightData, pRightOut);
+      *pRightData = pRightOut;
+    } else if(typeRight == TSDB_DATA_TYPE_BINARY) {
+      convertBinaryToDouble(*pRightData, pRightOut);
+      *pRightData = pRightOut;
+    } else if(typeRight != type) {
+      convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
+      *pRightData = pRightOut;
+    }
+  }else if(type == TSDB_DATA_TYPE_BINARY){
+    if(typeLeft == TSDB_DATA_TYPE_NCHAR){
+      *pLeftData = ncharTobinary(*pLeftData);
+      *freeLeft = true;
+    }
+    if(typeRight == TSDB_DATA_TYPE_NCHAR){
+      *pRightData = ncharTobinary(*pRightData);
+      *freeRight = true;
+    }
+  }else{
+    ASSERT(0);
   }
 
-  if(typeRight == TSDB_DATA_TYPE_NCHAR) {
-    convertStringToDouble(*pRightData, pRightOut, typeRight, type);
-    *pRightData = pRightOut;
-  } else if(typeRight != type) {
-    convertNumberToNumber(*pRightData, pRightOut, typeRight, type);
-    *pRightData = pRightOut;
-  }
   return true;
 }
 
@@ -935,44 +982,6 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) {
   }
 }
 
-STagVal getJsonValue(char *json, char *key, bool *isExist) {
-  STagVal val = {.pKey = key};
-  bool find = tTagGet(((const STag *)json), &val);  // json value is null and not exist is different
-  if(isExist){
-    *isExist = find;
-  }
-  return val;
-}
-
-void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
-  SColumnInfoData *pOutputCol = pOut->columnData;
-
-  int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
-  int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1;
-
-  pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
-
-  char *pRightData = colDataGetVarData(pRight->columnData, 0);
-  char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1);
-  memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData));
-  for (; i >= 0 && i < pLeft->numOfRows; i += step) {
-    if (colDataIsNull_var(pLeft->columnData, i)) {
-      colDataSetNull_var(pOutputCol, i);
-      pOutputCol->hasNull = true;
-      continue;
-    }
-    char *pLeftData = colDataGetVarData(pLeft->columnData, i);
-    bool isExist = false;
-    STagVal value = getJsonValue(pLeftData, jsonKey, &isExist);
-    char *data = isExist ? tTagValToData(&value, true) : NULL;
-    colDataAppend(pOutputCol, i, data, data == NULL);
-    if(isExist && IS_VAR_DATA_TYPE(value.type) && data){
-      taosMemoryFree(data);
-    }
-  }
-  taosMemoryFree(jsonKey);
-}
-
 void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   SColumnInfoData *pOutputCol = pOut->columnData;
 
@@ -1510,6 +1519,38 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
   doReleaseVec(pRightCol, rightConvert);
 }
 
+#define VEC_COM_INNER(pCol, index1, index2)    \
+  for (; i < pCol->numOfRows && i >= 0; i += step) {\
+    if (IS_HELPER_NULL(pLeft->columnData, index1) || IS_HELPER_NULL(pRight->columnData, index2)) {\
+      bool  res = false;\
+      colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\
+      continue;\
+    }\
+    char *pLeftData = colDataGetData(pLeft->columnData, index1);\
+    char *pRightData = colDataGetData(pRight->columnData, index2);\
+    int64_t leftOut = 0;\
+    int64_t rightOut = 0;\
+    bool freeLeft = false;\
+    bool freeRight = false;\
+    bool isJsonnull = false;\
+    bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight),\
+                                      &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight);\
+    if(isJsonnull){\
+      ASSERT(0);\
+    }\
+    if(!pLeftData || !pRightData){\
+      result = false;\
+    }\
+    if(!result){\
+      colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);\
+    }else{\
+      bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);\
+      colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\
+    }\
+    if(freeLeft) taosMemoryFreeClear(pLeftData);\
+    if(freeRight) taosMemoryFreeClear(pRightData);\
+  }
+
 void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) {
   int32_t       i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
   int32_t       step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
@@ -1533,79 +1574,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *
   }
 
   if (pLeft->numOfRows == pRight->numOfRows) {
-    for (; i < pRight->numOfRows && i >= 0; i += step) {
+    VEC_COM_INNER(pLeft, i, i)
-      if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, i)) {
-        bool  res = false;
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-        continue;  // TODO set null or ignore
-      }
-
-      char *pLeftData = colDataGetData(pLeft->columnData, i);
-      char *pRightData = colDataGetData(pRight->columnData, i);
-
-      int64_t leftOut = 0;
-      int64_t rightOut = 0;
-      bool isJsonnull = false;
-      bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
-      if(isJsonnull){
-        ASSERT(0);
-      }
-      if(!result){
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
-      }else{
-        bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-      }
-    }
   } else if (pRight->numOfRows == 1) {
-    ASSERT(pLeft->pHashFilter == NULL);
+    VEC_COM_INNER(pLeft, i, 0)
-    for (; i >= 0 && i < pLeft->numOfRows; i += step) {
-      if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, 0)) {
-        bool  res = false;
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-        continue;
-      }
-
-      char *pLeftData = colDataGetData(pLeft->columnData, i);
-      char *pRightData = colDataGetData(pRight->columnData, 0);
-      int64_t leftOut = 0;
-      int64_t rightOut = 0;
-      bool isJsonnull = false;
-      bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
-      if(isJsonnull){
-        ASSERT(0);
-      }
-      if(!result){
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
-      }else{
-        bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-      }
-    }
   } else if (pLeft->numOfRows == 1) {
-    for (; i >= 0 && i < pRight->numOfRows; i += step) {
+    VEC_COM_INNER(pRight, 0, i)
-      if (IS_HELPER_NULL(pRight->columnData, i) || IS_HELPER_NULL(pLeft->columnData, 0)) {
-        bool  res = false;
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-        continue;
-      }
-
-      char *pLeftData = colDataGetData(pLeft->columnData, 0);
-      char *pRightData = colDataGetData(pLeft->columnData, i);
-      int64_t leftOut = 0;
-      int64_t rightOut = 0;
-      bool isJsonnull = false;
-      bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull);
-      if(isJsonnull){
-        ASSERT(0);
-      }
-      if(!result){
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);
-      }else{
-        bool  res = filterDoCompare(fp, optr, pLeftData, pRightData);
-        colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);
-      }
-    }
   }
 }
 
@@ -1683,10 +1656,6 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu
   vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH);
 }
 
-void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
-  vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_JSON_CONTAINS);
-}
-
 void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
   for(int32_t i = 0; i < pLeft->numOfRows; ++i) {
     int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0;
@@ -1707,6 +1676,69 @@ void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut,
   vectorConvertImpl(pLeft, pOut);
 }
 
+STagVal getJsonValue(char *json, char *key, bool *isExist) {
+  STagVal val = {.pKey = key};
+  bool find = tTagGet(((const STag *)json), &val);  // json value is null and not exist is different
+  if(isExist){
+    *isExist = find;
+  }
+  return val;
+}
+
+void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
+  SColumnInfoData *pOutputCol = pOut->columnData;
+
+  int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
+  int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1;
+
+  pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
+
+  char *pRightData = colDataGetVarData(pRight->columnData, 0);
+  char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1);
+  memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData));
+  for (; i >= 0 && i < pLeft->numOfRows; i += step) {
+    bool isExist = false;
+
+    if (!colDataIsNull_var(pLeft->columnData, i)) {
+      char *pLeftData = colDataGetVarData(pLeft->columnData, i);
+      getJsonValue(pLeftData, jsonKey, &isExist);
+    }
+
+    colDataAppend(pOutputCol, i, (const char*)(&isExist), false);
+
+  }
+  taosMemoryFree(jsonKey);
+}
+
+void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) {
+  SColumnInfoData *pOutputCol = pOut->columnData;
+
+  int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1;
+  int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1;
+
+  pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows);
+
+  char *pRightData = colDataGetVarData(pRight->columnData, 0);
+  char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1);
+  memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData));
+  for (; i >= 0 && i < pLeft->numOfRows; i += step) {
+    if (colDataIsNull_var(pLeft->columnData, i)) {
+      colDataSetNull_var(pOutputCol, i);
+      pOutputCol->hasNull = true;
+      continue;
+    }
+    char *pLeftData = colDataGetVarData(pLeft->columnData, i);
+    bool isExist = false;
+    STagVal value = getJsonValue(pLeftData, jsonKey, &isExist);
+    char *data = isExist ? tTagValToData(&value, true) : NULL;
+    colDataAppend(pOutputCol, i, data, data == NULL);
+    if(isExist && IS_VAR_DATA_TYPE(value.type) && data){
+      taosMemoryFree(data);
+    }
+  }
+  taosMemoryFree(jsonKey);
+}
+
 _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) {
   switch (binFunctionId) {
     case OP_TYPE_ADD:

source/libs/scheduler/src/schJob.c

@@ -108,7 +108,7 @@ int32_t schInitJob(SSchJob **pSchJob, SQueryPlan *pDag, void *pTrans, SArray *pN
 
   pJob->refId = refId;
 
-  SCH_JOB_DLOG("job refId:%" PRIx64, pJob->refId);
+  SCH_JOB_DLOG("job refId:0x%" PRIx64" created", pJob->refId);
 
   pJob->status = JOB_TASK_STATUS_NOT_START;
 

source/libs/stream/inc/streamInc.h

@@ -13,15 +13,21 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#ifndef _STREAM_INC_H_
+#define _STREAM_INC_H_
+
+#include "executor.h"
+#include "tstream.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#ifndef _TSTREAM_H_
+int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb);
-#define _TSTREAM_H_
+int32_t streamSink1(SStreamTask* pTask, SMsgCb* pMsgCb);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /* ifndef _TSTREAM_H_ */
+#endif /* ifndef _STREAM_INC_H_ */

source/libs/stream/src/stream.c

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "streamInc.h"
+
+int32_t streamTriggerByWrite(SStreamTask* pTask, int32_t vgId, SMsgCb* pMsgCb) {
+  int8_t execStatus = atomic_load_8(&pTask->status);
+  if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) {
+    SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
+    if (pRunReq == NULL) return -1;
+
+    // TODO: do we need htonl?
+    pRunReq->head.vgId = vgId;
+    pRunReq->streamId = pTask->streamId;
+    pRunReq->taskId = pTask->taskId;
+    SRpcMsg msg = {
+        .msgType = TDMT_VND_TASK_RUN,
+        .pCont = pRunReq,
+        .contLen = sizeof(SStreamTaskRunReq),
+    };
+    tmsgPutToQueue(pMsgCb, FETCH_QUEUE, &msg);
+  }
+  return 0;
+}
+
+#if 1
+int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
+  SStreamDataBlock* pBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
+  int8_t            status;
+
+  // enqueue
+  if (pBlock != NULL) {
+    pBlock->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
+    pBlock->sourceVg = pReq->sourceVg;
+    pBlock->blocks = pReq->data;
+    /*pBlock->sourceVer = pReq->sourceVer;*/
+    if (streamTaskInput(pTask, (SStreamQueueItem*)pBlock) == 0) {
+      status = TASK_INPUT_STATUS__NORMAL;
+    } else {
+      status = TASK_INPUT_STATUS__FAILED;
+    }
+  } else {
+    streamTaskInputFail(pTask);
+    status = TASK_INPUT_STATUS__FAILED;
+  }
+
+  // rsp by input status
+  SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp));
+  pCont->inputStatus = status;
+  pCont->streamId = pReq->streamId;
+  pCont->taskId = pReq->sourceTaskId;
+  pRsp->pCont = pCont;
+  pRsp->contLen = sizeof(SStreamDispatchRsp);
+  tmsgSendRsp(pRsp);
+  return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1;
+}
+#endif
+
+int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
+  // 1. handle input
+  streamTaskEnqueue(pTask, pReq, pRsp);
+
+  // 2. try exec
+  // 2.1. idle: exec
+  // 2.2. executing: return
+  // 2.3. closing: keep trying
+  streamExec(pTask, pMsgCb);
+
+  // 3. handle output
+  // 3.1 check and set status
+  // 3.2 dispatch / sink
+  streamSink1(pTask, pMsgCb);
+
+  return 0;
+}
+
+int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp) {
+  if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) {
+    // TODO: init recover timer
+  }
+  // continue dispatch
+  streamSink1(pTask, pMsgCb);
+  return 0;
+}
+
+int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb) {
+  streamExec(pTask, pMsgCb);
+  streamSink1(pTask, pMsgCb);
+  return 0;
+}
+
+int32_t streamProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg) {
+  //
+  return 0;
+}
+
+int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) {
+  //
+  return 0;
+}

source/libs/stream/src/streamData.c

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tstream.h"
+
+#if 0
+int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI8(buf, pOutput->type);
+  tlen += taosEncodeFixedI32(buf, pOutput->sourceVg);
+  tlen += taosEncodeFixedI64(buf, pOutput->sourceVer);
+  ASSERT(pOutput->type == STREAM_INPUT__DATA_BLOCK);
+  tlen += tEncodeDataBlocks(buf, pOutput->blocks);
+  return tlen;
+}
+
+void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) {
+  buf = taosDecodeFixedI8(buf, &pInput->type);
+  buf = taosDecodeFixedI32(buf, &pInput->sourceVg);
+  buf = taosDecodeFixedI64(buf, &pInput->sourceVer);
+  ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK);
+  buf = tDecodeDataBlocks(buf, &pInput->blocks);
+  return (void*)buf;
+}
+#endif
+
+SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) {
+  SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
+  if (pDataSubmit == NULL) return NULL;
+  pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t));
+  if (pDataSubmit->dataRef == NULL) goto FAIL;
+  pDataSubmit->data = pReq;
+  *pDataSubmit->dataRef = 1;
+  pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT;
+  return pDataSubmit;
+FAIL:
+  taosFreeQitem(pDataSubmit);
+  return NULL;
+}
+
+static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) {
+  //
+  atomic_add_fetch_32(pDataSubmit->dataRef, 1);
+}
+
+SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit) {
+  SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
+  if (pSubmitClone == NULL) {
+    return NULL;
+  }
+  streamDataSubmitRefInc(pSubmit);
+  memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit));
+  return pSubmitClone;
+}
+
+#if 0
+int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq) {
+  int32_t tlen = 0;
+  tlen += taosEncodeFixedI64(buf, pReq->streamId);
+  tlen += taosEncodeFixedI32(buf, pReq->taskId);
+  tlen += tEncodeDataBlocks(buf, pReq->data);
+  return tlen;
+}
+
+void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq) {
+  buf = taosDecodeFixedI64(buf, &pReq->streamId);
+  buf = taosDecodeFixedI32(buf, &pReq->taskId);
+  buf = tDecodeDataBlocks(buf, &pReq->data);
+  return (void*)buf;
+}
+
+void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { taosArrayDestroy(pReq->data); }
+#endif

source/libs/stream/src/streamExec.c

@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "executor.h"
+#include "tstream.h"
+
+static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
+  void* exec = pTask->exec.executor;
+
+  // set input
+  if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
+    SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
+    ASSERT(pSubmit->type == STREAM_INPUT__DATA_SUBMIT);
+
+    qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
+  } else if (pTask->inputType == STREAM_INPUT__DATA_BLOCK) {
+    SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
+    ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);
+
+    SArray* blocks = pBlock->blocks;
+    qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
+  }
+
+  // exec
+  while (1) {
+    SSDataBlock* output = NULL;
+    uint64_t     ts = 0;
+    if (qExecTask(exec, &output, &ts) < 0) {
+      ASSERT(false);
+    }
+    if (output == NULL) break;
+    // TODO: do we need free memory?
+    SSDataBlock* outputCopy = createOneDataBlock(output, true);
+    taosArrayPush(pRes, outputCopy);
+  }
+  return 0;
+}
+
+static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
+  while (1) {
+    void* data = streamQueueNextItem(pTask->inputQueue);
+    if (data == NULL) break;
+
+    streamTaskExecImpl(pTask, data, pRes);
+
+    if (taosArrayGetSize(pRes) != 0) {
+      SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
+      if (qRes == NULL) {
+        streamQueueProcessFail(pTask->inputQueue);
+        taosArrayDestroy(pRes);
+        return NULL;
+      }
+      qRes->type = STREAM_INPUT__DATA_BLOCK;
+      qRes->blocks = pRes;
+      if (streamTaskOutput(pTask, qRes) < 0) {
+        streamQueueProcessFail(pTask->inputQueue);
+        taosArrayDestroy(pRes);
+        taosFreeQitem(qRes);
+        return NULL;
+      }
+
+      if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
+        streamDataSubmitRefDec((SStreamDataSubmit*)data);
+        taosFreeQitem(data);
+      } else {
+        taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
+        taosFreeQitem(data);
+      }
+      streamQueueProcessSuccess(pTask->inputQueue);
+      return taosArrayInit(0, sizeof(SSDataBlock));
+    }
+  }
+  return pRes;
+}
+
+// TODO: handle version
+int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) {
+  SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
+  if (pRes == NULL) return -1;
+  while (1) {
+    int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
+    if (execStatus == TASK_STATUS__IDLE) {
+      // first run
+      pRes = streamExecForQall(pTask, pRes);
+      if (pRes == NULL) goto FAIL;
+
+      // set status closing
+      atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
+
+      // second run, make sure inputQ and qall are cleared
+      pRes = streamExecForQall(pTask, pRes);
+      if (pRes == NULL) goto FAIL;
+
+      break;
+    } else if (execStatus == TASK_STATUS__CLOSING) {
+      continue;
+    } else if (execStatus == TASK_STATUS__EXECUTING) {
+      break;
+    } else {
+      ASSERT(0);
+    }
+  }
+  if (pRes) taosArrayDestroy(pRes);
+  atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
+  return 0;
+FAIL:
+  if (pRes) taosArrayDestroy(pRes);
+  atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
+  return -1;
+}
+

source/libs/stream/src/streamMsg.c

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tstream.h"
+
+int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* pReq) {
+  if (tStartEncode(pEncoder) < 0) return -1;
+  if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->sourceChildId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1;
+  ASSERT(taosArrayGetSize(pReq->data) == pReq->blockNum);
+  ASSERT(taosArrayGetSize(pReq->dataLen) == pReq->blockNum);
+  for (int32_t i = 0; i < pReq->blockNum; i++) {
+    int32_t len = *(int32_t*)taosArrayGet(pReq->dataLen, i);
+    void*   data = taosArrayGetP(pReq->data, i);
+    if (tEncodeI32(pEncoder, len) < 0) return -1;
+    if (tEncodeBinary(pEncoder, data, len) < 0) return -1;
+  }
+  tEndEncode(pEncoder);
+  return 0;
+}
+
+int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
+  if (tStartDecode(pDecoder) < 0) return -1;
+  if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->sourceChildId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pReq->blockNum) < 0) return -1;
+  ASSERT(pReq->blockNum > 0);
+  pReq->data = taosArrayInit(pReq->blockNum, sizeof(void*));
+  pReq->dataLen = taosArrayInit(pReq->blockNum, sizeof(int32_t));
+  for (int32_t i = 0; i < pReq->blockNum; i++) {
+    int32_t  len1;
+    uint64_t len2;
+    void*    data;
+    if (tDecodeI32(pDecoder, &len1) < 0) return -1;
+    if (tDecodeBinaryAlloc(pDecoder, &data, &len2) < 0) return -1;
+    ASSERT(len1 == len2);
+    taosArrayPush(pReq->dataLen, &len1);
+    taosArrayPush(pReq->data, &data);
+  }
+  tEndDecode(pDecoder);
+  return 0;
+}
+
+int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
+  SStreamDispatchReq req = {
+      .streamId = pTask->streamId,
+      .data = data,
+  };
+  return 0;
+}
+
+static int32_t streamBuildExecMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
+  SStreamTaskExecReq req = {
+      .streamId = pTask->streamId,
+      .data = data,
+  };
+
+  int32_t tlen = sizeof(SMsgHead) + tEncodeSStreamTaskExecReq(NULL, &req);
+  void*   buf = rpcMallocCont(tlen);
+
+  if (buf == NULL) {
+    return -1;
+  }
+
+  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
+    ((SMsgHead*)buf)->vgId = 0;
+    req.taskId = pTask->inplaceDispatcher.taskId;
+
+  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
+    ((SMsgHead*)buf)->vgId = htonl(pTask->fixedEpDispatcher.nodeId);
+    *ppEpSet = &pTask->fixedEpDispatcher.epSet;
+    req.taskId = pTask->fixedEpDispatcher.taskId;
+
+  } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
+    // TODO use general name rule of schemaless
+    char ctbName[TSDB_TABLE_FNAME_LEN + 22] = {0};
+    // all groupId must be the same in an array
+    SSDataBlock* pBlock = taosArrayGet(data, 0);
+    sprintf(ctbName, "%s:%ld", pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId);
+
+    // TODO: get hash function by hashMethod
+
+    // get groupId, compute hash value
+    uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));
+
+    // get node
+    // TODO: optimize search process
+    SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
+    int32_t sz = taosArrayGetSize(vgInfo);
+    int32_t nodeId = 0;
+    for (int32_t i = 0; i < sz; i++) {
+      SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
+      if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) {
+        nodeId = pVgInfo->vgId;
+        req.taskId = pVgInfo->taskId;
+        *ppEpSet = &pVgInfo->epSet;
+        break;
+      }
+    }
+    ASSERT(nodeId != 0);
+    ((SMsgHead*)buf)->vgId = htonl(nodeId);
+  }
+
+  void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
+  tEncodeSStreamTaskExecReq(&abuf, &req);
+
+  pMsg->pCont = buf;
+  pMsg->contLen = tlen;
+  pMsg->code = 0;
+  pMsg->msgType = pTask->dispatchMsgType;
+  pMsg->info.noResp = 1;
+
+  return 0;
+}
+
+static int32_t streamShuffleDispatch(SStreamTask* pTask, SMsgCb* pMsgCb, SHashObj* data) {
+  void* pIter = NULL;
+  while (1) {
+    pIter = taosHashIterate(data, pIter);
+    if (pIter == NULL) return 0;
+    SArray* pData = *(SArray**)pIter;
+    SRpcMsg dispatchMsg = {0};
+    SEpSet* pEpSet;
+    if (streamBuildExecMsg(pTask, pData, &dispatchMsg, &pEpSet) < 0) {
+      ASSERT(0);
+      return -1;
+    }
+    tmsgSendReq(pEpSet, &dispatchMsg);
+  }
+  return 0;
+}

source/libs/stream/src/streamQueue.c

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tstream.h"
+
+SStreamQueue* streamQueueOpen() {
+  SStreamQueue* pQueue = taosMemoryCalloc(1, sizeof(SStreamQueue));
+  if (pQueue == NULL) return NULL;
+  pQueue->queue = taosOpenQueue();
+  pQueue->qall = taosAllocateQall();
+  if (pQueue->queue == NULL || pQueue->qall == NULL) {
+    goto FAIL;
+  }
+  pQueue->status = STREAM_QUEUE__SUCESS;
+  return pQueue;
+FAIL:
+  if (pQueue->queue) taosCloseQueue(pQueue->queue);
+  if (pQueue->qall) taosFreeQall(pQueue->qall);
+  taosMemoryFree(pQueue);
+  return NULL;
+}
+
+void streamQueueClose(SStreamQueue* queue) {
+  while (1) {
+    void* qItem = streamQueueNextItem(queue);
+    if (qItem)
+      taosFreeQitem(qItem);
+    else
+      return;
+  }
+}

source/libs/stream/src/streamSink.c

@@ -0,0 +1,157 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "executor.h"
+#include "tstream.h"
+
+int32_t streamSink1(SStreamTask* pTask, SMsgCb* pMsgCb) {
+  SStreamQueue* queue;
+  if (pTask->execType == TASK_EXEC__NONE) {
+    queue = pTask->inputQueue;
+  } else {
+    queue = pTask->outputQueue;
+  }
+  /*if (streamDequeueBegin(queue) == true) {*/
+  /*return -1;*/
+  /*}*/
+
+  if (pTask->sinkType == TASK_SINK__TABLE || pTask->sinkType == TASK_SINK__SMA) {
+    ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
+    while (1) {
+      SStreamDataBlock* pBlock = streamQueueNextItem(queue);
+      if (pBlock == NULL) break;
+      ASSERT(pBlock->type == STREAM_DATA_TYPE_SSDATA_BLOCK);
+
+      // local sink
+      if (pTask->sinkType == TASK_SINK__TABLE) {
+        pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks);
+      } else if (pTask->sinkType == TASK_SINK__SMA) {
+        pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pBlock->blocks);
+      }
+
+      if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
+        ASSERT(queue == pTask->outputQueue);
+      } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
+        ASSERT(queue == pTask->outputQueue);
+      } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
+        ASSERT(queue == pTask->outputQueue);
+      }
+
+      streamQueueProcessSuccess(queue);
+    }
+  }
+
+  return 0;
+}
+
+#if 0
+int32_t streamSink(SStreamTask* pTask, SMsgCb* pMsgCb) {
+  bool firstRun = 1;
+  while (1) {
+    SStreamDataBlock* pBlock = NULL;
+    if (!firstRun) {
+      taosReadAllQitems(pTask->outputQ, pTask->outputQAll);
+    }
+    taosGetQitem(pTask->outputQAll, (void**)&pBlock);
+    if (pBlock == NULL) {
+      if (firstRun) {
+        firstRun = 0;
+        continue;
+      } else {
+        break;
+      }
+    }
+
+    SArray* pRes = pBlock->blocks;
+
+    // sink
+    if (pTask->sinkType == TASK_SINK__TABLE) {
+      // blockDebugShowData(pRes);
+      pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pRes);
+    } else if (pTask->sinkType == TASK_SINK__SMA) {
+      pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes);
+      //
+    } else if (pTask->sinkType == TASK_SINK__FETCH) {
+      //
+    } else {
+      ASSERT(pTask->sinkType == TASK_SINK__NONE);
+    }
+
+    // dispatch
+    // TODO dispatch guard
+    int8_t outputStatus = atomic_load_8(&pTask->outputStatus);
+    if (outputStatus == TASK_OUTPUT_STATUS__NORMAL) {
+      if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
+        SRpcMsg dispatchMsg = {0};
+        if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, NULL) < 0) {
+          ASSERT(0);
+          return -1;
+        }
+
+        int32_t qType;
+        if (pTask->dispatchMsgType == TDMT_VND_TASK_DISPATCH || pTask->dispatchMsgType == TDMT_SND_TASK_DISPATCH) {
+          qType = FETCH_QUEUE;
+          /*} else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC ||*/
+          /*pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) {*/
+          /*qType = MERGE_QUEUE;*/
+          /*} else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) {*/
+          /*qType = WRITE_QUEUE;*/
+        } else {
+          ASSERT(0);
+        }
+        tmsgPutToQueue(pMsgCb, qType, &dispatchMsg);
+
+      } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
+        SRpcMsg dispatchMsg = {0};
+        SEpSet* pEpSet = NULL;
+        if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) {
+          ASSERT(0);
+          return -1;
+        }
+
+        tmsgSendReq(pEpSet, &dispatchMsg);
+
+      } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
+        SHashObj* pShuffleRes = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+        if (pShuffleRes == NULL) {
+          return -1;
+        }
+
+        int32_t sz = taosArrayGetSize(pRes);
+        for (int32_t i = 0; i < sz; i++) {
+          SSDataBlock* pDataBlock = taosArrayGet(pRes, i);
+          SArray*      pArray = taosHashGet(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t));
+          if (pArray == NULL) {
+            pArray = taosArrayInit(0, sizeof(SSDataBlock));
+            if (pArray == NULL) {
+              return -1;
+            }
+            taosHashPut(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t), &pArray, sizeof(void*));
+          }
+          taosArrayPush(pArray, pDataBlock);
+        }
+
+        if (streamShuffleDispatch(pTask, pMsgCb, pShuffleRes) < 0) {
+          return -1;
+        }
+
+      } else {
+        ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
+      }
+    }
+  }
+  return 0;
+}
+#endif

source/libs/stream/src/streamTask.c

@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "executor.h"
+#include "tstream.h"
+
+SStreamTask* tNewSStreamTask(int64_t streamId, int32_t childId) {
+  SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
+  if (pTask == NULL) {
+    return NULL;
+  }
+  pTask->taskId = tGenIdPI32();
+  pTask->streamId = streamId;
+  pTask->childId = childId;
+  pTask->status = TASK_STATUS__IDLE;
+  pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
+  pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
+
+  return pTask;
+}
+
+int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
+  /*if (tStartEncode(pEncoder) < 0) return -1;*/
+  if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->status) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1;
+  if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1;
+  if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1;
+
+  if (tEncodeI32(pEncoder, pTask->childId) < 0) return -1;
+  if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1;
+  if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1;
+
+  if (pTask->execType != TASK_EXEC__NONE) {
+    if (tEncodeI8(pEncoder, pTask->exec.parallelizable) < 0) return -1;
+    if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1;
+  }
+
+  if (pTask->sinkType == TASK_SINK__TABLE) {
+    if (tEncodeI64(pEncoder, pTask->tbSink.stbUid) < 0) return -1;
+    if (tEncodeCStr(pEncoder, pTask->tbSink.stbFullName) < 0) return -1;
+    if (tEncodeSSchemaWrapper(pEncoder, pTask->tbSink.pSchemaWrapper) < 0) return -1;
+  } else if (pTask->sinkType == TASK_SINK__SMA) {
+    if (tEncodeI64(pEncoder, pTask->smaSink.smaId) < 0) return -1;
+  } else if (pTask->sinkType == TASK_SINK__FETCH) {
+    if (tEncodeI8(pEncoder, pTask->fetchSink.reserved) < 0) return -1;
+  } else {
+    ASSERT(pTask->sinkType == TASK_SINK__NONE);
+  }
+
+  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
+    if (tEncodeI32(pEncoder, pTask->inplaceDispatcher.taskId) < 0) return -1;
+  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
+    if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1;
+    if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1;
+    if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1;
+  } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
+    if (tSerializeSUseDbRspImp(pEncoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1;
+    /*if (tEncodeI8(pEncoder, pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/
+  }
+
+  /*tEndEncode(pEncoder);*/
+  return pEncoder->pos;
+}
+
+int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
+  /*if (tStartDecode(pDecoder) < 0) return -1;*/
+  if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1;
+  if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1;
+  if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1;
+
+  if (tDecodeI32(pDecoder, &pTask->childId) < 0) return -1;
+  if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1;
+  if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1;
+
+  if (pTask->execType != TASK_EXEC__NONE) {
+    if (tDecodeI8(pDecoder, &pTask->exec.parallelizable) < 0) return -1;
+    if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1;
+  }
+
+  if (pTask->sinkType == TASK_SINK__TABLE) {
+    if (tDecodeI64(pDecoder, &pTask->tbSink.stbUid) < 0) return -1;
+    if (tDecodeCStrTo(pDecoder, pTask->tbSink.stbFullName) < 0) return -1;
+    pTask->tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper));
+    if (pTask->tbSink.pSchemaWrapper == NULL) return -1;
+    if (tDecodeSSchemaWrapper(pDecoder, pTask->tbSink.pSchemaWrapper) < 0) return -1;
+  } else if (pTask->sinkType == TASK_SINK__SMA) {
+    if (tDecodeI64(pDecoder, &pTask->smaSink.smaId) < 0) return -1;
+  } else if (pTask->sinkType == TASK_SINK__FETCH) {
+    if (tDecodeI8(pDecoder, &pTask->fetchSink.reserved) < 0) return -1;
+  } else {
+    ASSERT(pTask->sinkType == TASK_SINK__NONE);
+  }
+
+  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
+    if (tDecodeI32(pDecoder, &pTask->inplaceDispatcher.taskId) < 0) return -1;
+  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
+    if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1;
+    if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1;
+    if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1;
+  } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
+    /*if (tDecodeI8(pDecoder, &pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/
+    if (tDeserializeSUseDbRspImp(pDecoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1;
+  }
+
+  /*tEndDecode(pDecoder);*/
+  return 0;
+}
+
+void tFreeSStreamTask(SStreamTask* pTask) {
+  streamQueueClose(pTask->inputQueue);
+  streamQueueClose(pTask->outputQueue);
+  if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg);
+  qDestroyTask(pTask->exec.executor);
+  taosMemoryFree(pTask);
+}

source/libs/stream/src/tstream.c

@@ -1,594 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "tstream.h"
-#include "executor.h"
-
-int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI8(buf, pOutput->type);
-  tlen += taosEncodeFixedI32(buf, pOutput->sourceVg);
-  tlen += taosEncodeFixedI64(buf, pOutput->sourceVer);
-  ASSERT(pOutput->type == STREAM_INPUT__DATA_BLOCK);
-  tlen += tEncodeDataBlocks(buf, pOutput->blocks);
-  return tlen;
-}
-
-void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) {
-  buf = taosDecodeFixedI8(buf, &pInput->type);
-  buf = taosDecodeFixedI32(buf, &pInput->sourceVg);
-  buf = taosDecodeFixedI64(buf, &pInput->sourceVer);
-  ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK);
-  buf = tDecodeDataBlocks(buf, &pInput->blocks);
-  return (void*)buf;
-}
-
-SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit) {
-  SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM);
-  if (pSubmitClone == NULL) {
-    return NULL;
-  }
-  streamDataSubmitRefInc(pSubmit);
-  memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit));
-  return pSubmitClone;
-}
-
-static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
-  SStreamDispatchReq req = {
-      .streamId = pTask->streamId,
-      .data = data,
-  };
-  return 0;
-}
-
-static int32_t streamBuildExecMsg(SStreamTask* pTask, SArray* data, SRpcMsg* pMsg, SEpSet** ppEpSet) {
-  SStreamTaskExecReq req = {
-      .streamId = pTask->streamId,
-      .data = data,
-  };
-
-  int32_t tlen = sizeof(SMsgHead) + tEncodeSStreamTaskExecReq(NULL, &req);
-  void*   buf = rpcMallocCont(tlen);
-
-  if (buf == NULL) {
-    return -1;
-  }
-
-  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
-    ((SMsgHead*)buf)->vgId = 0;
-    req.taskId = pTask->inplaceDispatcher.taskId;
-
-  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
-    ((SMsgHead*)buf)->vgId = htonl(pTask->fixedEpDispatcher.nodeId);
-    *ppEpSet = &pTask->fixedEpDispatcher.epSet;
-    req.taskId = pTask->fixedEpDispatcher.taskId;
-
-  } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
-    // TODO use general name rule of schemaless
-    char ctbName[TSDB_TABLE_FNAME_LEN + 22] = {0};
-    // all groupId must be the same in an array
-    SSDataBlock* pBlock = taosArrayGet(data, 0);
-    sprintf(ctbName, "%s:%ld", pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId);
-
-    // TODO: get hash function by hashMethod
-
-    // get groupId, compute hash value
-    uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));
-
-    // get node
-    // TODO: optimize search process
-    SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
-    int32_t sz = taosArrayGetSize(vgInfo);
-    int32_t nodeId = 0;
-    for (int32_t i = 0; i < sz; i++) {
-      SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i);
-      if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) {
-        nodeId = pVgInfo->vgId;
-        req.taskId = pVgInfo->taskId;
-        *ppEpSet = &pVgInfo->epSet;
-        break;
-      }
-    }
-    ASSERT(nodeId != 0);
-    ((SMsgHead*)buf)->vgId = htonl(nodeId);
-  }
-
-  void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
-  tEncodeSStreamTaskExecReq(&abuf, &req);
-
-  pMsg->pCont = buf;
-  pMsg->contLen = tlen;
-  pMsg->code = 0;
-  pMsg->msgType = pTask->dispatchMsgType;
-  pMsg->info.noResp = 1;
-
-  return 0;
-}
-
-static int32_t streamShuffleDispatch(SStreamTask* pTask, SMsgCb* pMsgCb, SHashObj* data) {
-  void* pIter = NULL;
-  while (1) {
-    pIter = taosHashIterate(data, pIter);
-    if (pIter == NULL) return 0;
-    SArray* pData = *(SArray**)pIter;
-    SRpcMsg dispatchMsg = {0};
-    SEpSet* pEpSet;
-    if (streamBuildExecMsg(pTask, pData, &dispatchMsg, &pEpSet) < 0) {
-      ASSERT(0);
-      return -1;
-    }
-    tmsgSendReq(pEpSet, &dispatchMsg);
-  }
-  return 0;
-}
-
-int32_t streamEnqueueDataSubmit(SStreamTask* pTask, SStreamDataSubmit* input) {
-  ASSERT(pTask->inputType == TASK_INPUT_TYPE__SUMBIT_BLOCK);
-  int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
-  if (inputStatus == TASK_INPUT_STATUS__NORMAL) {
-    streamDataSubmitRefInc(input);
-    taosWriteQitem(pTask->inputQ, input);
-  }
-  return inputStatus;
-}
-
-int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) {
-  ASSERT(pTask->inputType == TASK_INPUT_TYPE__DATA_BLOCK);
-  taosWriteQitem(pTask->inputQ, input);
-  int8_t inputStatus = atomic_load_8(&pTask->inputStatus);
-  return inputStatus;
-}
-
-static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) {
-  void* exec = pTask->exec.executor;
-
-  // set input
-  if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
-    SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data;
-    ASSERT(pSubmit->type == STREAM_INPUT__DATA_SUBMIT);
-
-    qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
-  } else if (pTask->inputType == STREAM_INPUT__DATA_BLOCK) {
-    SStreamDataBlock* pBlock = (SStreamDataBlock*)data;
-    ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK);
-
-    SArray* blocks = pBlock->blocks;
-    qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
-  }
-
-  // exec
-  while (1) {
-    SSDataBlock* output = NULL;
-    uint64_t     ts = 0;
-    if (qExecTask(exec, &output, &ts) < 0) {
-      ASSERT(false);
-    }
-    if (output == NULL) break;
-    // TODO: do we need free memory?
-    SSDataBlock* outputCopy = createOneDataBlock(output, true);
-    taosArrayPush(pRes, outputCopy);
-  }
-
-  // destroy
-  if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
-    streamDataSubmitRefDec((SStreamDataSubmit*)data);
-    taosFreeQitem(data);
-  } else {
-    taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
-    taosFreeQitem(data);
-  }
-  return 0;
-}
-
-static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
-  while (1) {
-    void* data = NULL;
-    taosGetQitem(pTask->inputQAll, &data);
-    if (data == NULL) break;
-
-    streamTaskExecImpl(pTask, data, pRes);
-
-    if (taosArrayGetSize(pRes) != 0) {
-      SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
-      qRes->type = STREAM_INPUT__DATA_BLOCK;
-      qRes->blocks = pRes;
-      taosWriteQitem(pTask->outputQ, qRes);
-      return taosArrayInit(0, sizeof(SSDataBlock));
-    }
-  }
-  return pRes;
-}
-
-// TODO: handle version
-int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) {
-  SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
-  if (pRes == NULL) return -1;
-  while (1) {
-    int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING);
-    if (execStatus == TASK_STATUS__IDLE) {
-      // first run, from qall, handle failure from last exec
-      pRes = streamExecForQall(pTask, pRes);
-      if (pRes == NULL) goto FAIL;
-
-      // second run, from inputQ
-      taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
-      pRes = streamExecForQall(pTask, pRes);
-      if (pRes == NULL) goto FAIL;
-
-      // set status closing
-      atomic_store_8(&pTask->status, TASK_STATUS__CLOSING);
-
-      // third run, make sure inputQ and qall are cleared
-      taosReadAllQitems(pTask->inputQ, pTask->inputQAll);
-      pRes = streamExecForQall(pTask, pRes);
-      if (pRes == NULL) goto FAIL;
-
-      atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
-      break;
-    } else if (execStatus == TASK_STATUS__CLOSING) {
-      continue;
-    } else if (execStatus == TASK_STATUS__EXECUTING) {
-      break;
-    } else {
-      ASSERT(0);
-    }
-  }
-  return 0;
-FAIL:
-  atomic_store_8(&pTask->status, TASK_STATUS__IDLE);
-  return -1;
-}
-
-int32_t streamSink(SStreamTask* pTask, SMsgCb* pMsgCb) {
-  bool firstRun = 1;
-  while (1) {
-    SStreamDataBlock* pBlock = NULL;
-    if (!firstRun) {
-      taosReadAllQitems(pTask->outputQ, pTask->outputQAll);
-    }
-    taosGetQitem(pTask->outputQAll, (void**)&pBlock);
-    if (pBlock == NULL) {
-      if (firstRun) {
-        firstRun = 0;
-        continue;
-      } else {
-        break;
-      }
-    }
-
-    SArray* pRes = pBlock->blocks;
-
-    // sink
-    if (pTask->sinkType == TASK_SINK__TABLE) {
-      // blockDebugShowData(pRes);
-      pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pRes);
-    } else if (pTask->sinkType == TASK_SINK__SMA) {
-      pTask->smaSink.smaSink(pTask->ahandle, pTask->smaSink.smaId, pRes);
-      //
-    } else if (pTask->sinkType == TASK_SINK__FETCH) {
-      //
-    } else {
-      ASSERT(pTask->sinkType == TASK_SINK__NONE);
-    }
-
-    // dispatch
-    // TODO dispatch guard
-    int8_t outputStatus = atomic_load_8(&pTask->outputStatus);
-    if (outputStatus == TASK_OUTPUT_STATUS__NORMAL) {
-      if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
-        SRpcMsg dispatchMsg = {0};
-        if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, NULL) < 0) {
-          ASSERT(0);
-          return -1;
-        }
-
-        int32_t qType;
-        if (pTask->dispatchMsgType == TDMT_VND_TASK_DISPATCH || pTask->dispatchMsgType == TDMT_SND_TASK_DISPATCH) {
-          qType = FETCH_QUEUE;
-          /*} else if (pTask->dispatchMsgType == TDMT_VND_TASK_MERGE_EXEC ||*/
-          /*pTask->dispatchMsgType == TDMT_SND_TASK_MERGE_EXEC) {*/
-          /*qType = MERGE_QUEUE;*/
-          /*} else if (pTask->dispatchMsgType == TDMT_VND_TASK_WRITE_EXEC) {*/
-          /*qType = WRITE_QUEUE;*/
-        } else {
-          ASSERT(0);
-        }
-        tmsgPutToQueue(pMsgCb, qType, &dispatchMsg);
-
-      } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
-        SRpcMsg dispatchMsg = {0};
-        SEpSet* pEpSet = NULL;
-        if (streamBuildExecMsg(pTask, pRes, &dispatchMsg, &pEpSet) < 0) {
-          ASSERT(0);
-          return -1;
-        }
-
-        tmsgSendReq(pEpSet, &dispatchMsg);
-
-      } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
-        SHashObj* pShuffleRes = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
-        if (pShuffleRes == NULL) {
-          return -1;
-        }
-
-        int32_t sz = taosArrayGetSize(pRes);
-        for (int32_t i = 0; i < sz; i++) {
-          SSDataBlock* pDataBlock = taosArrayGet(pRes, i);
-          SArray*      pArray = taosHashGet(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t));
-          if (pArray == NULL) {
-            pArray = taosArrayInit(0, sizeof(SSDataBlock));
-            if (pArray == NULL) {
-              return -1;
-            }
-            taosHashPut(pShuffleRes, &pDataBlock->info.groupId, sizeof(int64_t), &pArray, sizeof(void*));
-          }
-          taosArrayPush(pArray, pDataBlock);
-        }
-
-        if (streamShuffleDispatch(pTask, pMsgCb, pShuffleRes) < 0) {
-          return -1;
-        }
-
-      } else {
-        ASSERT(pTask->dispatchType == TASK_DISPATCH__NONE);
-      }
-    }
-  }
-  return 0;
-}
-
-int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
-  SStreamDataBlock* pBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
-  int8_t            status;
-
-  // 1.1 update status
-  // TODO cal backpressure
-  if (pBlock == NULL) {
-    atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
-    status = TASK_INPUT_STATUS__FAILED;
-  } else {
-    status = atomic_load_8(&pTask->inputStatus);
-  }
-
-  // 1.2 enqueue
-  pBlock->type = STREAM_DATA_TYPE_SSDATA_BLOCK;
-  pBlock->sourceVg = pReq->sourceVg;
-  /*pBlock->sourceVer = pReq->sourceVer;*/
-  taosWriteQitem(pTask->inputQ, pBlock);
-
-  // 1.3 rsp by input status
-  SStreamDispatchRsp* pCont = rpcMallocCont(sizeof(SStreamDispatchRsp));
-  pCont->inputStatus = status;
-  pCont->streamId = pReq->streamId;
-  pCont->taskId = pReq->sourceTaskId;
-  pRsp->pCont = pCont;
-  pRsp->contLen = sizeof(SStreamDispatchRsp);
-  tmsgSendRsp(pRsp);
-
-  return 0;
-}
-
-int32_t streamProcessDispatchReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchReq* pReq, SRpcMsg* pRsp) {
-  // 1. handle input
-  streamTaskEnqueue(pTask, pReq, pRsp);
-
-  // 2. try exec
-  // 2.1. idle: exec
-  // 2.2. executing: return
-  // 2.3. closing: keep trying
-  streamExec(pTask, pMsgCb);
-
-  // 3. handle output
-  // 3.1 check and set status
-  // 3.2 dispatch / sink
-  streamSink(pTask, pMsgCb);
-
-  return 0;
-}
-
-int32_t streamProcessDispatchRsp(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDispatchRsp* pRsp) {
-  atomic_store_8(&pTask->inputStatus, pRsp->inputStatus);
-  if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) {
-    // TODO: init recover timer
-  }
-  // continue dispatch
-  streamSink(pTask, pMsgCb);
-  return 0;
-}
-
-int32_t streamTaskProcessRunReq(SStreamTask* pTask, SMsgCb* pMsgCb) {
-  streamExec(pTask, pMsgCb);
-  streamSink(pTask, pMsgCb);
-  return 0;
-}
-
-int32_t streamProcessRecoverReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg) {
-  //
-  return 0;
-}
-
-int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) {
-  //
-  return 0;
-}
-
-int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* pReq) {
-  if (tStartEncode(pEncoder) < 0) return -1;
-  if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1;
-  tEndEncode(pEncoder);
-  return 0;
-}
-
-int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) {
-  if (tStartDecode(pDecoder) < 0) return -1;
-  if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1;
-  tEndDecode(pDecoder);
-  return 0;
-}
-
-int32_t tEncodeSStreamTaskExecReq(void** buf, const SStreamTaskExecReq* pReq) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI64(buf, pReq->streamId);
-  tlen += taosEncodeFixedI32(buf, pReq->taskId);
-  tlen += tEncodeDataBlocks(buf, pReq->data);
-  return tlen;
-}
-
-void* tDecodeSStreamTaskExecReq(const void* buf, SStreamTaskExecReq* pReq) {
-  buf = taosDecodeFixedI64(buf, &pReq->streamId);
-  buf = taosDecodeFixedI32(buf, &pReq->taskId);
-  buf = tDecodeDataBlocks(buf, &pReq->data);
-  return (void*)buf;
-}
-
-void tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) { taosArrayDestroy(pReq->data); }
-
-SStreamTask* tNewSStreamTask(int64_t streamId) {
-  SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
-  if (pTask == NULL) {
-    return NULL;
-  }
-  pTask->taskId = tGenIdPI32();
-  pTask->streamId = streamId;
-  pTask->status = TASK_STATUS__IDLE;
-
-  return pTask;
-}
-
-int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
-  /*if (tStartEncode(pEncoder) < 0) return -1;*/
-  if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->status) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->sourceType) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1;
-  if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1;
-  if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1;
-
-  if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1;
-  if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1;
-
-  if (pTask->execType != TASK_EXEC__NONE) {
-    if (tEncodeI8(pEncoder, pTask->exec.parallelizable) < 0) return -1;
-    if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1;
-  }
-
-  if (pTask->sinkType == TASK_SINK__TABLE) {
-    if (tEncodeI64(pEncoder, pTask->tbSink.stbUid) < 0) return -1;
-    if (tEncodeCStr(pEncoder, pTask->tbSink.stbFullName) < 0) return -1;
-    if (tEncodeSSchemaWrapper(pEncoder, pTask->tbSink.pSchemaWrapper) < 0) return -1;
-  } else if (pTask->sinkType == TASK_SINK__SMA) {
-    if (tEncodeI64(pEncoder, pTask->smaSink.smaId) < 0) return -1;
-  } else if (pTask->sinkType == TASK_SINK__FETCH) {
-    if (tEncodeI8(pEncoder, pTask->fetchSink.reserved) < 0) return -1;
-  } else {
-    ASSERT(pTask->sinkType == TASK_SINK__NONE);
-  }
-
-  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
-    if (tEncodeI32(pEncoder, pTask->inplaceDispatcher.taskId) < 0) return -1;
-  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
-    if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1;
-    if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1;
-    if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1;
-  } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
-    if (tSerializeSUseDbRspImp(pEncoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1;
-    /*if (tEncodeI8(pEncoder, pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/
-  }
-
-  /*tEndEncode(pEncoder);*/
-  return pEncoder->pos;
-}
-
-int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
-  /*if (tStartDecode(pDecoder) < 0) return -1;*/
-  if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
-  if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->sourceType) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1;
-  if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1;
-  if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1;
-
-  if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1;
-  if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1;
-
-  if (pTask->execType != TASK_EXEC__NONE) {
-    if (tDecodeI8(pDecoder, &pTask->exec.parallelizable) < 0) return -1;
-    if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1;
-  }
-
-  if (pTask->sinkType == TASK_SINK__TABLE) {
-    if (tDecodeI64(pDecoder, &pTask->tbSink.stbUid) < 0) return -1;
-    if (tDecodeCStrTo(pDecoder, pTask->tbSink.stbFullName) < 0) return -1;
-    pTask->tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper));
-    if (pTask->tbSink.pSchemaWrapper == NULL) return -1;
-    if (tDecodeSSchemaWrapper(pDecoder, pTask->tbSink.pSchemaWrapper) < 0) return -1;
-  } else if (pTask->sinkType == TASK_SINK__SMA) {
-    if (tDecodeI64(pDecoder, &pTask->smaSink.smaId) < 0) return -1;
-  } else if (pTask->sinkType == TASK_SINK__FETCH) {
-    if (tDecodeI8(pDecoder, &pTask->fetchSink.reserved) < 0) return -1;
-  } else {
-    ASSERT(pTask->sinkType == TASK_SINK__NONE);
-  }
-
-  if (pTask->dispatchType == TASK_DISPATCH__INPLACE) {
-    if (tDecodeI32(pDecoder, &pTask->inplaceDispatcher.taskId) < 0) return -1;
-  } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) {
-    if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1;
-    if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1;
-    if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1;
-  } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) {
-    /*if (tDecodeI8(pDecoder, &pTask->shuffleDispatcher.hashMethod) < 0) return -1;*/
-    if (tDeserializeSUseDbRspImp(pDecoder, &pTask->shuffleDispatcher.dbInfo) < 0) return -1;
-  }
-
-  /*tEndDecode(pDecoder);*/
-  return 0;
-}
-
-void tFreeSStreamTask(SStreamTask* pTask) {
-  taosCloseQueue(pTask->inputQ);
-  taosCloseQueue(pTask->outputQ);
-  // TODO
-  if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg);
-  qDestroyTask(pTask->exec.executor);
-  taosMemoryFree(pTask);
-}
-
-#if 0
-int32_t tEncodeSStreamTaskExecReq(SCoder* pEncoder, const SStreamTaskExecReq* pReq) {
-  if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
-  if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1;
-  /*if (tEncodeDataBlocks(buf, pReq->streamId) < 0) return -1;*/
-  return pEncoder->size;
-}
-int32_t tDecodeSStreamTaskExecReq(SCoder* pDecoder, SStreamTaskExecReq* pReq) {
-  return pEncoder->size;
-}
-void    tFreeSStreamTaskExecReq(SStreamTaskExecReq* pReq) {
-  taosArrayDestroyEx(pReq->data, tDeleteSSDataBlock);
-}
-#endif

source/util/src/tcompare.c

@@ -222,11 +222,6 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) {
   return compareLenPrefixedWStr(pRight, pLeft);
 }
 
-int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) {
-  if(pLeft) return 0;
-  return 1;
-}
-
 // string > number > bool > null
 // ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison
 int32_t compareJsonVal(const void *pLeft, const void *pRight) {

source/util/src/terror.c

@@ -222,17 +222,19 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_INDEX_NOT_EXIST,       "Index not exist")
 // mnode-vgroup
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE,  "Vgroup already in dnode")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE,      "Vgroup not in dnode")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST,         "VGroup does not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST,         "Vgroup does not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_UN_CHANGED,        "Vgroup distribution has not changed")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_HAS_OFFLINE_DNODE,        "Offline dnode exists")
 
 // mnode-stable
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST,        "Stable already exists")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST,        "STable already exists")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST,            "Stable not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST,            "STable not exist")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC, "Stable confilct with topic")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC, "STable confilct with topic")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STBS,            "Too many stables")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB,              "Invalid stable name")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_OPTION,       "Invalid stable options")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_ALTER_OPTION, "Invalid stable alter options")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED,     "Stable option unchanged")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED,     "STable option unchanged")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ROW_BYTES,        "Invalid row bytes")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS,            "Too many tags")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREADY_EXIST,        "Tag already exists")
@@ -240,6 +242,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST,            "Tag does not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS,         "Too many columns")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST,     "Column already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST,         "Column does not exist")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC,"Field used by topic")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_SINGLE_STB_MODE_DB,       "Database is single stable mode")
 
 // mnode-infoSchema

source/util/src/tqueue.c

@@ -34,11 +34,11 @@ typedef struct STaosQnode {
 } STaosQnode;
 
 typedef struct STaosQueue {
-  STaosQnode *  head;
+  STaosQnode   *head;
-  STaosQnode *  tail;
+  STaosQnode   *tail;
-  STaosQueue *  next;     // for queue set
+  STaosQueue   *next;     // for queue set
-  STaosQset *   qset;     // for queue set
+  STaosQset    *qset;     // for queue set
-  void *        ahandle;  // for queue set
+  void         *ahandle;  // for queue set
   FItem         itemFp;
   FItems        itemsFp;
   TdThreadMutex mutex;
@@ -47,8 +47,8 @@ typedef struct STaosQueue {
 } STaosQueue;
 
 typedef struct STaosQset {
-  STaosQueue *  head;
+  STaosQueue   *head;
-  STaosQueue *  current;
+  STaosQueue   *current;
   TdThreadMutex mutex;
   tsem_t        sem;
   int32_t       numOfQueues;
@@ -86,7 +86,7 @@ void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp) {
 void taosCloseQueue(STaosQueue *queue) {
   if (queue == NULL) return;
   STaosQnode *pTemp;
-  STaosQset * qset;
+  STaosQset  *qset;
 
   taosThreadMutexLock(&queue->mutex);
   STaosQnode *pNode = queue->head;
@@ -282,6 +282,8 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) {
     *ppItem = pNode->item;
     num = 1;
     uTrace("item:%p is fetched", *ppItem);
+  } else {
+    *ppItem = NULL;
   }
 
   return num;

tests/pytest/util/dnodes.py

@@ -388,14 +388,14 @@ class TDDnode:
             if os.system(cmd) != 0:
                 tdLog.exit(cmd)
         else:
-            self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)\ntdDnodes.startWithoutSleep(%d)"%(self.index, self.index))
+            self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].deployed=1\ntdDnodes.dnodes[%d].logDir=\"%%s/sim/dnode%%d/log\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.dnodes[%d].cfgDir=\"%%s/sim/dnode%%d/cfg\"%%(tdDnodes.dnodes[%d].path,%d)\ntdDnodes.startWithoutSleep(%d)"%(self.index-1,self.index-1,self.index-1,self.index,self.index-1,self.index-1,self.index,self.index))
 
         self.running = 1
         tdLog.debug("dnode:%d is running with %s " % (self.index, cmd))
 
     def stop(self):
         if (not self.remoteIP == ""):
-            self.remoteExec(self.cfgDict, "tdDnodes.stop(%d)"%self.index)
+            self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1))
             tdLog.info("stop dnode%d"%self.index)
             return
         if self.valgrind == 0:
@@ -426,7 +426,7 @@ class TDDnode:
 
     def forcestop(self):
         if (not self.remoteIP == ""):
-            self.remoteExec(self.cfgDict, "tdDnodes.forcestop(%d)"%self.index)
+            self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].forcestop()"%(self.index-1,self.index-1))
             return
         if self.valgrind == 0:
             toBeKilled = "taosd"
@@ -497,46 +497,12 @@ class TDDnodes:
         self.killValgrind = 1
 
     def init(self, path, remoteIP = ""):
-        psCmd = "ps -ef|grep -w taosd| grep -v grep| grep -v defunct | awk '{print $2}'"
-        processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
-        while(processID):
-            killCmd = "kill -9 %s > /dev/null 2>&1" % processID
-            os.system(killCmd)
-            time.sleep(1)
-            processID = subprocess.check_output(
-                psCmd, shell=True).decode("utf-8")
-
-        if self.killValgrind == 1:
-            psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'"
-            processID = subprocess.check_output(psCmd, shell=True).decode("utf-8")
-            while(processID):
-                killCmd = "kill -9 %s > /dev/null 2>&1" % processID
-                os.system(killCmd)
-                time.sleep(1)
-                processID = subprocess.check_output(
-                    psCmd, shell=True).decode("utf-8")
-
         binPath = self.dnodes[0].getPath() + "/../../../"
         # tdLog.debug("binPath %s" % (binPath))
         binPath = os.path.realpath(binPath)
         # tdLog.debug("binPath real path %s" % (binPath))
 
-        # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath)
-        # tdLog.debug(cmd)
-        # os.system(cmd)
-
-        # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath)
-        # if os.system(cmd) != 0 :
-        #  tdLog.exit(cmd)
-        # tdLog.debug("execute %s" % (cmd))
-
-        # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath)
-        # if os.system(cmd) != 0 :
-        # tdLog.exit(cmd)
-        # tdLog.debug("execute %s" % (cmd))
-
         if path == "":
-            # self.path = os.path.expanduser('~')
             self.path = os.path.abspath(binPath + "../../")
         else:
             self.path = os.path.realpath(path)

tests/script/tsim/dnode/basic1.sim

@@ -183,15 +183,15 @@ if $rows != 15 then
 endi
 
 print =============== drop dnode
-sql drop dnode 2;
+#sql drop dnode 2;
-sql show dnodes;
+#sql show dnodes;
-if $rows != 1 then 
+#if $rows != 1 then 
-  return -1
+#  return -1
-endi
+#endi
 
-if $data00 != 1 then 
+#if $data00 != 1 then 
-  return -1
+#  return -1
-endi
+#endi
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT

tests/script/tsim/stable/column_drop.sim

@@ -118,6 +118,7 @@ if $data[0][5] != 5 then
   return -1
 endi
 if $data[0][6] != 101 then
+  print expect 101, actual: $data06
   return -1
 endi
 

tests/script/tsim/testsuit.sim

@@ -7,57 +7,57 @@
 #run tsim/table/basic1.sim
 #run tsim/trans/lossdata1.sim
 #run tsim/trans/create_db.sim
-##run tsim/stable/alter_metrics.sim
+#run tsim/stable/alter_metrics.sim
-##run tsim/stable/tag_modify.sim
+#run tsim/stable/tag_modify.sim
-##run tsim/stable/alter_comment.sim
+#run tsim/stable/alter_comment.sim
-##run tsim/stable/column_drop.sim
+#run tsim/stable/column_drop.sim
-##run tsim/stable/column_modify.sim
+#run tsim/stable/column_modify.sim
-##run tsim/stable/tag_rename.sim
+#run tsim/stable/tag_rename.sim
 #run tsim/stable/vnode3.sim
 #run tsim/stable/metrics.sim
-##run tsim/stable/alter_insert2.sim
+#run tsim/stable/alter_insert2.sim
 #run tsim/stable/show.sim
 #run tsim/stable/alter_import.sim
-##run tsim/stable/tag_add.sim
+#run tsim/stable/tag_add.sim
 #run tsim/stable/tag_drop.sim
 #run tsim/stable/column_add.sim
 #run tsim/stable/alter_count.sim
 #run tsim/stable/values.sim
-#run tsim/stable/dnode3.sim
+run tsim/stable/dnode3.sim
 #run tsim/stable/alter_insert1.sim
 #run tsim/stable/refcount.sim
 #run tsim/stable/disk.sim
-#run tsim/db/basic1.sim
+run tsim/db/basic1.sim
-#run tsim/db/basic3.sim
+run tsim/db/basic3.sim
-##run tsim/db/basic7.sim
+#run tsim/db/basic7.sim
-#run tsim/db/basic6.sim
+run tsim/db/basic6.sim
-#run tsim/db/create_all_options.sim
+run tsim/db/create_all_options.sim
-#run tsim/db/basic2.sim
+run tsim/db/basic2.sim
-#run tsim/db/error1.sim
+run tsim/db/error1.sim
-#run tsim/db/taosdlog.sim
+run tsim/db/taosdlog.sim
-#run tsim/db/alter_option.sim
+run tsim/db/alter_option.sim
-#run tsim/mnode/basic1.sim
+run tsim/mnode/basic1.sim
-#run tsim/mnode/basic3.sim
+run tsim/mnode/basic3.sim
-#run tsim/mnode/basic2.sim
+run tsim/mnode/basic2.sim
-#run tsim/parser/fourArithmetic-basic.sim
+run tsim/parser/fourArithmetic-basic.sim
-#run tsim/parser/groupby-basic.sim
+run tsim/parser/groupby-basic.sim
-#run tsim/snode/basic1.sim
+run tsim/snode/basic1.sim
-#run tsim/query/time_process.sim
+run tsim/query/time_process.sim
-#run tsim/query/stddev.sim
+run tsim/query/stddev.sim
-#run tsim/query/interval-offset.sim
+run tsim/query/interval-offset.sim
-#run tsim/query/charScalarFunction.sim
+run tsim/query/charScalarFunction.sim
-#run tsim/query/complex_select.sim
+run tsim/query/complex_select.sim
-#run tsim/query/explain.sim
+run tsim/query/explain.sim
-#run tsim/query/crash_sql.sim
+run tsim/query/crash_sql.sim
-#run tsim/query/diff.sim
+run tsim/query/diff.sim
-#run tsim/query/complex_limit.sim
+run tsim/query/complex_limit.sim
-#run tsim/query/complex_having.sim
+run tsim/query/complex_having.sim
-#run tsim/query/udf.sim
+run tsim/query/udf.sim
-#run tsim/query/complex_group.sim
+run tsim/query/complex_group.sim
-#run tsim/query/interval.sim
+run tsim/query/interval.sim
-#run tsim/query/session.sim
+run tsim/query/session.sim
-#run tsim/query/scalarFunction.sim
+run tsim/query/scalarFunction.sim
-##run tsim/query/scalarNull.sim
+#run tsim/query/scalarNull.sim
 run tsim/query/complex_where.sim
 run tsim/tmq/basic1.sim
 run tsim/tmq/basic4.sim

tests/system-test/0-others/taosShellNetChk.py

@@ -187,50 +187,51 @@ class TDTestCase:
         # stop taosd
         tdDnodes.stop(1)        
 
-        role   = 'server'
+        try:
-        if platform.system().lower() == 'windows':
+            role   = 'server'
-            taosCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\taos.exe -c ' + keyDict['c']
+            if platform.system().lower() == 'windows':
-            taosCmd = taosCmd.replace('\\','\\\\')
+                taosCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\taos.exe -c ' + keyDict['c']
-            taosCmd = taosCmd + ' -n ' + role
+                taosCmd = taosCmd.replace('\\','\\\\')
-        else:
+                taosCmd = taosCmd + ' -n ' + role
-            taosCmd = 'nohup ' + buildPath + '/build/bin/taos -c ' + keyDict['c']
+            else:
-            taosCmd = taosCmd + ' -n ' + role + ' > /dev/null 2>&1 &'
+                taosCmd = 'nohup ' + buildPath + '/build/bin/taos -c ' + keyDict['c']
-        print (taosCmd)
+                taosCmd = taosCmd + ' -n ' + role + ' > /dev/null 2>&1 &'
-        os.system(taosCmd)
+            print (taosCmd)
+            os.system(taosCmd)
 
-        pktLen = '2000'
+            pktLen = '2000'
-        pktNum = '10'
+            pktNum = '10'
-        role   = 'client'
+            role   = 'client'
-        if platform.system().lower() == 'windows':
+            if platform.system().lower() == 'windows':
-            taosCmd = buildPath + '\\build\\bin\\taos.exe -h 127.0.0.1 -c ' + keyDict['c']
+                taosCmd = buildPath + '\\build\\bin\\taos.exe -h 127.0.0.1 -c ' + keyDict['c']
-            taosCmd = taosCmd.replace('\\','\\\\')
+                taosCmd = taosCmd.replace('\\','\\\\')
-        else:
+            else:
-            taosCmd = buildPath + '/build/bin/taos -c ' + keyDict['c']
+                taosCmd = buildPath + '/build/bin/taos -c ' + keyDict['c']
-        taosCmd = taosCmd + ' -n ' + role + ' -l ' + pktLen + ' -N ' +  pktNum
+            taosCmd = taosCmd + ' -n ' + role + ' -l ' + pktLen + ' -N ' +  pktNum
-        print (taosCmd)
+            print (taosCmd)
-        child = taosExpect.spawn(taosCmd, timeout=3)
+            child = taosExpect.spawn(taosCmd, timeout=3)
-        i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=6)
+            i = child.expect([taosExpect.TIMEOUT, taosExpect.EOF], timeout=6)
 
-        if platform.system().lower() == 'windows':
+            if platform.system().lower() == 'windows':
-            retResult = child.before
+                retResult = child.before
-        else:
+            else:
-            retResult = child.before.decode()
+                retResult = child.before.decode()
-        print("expect() return code: %d, content:\n %s\n"%(i, retResult))
+            print("expect() return code: %d, content:\n %s\n"%(i, retResult))
-        #print(child.after.decode())
+            #print(child.after.decode())
-        if i == 0:
+            if i == 0:
-            tdLog.exit('taos -n server fail!')
+                tdLog.exit('taos -n server fail!')
-        
+            
-        expectString1 = 'response is received, size:' + pktLen
+            expectString1 = 'response is received, size:' + pktLen
-        expectSTring2 = pktNum + '/' + pktNum
+            expectSTring2 = pktNum + '/' + pktNum
-        if expectString1 in retResult and expectSTring2 in retResult:
+            if expectString1 in retResult and expectSTring2 in retResult:
-            tdLog.info("taos -n client success")
+                tdLog.info("taos -n client success")
-        else:
+            else:
-            tdLog.exit('taos -n client fail!')
+                tdLog.exit('taos -n client fail!')
-
+        finally:
-        if platform.system().lower() == 'windows':
+            if platform.system().lower() == 'windows':
-            os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9')
+                os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9')
-        else:
+            else:
-            os.system('pkill taos')
+                os.system('pkill taos')
 
     def stop(self):
         tdSql.close()

tests/system-test/0-others/taosdMonitor.py

@@ -8,6 +8,7 @@ import http.server
 import gzip
 import threading
 import json
+import pickle
 
 from util.log import *
 from util.sql import *
@@ -15,206 +16,203 @@ from util.cases import *
 from util.dnodes import *
 
 telemetryPort = '6043'
+serverPort = '7080'
+hostname = socket.gethostname()
 
+class RequestHandlerImpl(http.server.BaseHTTPRequestHandler):
+    hostPort = hostname + ":" + serverPort
 
-def telemetryInfoCheck(infoDict=''):
+    def telemetryInfoCheck(self, infoDict=''):
+        if  "ts" not in infoDict or len(infoDict["ts"]) == 0:
+            tdLog.exit("ts is null!")
 
-    hostname = socket.gethostname()
+        if "dnode_id" not in infoDict or infoDict["dnode_id"] != 1:
-    serverPort = 7080
+            tdLog.exit("dnode_id is null!")
 
-    if  "ts" not in infoDict or len(infoDict["ts"]) == 0:
+        if "dnode_ep" not in infoDict:
-        tdLog.exit("ts is null!")
+            tdLog.exit("dnode_ep is null!")
 
-    if "dnode_id" not in infoDict or infoDict["dnode_id"] != 1:
+        if "cluster_id" not in infoDict:
-        tdLog.exit("dnode_id is null!")
+            tdLog.exit("cluster_id is null!")
 
-    if "dnode_ep" not in infoDict:
+        if "protocol" not in infoDict or infoDict["protocol"] != 1:
-        tdLog.exit("dnode_ep is null!")
+            tdLog.exit("protocol is null!")
 
-    if "cluster_id" not in infoDict:
+        if "cluster_info" not in infoDict :
-        tdLog.exit("cluster_id is null!")
+            tdLog.exit("cluster_info is null!")
 
-    if "protocol" not in infoDict or infoDict["protocol"] != 1:
+        # cluster_info  ====================================
-        tdLog.exit("protocol is null!")
 
-    if "cluster_info" not in infoDict :
+        if "first_ep" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep"] == None:
-        tdLog.exit("cluster_info is null!")
+            tdLog.exit("first_ep is null!")
 
-    # cluster_info  ====================================
+        if "first_ep_dnode_id" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep_dnode_id"] != 1:
+            tdLog.exit("first_ep_dnode_id is null!")
 
-    if "first_ep" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep"] == None:
+        if "version" not in infoDict["cluster_info"] or infoDict["cluster_info"]["version"] == None:
-        tdLog.exit("first_ep is null!")
+            tdLog.exit("first_ep_dnode_id is null!")
-
-    if "first_ep_dnode_id" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep_dnode_id"] != 1:
-        tdLog.exit("first_ep_dnode_id is null!")
-
-    if "version" not in infoDict["cluster_info"] or infoDict["cluster_info"]["version"] == None:
-        tdLog.exit("first_ep_dnode_id is null!")
-    
-    if "master_uptime" not in infoDict["cluster_info"] or infoDict["cluster_info"]["master_uptime"] == None:
-        tdLog.exit("master_uptime is null!")
-
-    if "monitor_interval" not in infoDict["cluster_info"] or infoDict["cluster_info"]["monitor_interval"] !=5:
-        tdLog.exit("monitor_interval is null!")
-
-    if "vgroups_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_total"] < 0:
-        tdLog.exit("vgroups_total is null!")
-
-    if "vgroups_alive" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_alive"] < 0:
-        tdLog.exit("vgroups_alive is null!")
-
-    if "connections_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["connections_total"] < 0 :
-        tdLog.exit("connections_total is null!")
-
-    if "dnodes" not in infoDict["cluster_info"] or infoDict["cluster_info"]["dnodes"] == None :
-        tdLog.exit("dnodes is null!")
-    
-    dnodes_info = { "dnode_id": 1,"dnode_ep": f"{hostname}:{serverPort}","status":"ready"}
-    
-    for k ,v in dnodes_info.items():
-        if k not in infoDict["cluster_info"]["dnodes"][0] or v != infoDict["cluster_info"]["dnodes"][0][k] :
-            tdLog.exit("dnodes info is null!")
-    
-    mnodes_info = { "mnode_id":1, "mnode_ep":f"{hostname}:{serverPort}","role": "leader" }
-
-    for k ,v in mnodes_info.items():
-        if k not in infoDict["cluster_info"]["mnodes"][0] or v != infoDict["cluster_info"]["mnodes"][0][k] :
-            tdLog.exit("mnodes info is null!")
-
-    # vgroup_infos  ====================================
-
-    if "vgroup_infos" not in infoDict or infoDict["vgroup_infos"]== None:
-        tdLog.exit("vgroup_infos is null!")
-    
-    vgroup_infos_nums = len(infoDict["vgroup_infos"])
-
-    for  index in range(vgroup_infos_nums):
-        if "vgroup_id" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vgroup_id"]<0:
-            tdLog.exit("vgroup_id is null!")
-        if "database_name" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["database_name"]) < 0:
-            tdLog.exit("database_name is null!")
-        if "tables_num" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["tables_num"]!= 0:
-            tdLog.exit("tables_num is null!")
-        if "status" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["status"]) < 0 :
-            tdLog.exit("status is null!")
-        if "vnodes" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vnodes"] ==None :
-            tdLog.exit("vnodes is null!")
-        if "dnode_id" not in infoDict["vgroup_infos"][index]["vnodes"][0] or infoDict["vgroup_infos"][index]["vnodes"][0]["dnode_id"] < 0 :
-            tdLog.exit("vnodes is null!")
-
-    # grant_info  ====================================
-
-    if "grant_info" not in infoDict or infoDict["grant_info"]== None:
-        tdLog.exit("grant_info is null!")
-
-    if "expire_time" not in infoDict["grant_info"] or not infoDict["grant_info"]["expire_time"] > 0:
-        tdLog.exit("expire_time is null!")
-
-    if "timeseries_used" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_used"] > 0:
-        tdLog.exit("timeseries_used is null!")
-
-    if "timeseries_total" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_total"] > 0:
-        tdLog.exit("timeseries_total is null!")
         
-    # dnode_info  ====================================
+        if "master_uptime" not in infoDict["cluster_info"] or infoDict["cluster_info"]["master_uptime"] == None:
+            tdLog.exit("master_uptime is null!")
 
-    if "dnode_info" not in infoDict or infoDict["dnode_info"]== None:
+        if "monitor_interval" not in infoDict["cluster_info"] or infoDict["cluster_info"]["monitor_interval"] !=5:
-        tdLog.exit("dnode_info is null!")
+            tdLog.exit("monitor_interval is null!")
 
-    dnode_infos =  ['uptime', 'cpu_engine', 'cpu_system', 'cpu_cores', 'mem_engine', 'mem_system', 'mem_total', 'disk_engine', 
+        if "vgroups_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_total"] < 0:
-    'disk_used', 'disk_total', 'net_in', 'net_out', 'io_read', 'io_write', 'io_read_disk', 'io_write_disk', 'req_select', 
+            tdLog.exit("vgroups_total is null!")
-    'req_select_rate', 'req_insert', 'req_insert_success', 'req_insert_rate', 'req_insert_batch', 'req_insert_batch_success',
-     'req_insert_batch_rate', 'errors', 'vnodes_num', 'masters', 'has_mnode', 'has_qnode', 'has_snode', 'has_bnode']
-    for elem in dnode_infos:
-        if elem not in infoDict["dnode_info"] or  infoDict["dnode_info"][elem] < 0:
-            tdLog.exit(f"{elem} is null!")
 
-    # dnode_info  ====================================
+        if "vgroups_alive" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_alive"] < 0:
+            tdLog.exit("vgroups_alive is null!")
 
-    if "disk_infos" not in infoDict or infoDict["disk_infos"]== None:
+        if "connections_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["connections_total"] < 0 :
-        tdLog.exit("disk_infos is null!")
+            tdLog.exit("connections_total is null!")
-    
-    # bug for data_dir
-    if "datadir" not in infoDict["disk_infos"] or len(infoDict["disk_infos"]["datadir"]) <=0 :
-        tdLog.exit("datadir is null!")
 
-    if "name" not in infoDict["disk_infos"]["datadir"][0] or len(infoDict["disk_infos"]["datadir"][0]["name"]) <= 0:
+        if "dnodes" not in infoDict["cluster_info"] or infoDict["cluster_info"]["dnodes"] == None :
-        tdLog.exit("name is null!")
+            tdLog.exit("dnodes is null!")
+        
+        dnodes_info = { "dnode_id": 1,"dnode_ep": self.hostPort,"status":"ready"}
+        
+        for k ,v in dnodes_info.items():
+            if k not in infoDict["cluster_info"]["dnodes"][0] or v != infoDict["cluster_info"]["dnodes"][0][k] :
+                tdLog.exit("dnodes info is null!")
+        
+        mnodes_info = { "mnode_id":1, "mnode_ep": self.hostPort,"role": "leader" }
 
-    if "level" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["level"] < 0:
+        for k ,v in mnodes_info.items():
-        tdLog.exit("level is null!")
+            if k not in infoDict["cluster_info"]["mnodes"][0] or v != infoDict["cluster_info"]["mnodes"][0][k] :
+                tdLog.exit("mnodes info is null!")
 
-    if "avail" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["avail"] <= 0:
+        # vgroup_infos  ====================================
-        tdLog.exit("avail is null!")
 
-    if "used" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["used"] <= 0:
+        if "vgroup_infos" not in infoDict or infoDict["vgroup_infos"]== None:
-        tdLog.exit("used is null!")
+            tdLog.exit("vgroup_infos is null!")
+        
+        vgroup_infos_nums = len(infoDict["vgroup_infos"])
 
-    if "total" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["total"] <= 0:
+        for  index in range(vgroup_infos_nums):
-        tdLog.exit("total is null!")
+            if "vgroup_id" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vgroup_id"]<0:
+                tdLog.exit("vgroup_id is null!")
+            if "database_name" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["database_name"]) < 0:
+                tdLog.exit("database_name is null!")
+            if "tables_num" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["tables_num"]!= 0:
+                tdLog.exit("tables_num is null!")
+            if "status" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["status"]) < 0 :
+                tdLog.exit("status is null!")
+            if "vnodes" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vnodes"] ==None :
+                tdLog.exit("vnodes is null!")
+            if "dnode_id" not in infoDict["vgroup_infos"][index]["vnodes"][0] or infoDict["vgroup_infos"][index]["vnodes"][0]["dnode_id"] < 0 :
+                tdLog.exit("vnodes is null!")
+
+        # grant_info  ====================================
+
+        if "grant_info" not in infoDict or infoDict["grant_info"]== None:
+            tdLog.exit("grant_info is null!")
+
+        if "expire_time" not in infoDict["grant_info"] or not infoDict["grant_info"]["expire_time"] > 0:
+            tdLog.exit("expire_time is null!")
+
+        if "timeseries_used" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_used"] > 0:
+            tdLog.exit("timeseries_used is null!")
+
+        if "timeseries_total" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_total"] > 0:
+            tdLog.exit("timeseries_total is null!")
+            
+        # dnode_info  ====================================
+
+        if "dnode_info" not in infoDict or infoDict["dnode_info"]== None:
+            tdLog.exit("dnode_info is null!")
+
+        dnode_infos =  ['uptime', 'cpu_engine', 'cpu_system', 'cpu_cores', 'mem_engine', 'mem_system', 'mem_total', 'disk_engine', 
+        'disk_used', 'disk_total', 'net_in', 'net_out', 'io_read', 'io_write', 'io_read_disk', 'io_write_disk', 'req_select', 
+        'req_select_rate', 'req_insert', 'req_insert_success', 'req_insert_rate', 'req_insert_batch', 'req_insert_batch_success',
+        'req_insert_batch_rate', 'errors', 'vnodes_num', 'masters', 'has_mnode', 'has_qnode', 'has_snode', 'has_bnode']
+        for elem in dnode_infos:
+            if elem not in infoDict["dnode_info"] or  infoDict["dnode_info"][elem] < 0:
+                tdLog.exit(f"{elem} is null!")
+
+        # dnode_info  ====================================
+
+        if "disk_infos" not in infoDict or infoDict["disk_infos"]== None:
+            tdLog.exit("disk_infos is null!")
+        
+        # bug for data_dir
+        if "datadir" not in infoDict["disk_infos"] or len(infoDict["disk_infos"]["datadir"]) <=0 :
+            tdLog.exit("datadir is null!")
+
+        if "name" not in infoDict["disk_infos"]["datadir"][0] or len(infoDict["disk_infos"]["datadir"][0]["name"]) <= 0:
+            tdLog.exit("name is null!")
+
+        if "level" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["level"] < 0:
+            tdLog.exit("level is null!")
+
+        if "avail" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["avail"] <= 0:
+            tdLog.exit("avail is null!")
+
+        if "used" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["used"] <= 0:
+            tdLog.exit("used is null!")
+
+        if "total" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["total"] <= 0:
+            tdLog.exit("total is null!")
 
 
-    if "logdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["logdir"]== None:
+        if "logdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["logdir"]== None:
-        tdLog.exit("logdir is null!")
+            tdLog.exit("logdir is null!")
 
-    if "name" not in infoDict["disk_infos"]["logdir"] or len(infoDict["disk_infos"]["logdir"]["name"]) <= 0:
+        if "name" not in infoDict["disk_infos"]["logdir"] or len(infoDict["disk_infos"]["logdir"]["name"]) <= 0:
-        tdLog.exit("name is null!")
+            tdLog.exit("name is null!")
 
-    if "avail" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["avail"] <= 0:
+        if "avail" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["avail"] <= 0:
-        tdLog.exit("avail is null!")
+            tdLog.exit("avail is null!")
 
-    if "used" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["used"] <= 0:
+        if "used" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["used"] <= 0:
-        tdLog.exit("used is null!")
+            tdLog.exit("used is null!")
 
-    if "total" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["total"] <= 0:
+        if "total" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["total"] <= 0:
-        tdLog.exit("total is null!")
+            tdLog.exit("total is null!")
 
+        if "tempdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["tempdir"]== None:
+            tdLog.exit("tempdir is null!")
+
+        if "name" not in infoDict["disk_infos"]["tempdir"] or len(infoDict["disk_infos"]["tempdir"]["name"]) <= 0:
+            tdLog.exit("name is null!")
+
+        if "avail" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["avail"] <= 0:
+            tdLog.exit("avail is null!")
+
+        if "used" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["used"] <= 0:
+            tdLog.exit("used is null!")
+
+        if "total" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["total"] <= 0:
+            tdLog.exit("total is null!")
+
+
+        # log_infos  ====================================
+        
+        if "log_infos" not in infoDict or infoDict["log_infos"]== None:
+            tdLog.exit("log_infos is null!")
+
+        if "logs" not in infoDict["log_infos"] or len(infoDict["log_infos"]["logs"])!= 10:
+            tdLog.exit("logs is null!")
+
+        if "ts" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 10:
+            tdLog.exit("ts is null!")
+
+        if "level" not in infoDict["log_infos"]["logs"][0] or infoDict["log_infos"]["logs"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]:
+            tdLog.exit("level is null!")
+
+        if "content" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 1:
+            tdLog.exit("content is null!")
+
+        if "summary" not in infoDict["log_infos"] or len(infoDict["log_infos"]["summary"])!= 4:
+            tdLog.exit("summary is null!")
+
+        
+        if "total" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["total"] < 0 :
+            tdLog.exit("total is null!")
+
+        if "level" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]:
+            tdLog.exit("level is null!")
      
-
-    if "tempdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["tempdir"]== None:
-        tdLog.exit("tempdir is null!")
-
-    if "name" not in infoDict["disk_infos"]["tempdir"] or len(infoDict["disk_infos"]["tempdir"]["name"]) <= 0:
-        tdLog.exit("name is null!")
-
-    if "avail" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["avail"] <= 0:
-        tdLog.exit("avail is null!")
-
-    if "used" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["used"] <= 0:
-        tdLog.exit("used is null!")
-
-    if "total" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["total"] <= 0:
-        tdLog.exit("total is null!")
-
-
-    # log_infos  ====================================
-    
-    if "log_infos" not in infoDict or infoDict["log_infos"]== None:
-        tdLog.exit("log_infos is null!")
-
-    if "logs" not in infoDict["log_infos"] or len(infoDict["log_infos"]["logs"])!= 10:
-        tdLog.exit("logs is null!")
-
-    if "ts" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 10:
-        tdLog.exit("ts is null!")
-
-    if "level" not in infoDict["log_infos"]["logs"][0] or infoDict["log_infos"]["logs"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]:
-        tdLog.exit("level is null!")
-
-    if "content" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 1:
-        tdLog.exit("content is null!")
-
-    if "summary" not in infoDict["log_infos"] or len(infoDict["log_infos"]["summary"])!= 4:
-        tdLog.exit("summary is null!")
-
-    
-    if "total" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["total"] < 0 :
-        tdLog.exit("total is null!")
-
-    if "level" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]:
-        tdLog.exit("level is null!")
-    
-class RequestHandlerImpl(http.server.BaseHTTPRequestHandler):    
     def do_GET(self):
         """
         process GET request
@@ -245,17 +243,23 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler):
         infoDict = json.loads(plainText)
         #print("================")
         # print(infoDict)
-        telemetryInfoCheck(infoDict)
+        self.telemetryInfoCheck(infoDict)
 
         # 4. shutdown the server and exit case 
-        assassin = threading.Thread(target=httpServer.shutdown)
+        assassin = threading.Thread(target=self.server.shutdown)
         assassin.daemon = True
         assassin.start()
         print ("==== shutdown http server ====")
 
 class TDTestCase:
-    hostname = socket.gethostname()
+    global hostname
-    serverPort = '7080'
+    global serverPort
+    if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""):
+        try:
+            config = eval(tdDnodes.dnodes[0].remoteIP )
+            hostname = config["host"]
+        except Exception:
+            hostname = tdDnodes.dnodes[0].remoteIP
     rpcDebugFlagVal = '143'
     clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
     clientCfgDict["serverPort"]    = serverPort
@@ -291,21 +295,19 @@ class TDTestCase:
         sql = "create database db3 vgroups " + vgroups
         tdSql.query(sql)
 
-        # loop to wait request
+        # create http server: bing ip/port , and  request processor
-        httpServer.serve_forever()
+        if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""):
+            RequestHandlerImplStr = base64.b64encode(pickle.dumps(RequestHandlerImpl)).decode()
+            cmdStr = "import pickle\nimport http\nRequestHandlerImpl=pickle.loads(base64.b64decode(\"%s\".encode()))\nclass NewRequestHandlerImpl(RequestHandlerImpl):\n    hostPort = \'%s\'\nhttp.server.HTTPServer((\"\", %s), NewRequestHandlerImpl).serve_forever()"%(RequestHandlerImplStr,hostname+":"+serverPort,telemetryPort)
+            tdDnodes.dnodes[0].remoteExec({}, cmdStr)
+        else:
+            serverAddress = ("", int(telemetryPort))
+            http.server.HTTPServer(serverAddress, RequestHandlerImpl).serve_forever()
 
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")
 
-# create http server: bing ip/port , and  request processor
-serverAddress = ("", int(telemetryPort))
-httpServer = http.server.HTTPServer(serverAddress, RequestHandlerImpl)
-
 tdCases.addLinux(__file__, TDTestCase())
 tdCases.addWindows(__file__, TDTestCase())
 
-
-
-
-

tests/system-test/0-others/telemetry.py

@@ -8,6 +8,7 @@ import http.server
 import gzip
 import threading
 import json
+import pickle
 
 from util.log import *
 from util.sql import *
@@ -136,13 +137,19 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler):
         telemetryInfoCheck(infoDict)
 
         # 4. shutdown the server and exit case 
-        assassin = threading.Thread(target=httpServer.shutdown)
+        assassin = threading.Thread(target=self.server.shutdown)
         assassin.daemon = True
         assassin.start()
         print ("==== shutdown http server ====")
 
 class TDTestCase:
     hostname = socket.gethostname()
+    if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""):
+        try:
+            config = eval(tdDnodes.dnodes[0].remoteIP)
+            hostname = config["host"]
+        except Exception:
+            hostname = tdDnodes.dnodes[0].remoteIP
     serverPort = '7080'
     rpcDebugFlagVal = '143'
     clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
@@ -177,17 +184,20 @@ class TDTestCase:
         sql = "create database db3 vgroups " + vgroups
         tdSql.query(sql)
 
-        # loop to wait request
+        # create http server: bing ip/port , and  request processor
-        httpServer.serve_forever()
+        if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""):
+            RequestHandlerImplStr = base64.b64encode(pickle.dumps(RequestHandlerImpl)).decode()
+            telemetryInfoCheckStr = base64.b64encode(pickle.dumps(telemetryInfoCheck)).decode()
+            cmdStr = "import pickle\nimport http\ntelemetryInfoCheck=pickle.loads(base64.b64decode(\"%s\".encode()))\nRequestHandlerImpl=pickle.loads(base64.b64decode(\"%s\".encode()))\nhttp.server.HTTPServer((\"\", %d), RequestHandlerImpl).serve_forever()"%(telemetryInfoCheckStr,RequestHandlerImplStr,int(telemetryPort))
+            tdDnodes.dnodes[0].remoteExec({}, cmdStr)
+        else:
+            serverAddress = ("", int(telemetryPort))
+            http.server.HTTPServer(serverAddress, RequestHandlerImpl).serve_forever()
 
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")
 
-# create http server: bing ip/port , and  request processor
-serverAddress = ("", int(telemetryPort))
-httpServer = http.server.HTTPServer(serverAddress, RequestHandlerImpl)
-
 tdCases.addLinux(__file__, TDTestCase())
 tdCases.addWindows(__file__, TDTestCase())
 

tests/system-test/0-others/udf_create.py

@@ -9,6 +9,8 @@ from util.sql import *
 from util.cases import *
 from util.dnodes import *
 import subprocess
+# import win32gui
+# import threading
 
 class TDTestCase:
 
@@ -533,8 +535,17 @@ class TDTestCase:
         return udf1_sqls ,udf2_sqls
 
 
+    # def checkRunTimeError(self):
+    #     while 1:
+    #         time.sleep(1)
+    #         hwnd = win32gui.FindWindow(None, "Microsoft Visual C++ Runtime Library")
+    #         if hwnd:
+    #             os.system("TASKKILL /F /IM udfd.exe")
 
     def unexpected_create(self):
+        # if (platform.system().lower() == 'windows' and tdDnodes.dnodes[0].remoteIP == ""):
+        #     checkErrorThread = threading.Thread(target=self.checkRunTimeError,daemon=True)
+        #     checkErrorThread.start()
 
         tdLog.info(" create function with out bufsize ")
         tdSql.query("drop function udf1 ")

tests/system-test/1-insert/create_table_comment.py

@@ -0,0 +1,113 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import random
+import string
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def get_long_name(self, length, mode="mixed"):
+        """
+        generate long name
+        mode could be numbers/letters/letters_mixed/mixed
+        """
+        if mode == "numbers":
+            population = string.digits
+        elif mode == "letters":
+            population = string.ascii_letters.lower()
+        elif mode == "letters_mixed":
+            population = string.ascii_letters.upper() + string.ascii_letters.lower()
+        else:
+            population = string.ascii_letters.lower() + string.digits
+        return "".join(random.choices(population, k=length))
+
+    def __create_tb(self,dbname,stbname,tbname,comment):
+        tdSql.execute(f'create database if not exists {dbname}')
+        tdSql.execute(f'use {dbname}')
+        tdSql.execute(
+            f'create table {stbname} (ts timestamp,c0 int) tags(t0 int) ')
+        tdSql.execute(
+            f'create table {tbname} using {stbname} tags(1) comment "{comment}"')
+    def __create_normaltb(self,dbname,tbname,comment):
+        tdSql.execute(f'create database if not exists {dbname}')
+        tdSql.execute(f'use {dbname}')
+        tdSql.execute(
+            f'create table {tbname} (ts timestamp,c0 int) comment "{comment}"')
+
+    def check_comment(self):
+        dbname = self.get_long_name(length=10, mode="letters")
+        ntbname = self.get_long_name(length=5, mode="letters")
+
+        # create normal table with comment
+        comment = self.get_long_name(length=10, mode="letters")
+        self.__create_normaltb(dbname,ntbname,comment)
+        ntb_kv_list = tdSql.getResult("show tables")
+        print(ntb_kv_list)
+        tdSql.checkEqual(ntb_kv_list[0][8], comment)
+        tdSql.error('alter table {ntbname} comment "test1"')
+        tdSql.execute(f'drop database {dbname}')
+
+        # max length(1024)
+        comment = self.get_long_name(length=1024, mode="letters")
+        self.__create_normaltb(dbname,ntbname,comment)
+        ntb_kv_list = tdSql.getResult("show tables")
+        tdSql.checkEqual(ntb_kv_list[0][8], comment)
+        tdSql.execute(f'drop database {dbname}')
+
+        # error overlength
+        comment = self.get_long_name(length=1025, mode="letters")
+        tdSql.execute(f'create database if not exists {dbname}')
+        tdSql.execute(f'use {dbname}')
+        tdSql.error(f"create table ntb (ts timestamp,c0 int) comment '{comment}'")
+        tdSql.execute(f'drop database {dbname}')
+
+        # create child table with comment
+        comment = self.get_long_name(length=10, mode="letters")
+        stbname = self.get_long_name(length=5, mode="letters")
+        tbname = self.get_long_name(length=3, mode="letters")
+        self.__create_tb(dbname,stbname,tbname,comment)
+        ntb_kv_list = tdSql.getResult("show tables")
+        tdSql.checkEqual(ntb_kv_list[0][8], comment)
+        tdSql.error(f'alter table {tbname} comment "test1"')
+        tdSql.execute(f'drop database {dbname}')
+
+        # max length 1024
+        comment = self.get_long_name(length=1024, mode="letters")
+        self.__create_tb(dbname,ntbname,comment)
+        ntb_kv_list = tdSql.getResult("show tables")
+        tdSql.checkEqual(ntb_kv_list[0][8], comment)
+        tdSql.execute(f'drop database {dbname}')
+
+        # error overlength
+        comment = self.get_long_name(length=1025, mode="letters")
+        tdSql.execute(f'create database if not exists {dbname}')
+        tdSql.execute(f'use {dbname}')
+        tdSql.execute(f"create table stb (ts timestamp,c0 int) tags(t0 int)")
+        tdSql.error(f'create table stb_1 us stb tags(1) comment "{comment}"')
+        tdSql.execute(f'drop database {dbname}')
+
+    def run(self):
+        self.check_comment()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/2-query/csum.py

@@ -240,7 +240,7 @@ class TDTestCase:
         tdSql.error("select csum(c1)   t1")         # no from
         tdSql.error("select csum( c1 )  from ")     # no table_expr
         # tdSql.error(self.csum_query_form(col="st1"))    # tag col
-        tdSql.error(self.csum_query_form(col=1))        # col is a value
+        # tdSql.error(self.csum_query_form(col=1))        # col is a value
         tdSql.error(self.csum_query_form(col="'c1'"))   # col is a string
         tdSql.error(self.csum_query_form(col=None))     # col is NULL 1
         tdSql.error(self.csum_query_form(col="NULL"))   # col is NULL 2
@@ -407,6 +407,14 @@ class TDTestCase:
         tdDnodes.start(index)
         self.csum_current_query()
         self.csum_error_query()
+        tdSql.query("select csum(1) from t1 ")
+        tdSql.checkRows(7)
+        tdSql.checkData(0,0,1)
+        tdSql.checkData(1,0,2)
+        tdSql.checkData(2,0,3)
+        tdSql.checkData(3,0,4)
+        tdSql.query("select csum(abs(c1))+2 from t1 ")
+        tdSql.checkRows(4)
 
     def run(self):
         import traceback

tests/system-test/2-query/elapsed.py

@@ -141,7 +141,7 @@ class TDTestCase:
         tablenames = ["sub_table1_1","sub_table1_2","sub_table1_3","sub_table2_1","sub_table2_2","sub_table2_3","regular_table_1","regular_table_2","regular_table_3"]
 
         abnormal_list = ["()","(NULL)","(*)","(abc)","( , )","(NULL,*)","( ,NULL)","(%)","(+)","(*,)","(*, /)","(ts,*)" "(ts,tbname*10)","(ts,tagname)",
-        "(ts,2d+3m-2s,NULL)","(ts+1d,10s)","(ts+10d,NULL)" ,"(ts,now -1m%1d)","(ts+10d)","(ts+10d,_c0)","(ts+10d,)","(ts,%)","(ts, , m)","(ts,abc)","(ts,/)","(ts,*)","(ts,1s,100)",
+        "(ts,2d+3m-2s,NULL)","(ts+10d,NULL)" ,"(ts,now -1m%1d)","(ts+10d,_c0)","(ts+10d,)","(ts,%)","(ts, , m)","(ts,abc)","(ts,/)","(ts,*)","(ts,1s,100)",
         "(ts,1s,abc)","(ts,1s,_c0)","(ts,1s,*)","(ts,1s,NULL)","(ts,,_c0)","(ts,tbname,ts)","(ts,0,tbname)","('2021-11-18 00:00:10')","('2021-11-18 00:00:10', 1s)",
         "('2021-11-18T00:00:10+0800', '1s')","('2021-11-18T00:00:10Z', '1s')","('2021-11-18T00:00:10+0800', 10000000d,)","('ts', ,2021-11-18T00:00:10+0800, )"]
         

tests/system-test/2-query/function_stateduration.py

@@ -85,8 +85,8 @@ class TDTestCase:
             "select stateduration(c1 ,'GT','*',1s) from t1",
             "select stateduration(c1 ,'GT',ts,1s) from t1",
             "select stateduration(c1 ,'GT',max(c1),1s) from t1",
-            "select stateduration(abs(c1) ,'GT',1,1s) from t1",
+            # "select stateduration(abs(c1) ,'GT',1,1s) from t1",
-            "select stateduration(c1+2 ,'GT',1,1s) from t1",
+            # "select stateduration(c1+2 ,'GT',1,1s) from t1",
             "select stateduration(c1 ,'GT',1,1u) from t1",
             "select stateduration(c1 ,'GT',1,now) from t1",
             "select stateduration(c1 ,'GT','1',1s) from t1",
@@ -323,6 +323,11 @@ class TDTestCase:
         tdSql.checkData(0, 0, None)
         tdSql.checkData(1, 0, 0.000000000)
         tdSql.checkData(3, 0, -86404.000000000)
+
+        tdSql.query("select stateduration(abs(c1) ,'GT',1,1s) from t1")
+        tdSql.checkRows(12)
+        tdSql.query("select stateduration(c1+2 ,'GT',1,1s) from t1")
+        tdSql.checkRows(12)
         
 
         # bug for stable

tests/system-test/2-query/hyperloglog.py

@@ -223,7 +223,7 @@ class TDTestCase:
         tdLog.printNoPrefix("===step 0: err case, must return err")
         tdSql.error( "select hyperloglog() from ct1" )
         tdSql.error( "select hyperloglog(c1, c2) from ct2" )
-        tdSql.error( "select hyperloglog(1) from ct2" )
+        # tdSql.error( "select hyperloglog(1) from ct2" )
         tdSql.error( f"select hyperloglog({NUM_COL[0]}, {NUM_COL[1]}) from ct4" )
         tdSql.error( ''' select hyperloglog(['c1 + c1', 'c1 + c2', 'c1 + c3', 'c1 + c4', 'c1 + c5', 'c1 + c6', 'c1 + c7', 'c1 + c8', 'c1 + c9', 'c1 + c10'])
                     from ct1