Merge branch '3.0' into cpwu/3.0

include/common/tdatablock.h

@@ -236,7 +236,7 @@ void        blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t*
                                 int8_t needCompress);
 const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
 
-void blockDebugShowData(const SArray* dataBlocks, const char* flag);
+void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag);
 // for debug
 char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf);
 

include/common/tdataformat.h

@@ -78,7 +78,6 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag);
 int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag);
 int32_t tTagToValArray(const STag *pTag, SArray **ppArray);
 void    debugPrintSTag(STag *pTag, const char *tag, int32_t ln);  // TODO: remove
-void    debugCheckTags(STag *pTag);                               // TODO: remove
 
 // STRUCT =================
 struct STColumn {

include/common/tmsg.h

@@ -1886,7 +1886,7 @@ typedef struct SVCreateStbReq {
   int8_t         rollup;
   SSchemaWrapper schemaRow;
   SSchemaWrapper schemaTag;
-  SRSmaParam     pRSmaParam;
+  SRSmaParam     rsmaParam;
 } SVCreateStbReq;
 
 int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq);

include/libs/nodes/plannodes.h

@@ -24,6 +24,8 @@ extern "C" {
 #include "querynodes.h"
 #include "tname.h"
 
+#define SLOT_NAME_LEN TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN
+
 typedef struct SLogicNode {
   ENodeType          type;
   SNodeList*         pTargets;  // SColumnNode
@@ -74,8 +76,8 @@ typedef struct SScanLogicNode {
   int16_t       tsColId;
   double        filesFactor;
   SArray*       pSmaIndexes;
-  SNodeList*    pPartTags;
+  SNodeList*    pGroupTags;
-  bool          partSort;
+  bool          groupSort;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -100,6 +102,7 @@ typedef struct SProjectLogicNode {
 typedef struct SIndefRowsFuncLogicNode {
   SLogicNode node;
   SNodeList* pFuncs;
+  bool       isTailFunc;
 } SIndefRowsFuncLogicNode;
 
 typedef struct SInterpFuncLogicNode {
@@ -138,6 +141,7 @@ typedef struct SMergeLogicNode {
   SNodeList* pInputs;
   int32_t    numOfChannels;
   int32_t    srcGroupId;
+  bool       groupSort;
 } SMergeLogicNode;
 
 typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType;
@@ -184,6 +188,7 @@ typedef struct SFillLogicNode {
 typedef struct SSortLogicNode {
   SLogicNode node;
   SNodeList* pSortKeys;
+  bool       groupSort;
 } SSortLogicNode;
 
 typedef struct SPartitionLogicNode {
@@ -230,6 +235,7 @@ typedef struct SSlotDescNode {
   bool      reserve;
   bool      output;
   bool      tag;
+  char      name[SLOT_NAME_LEN];
 } SSlotDescNode;
 
 typedef struct SDataBlockDescNode {
@@ -279,7 +285,8 @@ typedef struct STableScanPhysiNode {
   double         ratio;
   int32_t        dataRequired;
   SNodeList*     pDynamicScanFuncs;
-  SNodeList*     pPartitionTags;
+  SNodeList*     pGroupTags;
+  bool           groupSort;
   int64_t        interval;
   int64_t        offset;
   int64_t        sliding;
@@ -353,6 +360,7 @@ typedef struct SMergePhysiNode {
   SNodeList* pTargets;
   int32_t    numOfChannels;
   int32_t    srcGroupId;
+  bool       groupSort;
 } SMergePhysiNode;
 
 typedef struct SWinodwPhysiNode {

include/libs/nodes/querynodes.h

@@ -259,6 +259,7 @@ typedef struct SSelectStmt {
   bool        hasTailFunc;
   bool        hasInterpFunc;
   bool        hasLastRowFunc;
+  bool        groupSort;
 } SSelectStmt;
 
 typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;

include/libs/transport/trpc.h

@@ -69,7 +69,7 @@ typedef struct SRpcMsg {
 } SRpcMsg;
 
 typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf);
-typedef bool (*RpcRfp)(int32_t code);
+typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType);
 
 typedef struct SRpcInit {
   char     localFqdn[TSDB_FQDN_LEN];

include/os/osSystem.h

@@ -29,6 +29,9 @@ extern "C" {
     #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID
 #endif
 
+#define TAOS_CONSOLE_PROMPT_HEADER "taos> "
+#define TAOS_CONSOLE_PROMPT_CONTINUE "   -> "
+
 typedef struct TdCmd *TdCmdPtr;
 
 TdCmdPtr taosOpenCmd(const char* cmd);

source/client/src/clientEnv.c

@@ -84,9 +84,12 @@ void closeTransporter(STscObj *pTscObj) {
   rpcClose(pTscObj->pAppInfo->pTransporter);
 }
 
-static bool clientRpcRfp(int32_t code) {
+static bool clientRpcRfp(int32_t code, tmsg_t msgType) {
   if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
       code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
+    if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) {
+      return false;
+    }
     return true;
   } else {
     return false;

source/common/src/tdatablock.c

@@ -1605,7 +1605,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
   return buf;
 }
 
-void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
+void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) {
   char    pBuf[128] = {0};
   int32_t sz = taosArrayGetSize(dataBlocks);
   for (int32_t i = 0; i < sz; i++) {
@@ -1613,7 +1613,7 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) {
     size_t       numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
 
     int32_t rows = pDataBlock->info.rows;
-    printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId);
+    printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
     for (int32_t j = 0; j < rows; j++) {
       printf("%s |", flag);
       for (int32_t k = 0; k < numOfCols; k++) {

source/common/src/tdataformat.c

@@ -862,21 +862,6 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) {
   printf("\n");
 }
 
-void debugCheckTags(STag *pTag) {
-  switch (pTag->flags) {
-    case 0x0:
-    case 0x20:
-    case 0x40:
-    case 0x60:
-      break;
-    default:
-      ASSERT(0);
-  }
-
-  ASSERT(pTag->nTag <= 128 && pTag->nTag >= 0);
-  ASSERT(pTag->ver <= 512 && pTag->ver >= 0);  // temp condition for pTag->ver
-}
-
 static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
   int32_t n = 0;
 
@@ -999,7 +984,6 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) {
   debugPrintSTag(*ppTag, __func__, __LINE__);
 #endif
 
-  debugCheckTags(*ppTag);  // TODO: remove this line after debug
   return code;
 
 _err:

source/common/src/tmsg.c

@@ -4763,7 +4763,7 @@ int tEncodeSVCreateStbReq(SEncoder *pCoder, const SVCreateStbReq *pReq) {
   if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1;
   if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
   if (pReq->rollup) {
-    if (tEncodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1;
+    if (tEncodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1;
   }
 
   tEndEncode(pCoder);
@@ -4779,7 +4779,7 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
   if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1;
   if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
   if (pReq->rollup) {
-    if (tDecodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1;
+    if (tDecodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1;
   }
 
   tEndDecode(pCoder);

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -248,9 +248,12 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) {
   }
 }
 
-static bool rpcRfp(int32_t code) {
+static bool rpcRfp(int32_t code, tmsg_t msgType) {
   if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
       code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
+    if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) {
+      return false;
+    }
     return true;
   } else {
     return false;

source/dnode/mnode/impl/inc/mndTrans.h

@@ -39,8 +39,10 @@ typedef struct {
   int32_t   id;
   int32_t   errCode;
   int32_t   acceptableCode;
-  ETrnStage stage;
+  int32_t   retryCode;
   ETrnAct   actionType;
+  ETrnStage stage;
+  int8_t    reserved;
   int8_t    rawWritten;
   int8_t    msgSent;
   int8_t    msgReceived;

source/dnode/mnode/impl/src/mndShow.c

@@ -234,7 +234,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
   if (retrieveReq.user[0] != 0) {
     memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN);
   } else {
-    memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, TSDB_USER_LEN);
+    memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1);
   }
   if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) {
     return -1;

source/dnode/mnode/impl/src/mndStb.c

@@ -427,17 +427,17 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   req.schemaTag.pSchema = pStb->pTags;
 
   if (req.rollup) {
-    req.pRSmaParam.maxdelay[0] = pStb->maxdelay[0];
+    req.rsmaParam.maxdelay[0] = pStb->maxdelay[0];
-    req.pRSmaParam.maxdelay[1] = pStb->maxdelay[1];
+    req.rsmaParam.maxdelay[1] = pStb->maxdelay[1];
     if (pStb->ast1Len > 0) {
-      if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[0], &req.pRSmaParam.qmsgLen[0], pStb->pAst1, pStb->uid,
+      if (mndConvertRsmaTask(&req.rsmaParam.qmsg[0], &req.rsmaParam.qmsgLen[0], pStb->pAst1, pStb->uid,
-                             STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[0]) < 0) {
+                             STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[0]) < 0) {
         goto _err;
       }
     }
     if (pStb->ast2Len > 0) {
-      if (mndConvertRsmaTask(&req.pRSmaParam.qmsg[1], &req.pRSmaParam.qmsgLen[1], pStb->pAst2, pStb->uid,
+      if (mndConvertRsmaTask(&req.rsmaParam.qmsg[1], &req.rsmaParam.qmsgLen[1], pStb->pAst2, pStb->uid,
-                             STREAM_TRIGGER_WINDOW_CLOSE, req.pRSmaParam.watermark[1]) < 0) {
+                             STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[1]) < 0) {
         goto _err;
       }
     }
@@ -470,12 +470,12 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
   tEncoderClear(&encoder);
 
   *pContLen = contLen;
-  taosMemoryFreeClear(req.pRSmaParam.qmsg[0]);
+  taosMemoryFreeClear(req.rsmaParam.qmsg[0]);
-  taosMemoryFreeClear(req.pRSmaParam.qmsg[1]);
+  taosMemoryFreeClear(req.rsmaParam.qmsg[1]);
   return pHead;
 _err:
-  taosMemoryFreeClear(req.pRSmaParam.qmsg[0]);
+  taosMemoryFreeClear(req.rsmaParam.qmsg[0]);
-  taosMemoryFreeClear(req.pRSmaParam.qmsg[1]);
+  taosMemoryFreeClear(req.rsmaParam.qmsg[1]);
   return NULL;
 }
 

source/dnode/mnode/impl/src/mndTrans.c

@@ -15,15 +15,15 @@
 
 #define _DEFAULT_SOURCE
 #include "mndTrans.h"
-#include "mndPrivilege.h"
 #include "mndConsumer.h"
 #include "mndDb.h"
+#include "mndPrivilege.h"
 #include "mndShow.h"
 #include "mndSync.h"
 #include "mndUser.h"
 
-#define TRANS_VER_NUMBER 1
+#define TRANS_VER_NUMBER   1
-#define TRANS_ARRAY_SIZE 8
+#define TRANS_ARRAY_SIZE   8
 #define TRANS_RESERVE_SIZE 64
 
 static SSdbRaw *mndTransActionEncode(STrans *pTrans);
@@ -55,7 +55,7 @@ static bool    mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans);
 static bool    mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsMaster(pMnode); }
 
 static void    mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
-static int32_t mndProcessTransReq(SRpcMsg *pReq);
+static int32_t mndProcessTransTimer(SRpcMsg *pReq);
 static int32_t mndProcessTtl(SRpcMsg *pReq);
 static int32_t mndProcessKillTransReq(SRpcMsg *pReq);
 
@@ -73,7 +73,7 @@ int32_t mndInitTrans(SMnode *pMnode) {
       .deleteFp = (SdbDeleteFp)mndTransActionDelete,
   };
 
-  mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransTimer);
   mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq);
 
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndRetrieveTrans);
@@ -139,8 +139,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
     SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER)
     SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER)
     SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER)
+    SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER)
     SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER)
     SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER)
+    SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER)
     if (pAction->actionType == TRANS_ACTION_RAW) {
       int32_t len = sdbGetRawTotalSize(pAction->pRaw);
       SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER)
@@ -163,8 +165,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
     SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER)
     SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER)
     SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER)
+    SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER)
     SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER)
     SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER)
+    SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER)
     if (pAction->actionType == TRANS_ACTION_RAW) {
       int32_t len = sdbGetRawTotalSize(pAction->pRaw);
       SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER)
@@ -187,8 +191,10 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
     SDB_SET_INT32(pRaw, dataPos, pAction->id, _OVER)
     SDB_SET_INT32(pRaw, dataPos, pAction->errCode, _OVER)
     SDB_SET_INT32(pRaw, dataPos, pAction->acceptableCode, _OVER)
+    SDB_SET_INT32(pRaw, dataPos, pAction->retryCode, _OVER)
     SDB_SET_INT8(pRaw, dataPos, pAction->actionType, _OVER)
     SDB_SET_INT8(pRaw, dataPos, pAction->stage, _OVER)
+    SDB_SET_INT8(pRaw, dataPos, pAction->reserved, _OVER)
     if (pAction->actionType == TRANS_ACTION_RAW) {
       int32_t len = sdbGetRawTotalSize(pAction->pRaw);
       SDB_SET_INT8(pRaw, dataPos, pAction->rawWritten, _OVER)
@@ -291,10 +297,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
     SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER)
     SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER)
     SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER)
+    SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER)
     SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER)
     action.actionType = actionType;
     SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
     action.stage = stage;
+    SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER)
     if (action.actionType == TRANS_ACTION_RAW) {
       SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER)
       SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
@@ -324,10 +332,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
     SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER)
     SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER)
     SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER)
+    SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER)
     SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER)
     action.actionType = actionType;
     SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
     action.stage = stage;
+    SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER)
     if (action.actionType == TRANS_ACTION_RAW) {
       SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER)
       SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
@@ -357,10 +367,12 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
     SDB_GET_INT32(pRaw, dataPos, &action.id, _OVER)
     SDB_GET_INT32(pRaw, dataPos, &action.errCode, _OVER)
     SDB_GET_INT32(pRaw, dataPos, &action.acceptableCode, _OVER)
+    SDB_GET_INT32(pRaw, dataPos, &action.retryCode, _OVER)
     SDB_GET_INT8(pRaw, dataPos, &actionType, _OVER)
     action.actionType = actionType;
     SDB_GET_INT8(pRaw, dataPos, &stage, _OVER)
     action.stage = stage;
+    SDB_GET_INT8(pRaw, dataPos, &action.reserved, _OVER)
     if (action.actionType) {
       SDB_GET_INT8(pRaw, dataPos, &action.rawWritten, _OVER)
       SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER)
@@ -463,15 +475,25 @@ static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
     if (fp) {
       (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
     }
+    pTrans->startFunc = 0;
   }
 
   return 0;
 }
 
 static void mndTransDropData(STrans *pTrans) {
-  mndTransDropActions(pTrans->redoActions);
+  if (pTrans->redoActions != NULL) {
-  mndTransDropActions(pTrans->undoActions);
+    mndTransDropActions(pTrans->redoActions);
-  mndTransDropActions(pTrans->commitActions);
+    pTrans->redoActions = NULL;
+  }
+  if (pTrans->undoActions != NULL) {
+    mndTransDropActions(pTrans->undoActions);
+    pTrans->undoActions = NULL;
+  }
+  if (pTrans->commitActions != NULL) {
+    mndTransDropActions(pTrans->commitActions);
+    pTrans->commitActions = NULL;
+  }
   if (pTrans->rpcRsp != NULL) {
     taosMemoryFree(pTrans->rpcRsp);
     pTrans->rpcRsp = NULL;
@@ -492,6 +514,7 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) {
     if (fp) {
       (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
     }
+    pTrans->stopFunc = 0;
   }
 
   mndTransDropData(pTrans);
@@ -805,7 +828,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
       sendRsp = true;
     }
   } else {
-    if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 3) {
+    if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 6) {
       if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR;
       sendRsp = true;
     }
@@ -875,8 +898,8 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) {
     pAction->errCode = pRsp->code;
   }
 
-  mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x", transId, mndTransStr(pAction->stage), action,
+  mDebug("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId,
-         pRsp->code, pAction->acceptableCode);
+         mndTransStr(pAction->stage), action, pRsp->code, pAction->acceptableCode, pAction->retryCode);
   mndTransExecute(pMnode, pTrans);
 
 _OVER:
@@ -884,6 +907,21 @@ _OVER:
   return 0;
 }
 
+static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) {
+  pAction->rawWritten = 0;
+  pAction->msgSent = 0;
+  pAction->msgReceived = 0;
+  if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR ||
+      pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) {
+    pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps;
+    mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage),
+           pAction->id, pAction->epSet.inUse);
+  } else {
+    mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), pAction->id);
+  }
+  pAction->errCode = 0;
+}
+
 static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) {
   int32_t numOfActions = taosArrayGetSize(pArray);
 
@@ -894,18 +932,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray)
       continue;
     if (pAction->rawWritten && (pAction->errCode == 0 || pAction->errCode == pAction->acceptableCode)) continue;
 
-    pAction->rawWritten = 0;
+    mndTransResetAction(pMnode, pTrans, pAction);
-    pAction->msgSent = 0;
-    pAction->msgReceived = 0;
-    if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR ||
-        pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) {
-      pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps;
-      mDebug("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage),
-             action, pAction->epSet.inUse);
-    } else {
-      mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action);
-    }
-    pAction->errCode = 0;
   }
 }
 
@@ -1112,9 +1139,9 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
         if (pAction->msgReceived) {
           if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) {
             code = pAction->errCode;
-            pAction->msgSent = 0;
+            mndTransResetAction(pMnode, pTrans, pAction);
-            pAction->msgReceived = 0;
+          } else {
-            mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action);
+            mDebug("trans:%d, %s:%d execute successfully", pTrans->id, mndTransStr(pAction->stage), action);
           }
         } else {
           code = TSDB_CODE_ACTION_IN_PROGRESS;
@@ -1123,6 +1150,8 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
       if (pAction->rawWritten) {
         if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) {
           code = pAction->errCode;
+        } else {
+          mDebug("trans:%d, %s:%d write successfully", pTrans->id, mndTransStr(pAction->stage), action);
         }
       }
     }
@@ -1156,9 +1185,16 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
     } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
       mDebug("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id);
       break;
+    } else if (code == pAction->retryCode) {
+      mDebug("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code);
+      taosMsleep(300);
+      action--;
+      continue;
     } else {
       terrno = code;
       pTrans->code = code;
+      mDebug("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id,
+             mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes);
       break;
     }
   }
@@ -1343,7 +1379,7 @@ void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
   mndTransSendRpcRsp(pMnode, pTrans);
 }
 
-static int32_t mndProcessTransReq(SRpcMsg *pReq) {
+static int32_t mndProcessTransTimer(SRpcMsg *pReq) {
   mndTransPullup(pReq->info.node);
   return 0;
 }

source/dnode/mnode/impl/src/mndVgroup.c

@@ -15,10 +15,10 @@
 
 #define _DEFAULT_SOURCE
 #include "mndVgroup.h"
-#include "mndPrivilege.h"
 #include "mndDb.h"
 #include "mndDnode.h"
 #include "mndMnode.h"
+#include "mndPrivilege.h"
 #include "mndShow.h"
 #include "mndTrans.h"
 #include "mndUser.h"
@@ -896,6 +896,8 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD
   action.pCont = pHead;
   action.contLen = contLen;
   action.msgType = TDMT_VND_ALTER_CONFIRM;
+  // incorrect redirect result will cause this erro
+  action.retryCode = TSDB_CODE_VND_INVALID_VGROUP_ID;
 
   if (mndTransAppendRedoAction(pTrans, &action) != 0) {
     taosMemoryFree(pHead);
@@ -942,6 +944,8 @@ static int32_t mndAddSetVnodeStandByAction(SMnode *pMnode, STrans *pTrans, SDbOb
   action.contLen = contLen;
   action.msgType = TDMT_SYNC_SET_VNODE_STANDBY;
   action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED;
+  // Keep retrying until the target vnode is not the leader
+  action.retryCode = TSDB_CODE_SYN_IS_LEADER;
 
   if (isRedo) {
     if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@@ -1003,7 +1007,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
 
   mInfo("vgId:%d, will add 1 vnodes", pVgroup->vgId);
   if (mndAddVnodeToVgroup(pMnode, &newVg, pArray) != 0) return -1;
-  if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[1], true) != 0) return -1;
+  if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1], true) != 0) return -1;
   if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1;
   if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
 
@@ -1017,10 +1021,19 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
   if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1;
   if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
 
-  SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
+  {
-  if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1;
+    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
-  sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+    if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) return -1;
-  pRaw = NULL;
+    sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+    pRaw = NULL;
+  }
+
+  {
+    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
+    if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1;
+    sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+    pRaw = NULL;
+  }
 
   mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica);
   for (int32_t i = 0; i < newVg.replica; ++i) {
@@ -1168,10 +1181,19 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
     if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pOld3->id) != 0) goto _OVER;
   }
 
-  pRaw = mndVgroupActionEncode(&newVg);
+  {
-  if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
+    pRaw = mndVgroupActionEncode(&newVg);
-  sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+    if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER;
-  pRaw = NULL;
+    sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+    pRaw = NULL;
+  }
+
+  {
+    pRaw = mndVgroupActionEncode(&newVg);
+    if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
+    sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+    pRaw = NULL;
+  }
 
   mInfo("vgId:%d, vgroup info after redistribute, replica:%d", newVg.vgId, newVg.replica);
   for (int32_t i = 0; i < newVg.replica; ++i) {
@@ -1229,7 +1251,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
     }
 
     if (req.dnodeId1 == pVgroup->vnodeGid[0].dnodeId) {
-      terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
+      // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
+      code = 0;
       goto _OVER;
     }
 
@@ -1351,7 +1374,8 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
     }
 
     if (pNew1 == NULL && pOld1 == NULL && pNew2 == NULL && pOld2 == NULL && pNew3 == NULL && pOld3 == NULL) {
-      terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
+      // terrno = TSDB_CODE_MND_VGROUP_UN_CHANGED;
+      code = 0;
       goto _OVER;
     }
 
@@ -1424,13 +1448,25 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, S
     } else {
     }
 
-    SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
+    {
-    if (pVgRaw == NULL) return -1;
+      SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
-    if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
+      if (pVgRaw == NULL) return -1;
-      sdbFreeRaw(pVgRaw);
+      if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) {
-      return -1;
+        sdbFreeRaw(pVgRaw);
+        return -1;
+      }
+      sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
+    }
+
+    {
+      SSdbRaw *pVgRaw = mndVgroupActionEncode(&newVgroup);
+      if (pVgRaw == NULL) return -1;
+      if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
+        sdbFreeRaw(pVgRaw);
+        return -1;
+      }
+      sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
     }
-    sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
   }
 
   return 0;
@@ -1538,12 +1574,23 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD
   if (mndAddIncVgroupReplicaToTrans(pMnode, pTrans, pDb, &newVg, pDst->id) != 0) return -1;
   if (mndAddDecVgroupReplicaFromTrans(pMnode, pTrans, pDb, &newVg, pSrc->id) != 0) return -1;
 
-  SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
+  {
-  if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) {
+    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
-    sdbFreeRaw(pRaw);
+    if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) {
-    return -1;
+      sdbFreeRaw(pRaw);
+      return -1;
+    }
+    sdbSetRawStatus(pRaw, SDB_STATUS_READY);
+  }
+
+  {
+    SSdbRaw *pRaw = mndVgroupActionEncode(&newVg);
+    if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) {
+      sdbFreeRaw(pRaw);
+      return -1;
+    }
+    sdbSetRawStatus(pRaw, SDB_STATUS_READY);
   }
-  sdbSetRawStatus(pRaw, SDB_STATUS_READY);
 
   mInfo("vgId:%d, vgroup info after balance, replica:%d", newVg.vgId, newVg.replica);
   for (int32_t i = 0; i < newVg.replica; ++i) {
@@ -1552,7 +1599,8 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD
   return 0;
 }
 
-static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst) {
+static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst,
+                                            SHashObj *pBalancedVgroups) {
   void   *pIter = NULL;
   int32_t code = -1;
   SSdb   *pSdb = pMnode->pSdb;
@@ -1561,6 +1609,10 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno
     SVgObj *pVgroup = NULL;
     pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
     if (pIter == NULL) break;
+    if (taosHashGet(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t)) != NULL) {
+      sdbRelease(pSdb, pVgroup);
+      continue;
+    }
 
     bool existInSrc = false;
     bool existInDst = false;
@@ -1577,6 +1629,9 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno
 
     SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName);
     code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst);
+    if (code == 0) {
+      code = taosHashPut(pBalancedVgroups, &pVgroup->vgId, sizeof(int32_t), &pVgroup->vgId, sizeof(int32_t));
+    }
     mndReleaseDb(pMnode, pDb);
     sdbRelease(pSdb, pVgroup);
     sdbCancelFetch(pSdb, pIter);
@@ -1590,6 +1645,10 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
   int32_t code = -1;
   int32_t numOfVgroups = 0;
   STrans *pTrans = NULL;
+  SHashObj *pBalancedVgroups = NULL;
+
+  pBalancedVgroups = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+  if (pBalancedVgroups == NULL) goto _OVER;
 
   pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq);
   if (pTrans == NULL) goto _OVER;
@@ -1613,18 +1672,18 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) {
            pDst->id, dstScore);
 
     if (srcScore > dstScore - 0.000001) {
-      code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst);
+      code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst, pBalancedVgroups);
       if (code == 0) {
         pSrc->numOfVnodes--;
         pDst->numOfVnodes++;
         numOfVgroups++;
         continue;
       } else {
-        mError("trans:%d, failed to balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id);
+        mDebug("trans:%d, no vgroup need to balance from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id);
-        return -1;
+        break;
       }
     } else {
-      mDebug("trans:%d, no vgroup need to balance vgroup any more", pTrans->id);
+      mDebug("trans:%d, no vgroup need to balance any more", pTrans->id);
       break;
     }
   }

source/dnode/vnode/inc/vnode.h

@@ -199,15 +199,20 @@ typedef struct {
   uint64_t groupId;
 } STableKeyInfo;
 
+#define TABLE_ROLLUP_ON       ((int8_t)0x1)
+#define TABLE_IS_ROLLUP(FLG)  (((FLG) & (TABLE_ROLLUP_ON)) != 0)
+#define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON)
 struct SMetaEntry {
   int64_t  version;
   int8_t   type;
+  int8_t   flags;  // TODO: need refactor?
   tb_uid_t uid;
   char    *name;
   union {
     struct {
       SSchemaWrapper schemaRow;
       SSchemaWrapper schemaTag;
+      SRSmaParam     rsmaParam;
     } stbEntry;
     struct {
       int64_t  ctime;

source/dnode/vnode/src/inc/sma.h

@@ -176,15 +176,18 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) {
 
 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
+void          *tdFreeRSmaInfo(SRSmaInfo *pInfo);
 
-void *tdFreeRSmaInfo(SRSmaInfo *pInfo);
+int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
-
+int32_t tdProcessRSmaRestoreImpl(SSma *pSma);
 int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg);
 int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
 int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
 
 // smaFileUtil ================
 
+#define TD_FILE_HEAD_SIZE 512
+
 typedef struct STFInfo STFInfo;
 typedef struct STFile  STFile;
 
@@ -220,12 +223,14 @@ int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte);
 int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence);
 int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte);
 int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset);
+int64_t tdGetTFileSize(STFile *pTFile, int64_t *size);
 int32_t tdRemoveTFile(STFile *pTFile);
 int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo);
 int32_t tdUpdateTFileHeader(STFile *pTFile);
 void    tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
 void    tdCloseTFile(STFile *pTFile);
-void    tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName);
+
+void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/meta/metaEntry.c

@@ -24,22 +24,25 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
   if (tEncodeCStr(pCoder, pME->name) < 0) return -1;
 
   if (pME->type == TSDB_SUPER_TABLE) {
+    if (tEncodeI8(pCoder, pME->flags) < 0) return -1;  // TODO: need refactor?
     if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaRow) < 0) return -1;
     if (tEncodeSSchemaWrapper(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
+    if (TABLE_IS_ROLLUP(pME->flags)) {
+      if (tEncodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
+    }
   } else if (pME->type == TSDB_CHILD_TABLE) {
     if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1;
     if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1;
-    if (tEncodeI32(pCoder, pME->ctbEntry.commentLen) < 0) return -1;
+    if (tEncodeI32v(pCoder, pME->ctbEntry.commentLen) < 0) return -1;
     if (pME->ctbEntry.commentLen > 0){
       if (tEncodeCStr(pCoder, pME->ctbEntry.comment) < 0) return -1;
     }
     if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1;
-    debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug
     if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1;
   } else if (pME->type == TSDB_NORMAL_TABLE) {
     if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1;
     if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1;
-    if (tEncodeI32(pCoder, pME->ntbEntry.commentLen) < 0) return -1;
+    if (tEncodeI32v(pCoder, pME->ntbEntry.commentLen) < 0) return -1;
     if (pME->ntbEntry.commentLen > 0){
       if (tEncodeCStr(pCoder, pME->ntbEntry.comment) < 0) return -1;
     }
@@ -64,23 +67,26 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
   if (tDecodeCStr(pCoder, &pME->name) < 0) return -1;
 
   if (pME->type == TSDB_SUPER_TABLE) {
+    if (tDecodeI8(pCoder, &pME->flags) < 0) return -1;  // TODO: need refactor?
     if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaRow) < 0) return -1;
     if (tDecodeSSchemaWrapperEx(pCoder, &pME->stbEntry.schemaTag) < 0) return -1;
+    if (TABLE_IS_ROLLUP(pME->flags)) {
+      if (tDecodeSRSmaParam(pCoder, &pME->stbEntry.rsmaParam) < 0) return -1;
+    }
   } else if (pME->type == TSDB_CHILD_TABLE) {
     if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1;
-    if (tDecodeI32(pCoder, &pME->ctbEntry.commentLen) < 0) return -1;
+    if (tDecodeI32v(pCoder, &pME->ctbEntry.commentLen) < 0) return -1;
     if (pME->ctbEntry.commentLen > 0){
       if (tDecodeCStr(pCoder, &pME->ctbEntry.comment) < 0)
         return -1;
     }
     if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1;
     if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1;  // (TODO)
-    debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug
   } else if (pME->type == TSDB_NORMAL_TABLE) {
     if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1;
     if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1;
-    if (tDecodeI32(pCoder, &pME->ntbEntry.commentLen) < 0) return -1;
+    if (tDecodeI32v(pCoder, &pME->ntbEntry.commentLen) < 0) return -1;
     if (pME->ntbEntry.commentLen > 0){
       if (tDecodeCStr(pCoder, &pME->ntbEntry.comment) < 0) return -1;
     }

source/dnode/vnode/src/meta/metaQuery.c

@@ -342,6 +342,7 @@ SMStbCursor *metaOpenStbCursor(SMeta *pMeta, tb_uid_t suid) {
 
   pStbCur = (SMStbCursor *)taosMemoryCalloc(1, sizeof(*pStbCur));
   if (pStbCur == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     return NULL;
   }
 
@@ -351,6 +352,7 @@ SMStbCursor *metaOpenStbCursor(SMeta *pMeta, tb_uid_t suid) {
 
   ret = tdbTbcOpen(pMeta->pSuidIdx, &pStbCur->pCur, NULL);
   if (ret < 0) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
     metaULock(pMeta);
     taosMemoryFree(pStbCur);
     return NULL;

source/dnode/vnode/src/meta/metaTable.c

@@ -139,6 +139,10 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   me.name = pReq->name;
   me.stbEntry.schemaRow = pReq->schemaRow;
   me.stbEntry.schemaTag = pReq->schemaTag;
+  if (pReq->rollup) {
+    TABLE_SET_ROLLUP(me.flags);
+    me.stbEntry.rsmaParam = pReq->rsmaParam;
+  }
 
   if (metaHandleEntry(pMeta, &me) < 0) goto _err;
 

source/dnode/vnode/src/sma/smaEnv.c

@@ -156,6 +156,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
 
 static void tdDestroyTSmaStat(STSmaStat *pStat) {
   if (pStat) {
+    smaDebug("destroy tsma stat");
     tDestroyTSma(pStat->pTSma);
     taosMemoryFreeClear(pStat->pTSma);
     taosMemoryFreeClear(pStat->pTSchema);
@@ -170,15 +171,12 @@ static void *tdFreeTSmaStat(STSmaStat *pStat) {
 
 static void tdDestroyRSmaStat(SRSmaStat *pStat) {
   if (pStat) {
-    smaDebug("vgId:%d, %s:%d free rsma stat", SMA_VID(pStat->pSma), __func__, __LINE__);
+    smaDebug("vgId:%d destroy rsma stat", SMA_VID(pStat->pSma));
     // step 1: set persistence task cancelled
     atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED);
 
-    // step 2: clean timer
+    // step 2: stop the persistence timer
     taosTmrStopA(&RSMA_TMR_ID(pStat));
-    if (RSMA_TMR_HANDLE(pStat)) {
-      taosTmrCleanUp(RSMA_TMR_HANDLE(pStat));
-    }
 
     // step 3: wait the persistence thread to finish
     int32_t nLoops = 0;
@@ -194,7 +192,6 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) {
           sched_yield();
           nLoops = 0;
         }
-        taosMsleep(1000);  // TODO: remove this line when release
       }
     }
 
@@ -219,7 +216,11 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) {
         sched_yield();
         nLoops = 0;
       }
-      taosMsleep(1000); // TODO: remove this line when release
+    }
+
+    // step 6: cleanup the timer handle
+    if (RSMA_TMR_HANDLE(pStat)) {
+      taosTmrCleanUp(RSMA_TMR_HANDLE(pStat));
     }
   }
 }
@@ -245,16 +246,12 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) {
 int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
   if (pSmaStat) {
     if (smaType == TSDB_SMA_TYPE_TIME_RANGE) {
-      smaDebug("%s:%d destroy tsma stat", __func__, __LINE__);
       tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat));
     } else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
-      smaDebug("%s:%d destroy rsma stat", __func__, __LINE__);
       tdDestroyRSmaStat(SMA_RSMA_STAT(pSmaStat));
     } else {
       ASSERT(0);
     }
-  } else {
-    smaDebug("%s:%d no need to destroy rsma stat", __func__, __LINE__);
   }
   return TSDB_CODE_SUCCESS;
 }

source/dnode/vnode/src/sma/smaOpen.c

@@ -18,6 +18,7 @@
 
 static int32_t smaEvalDays(SRetention *r, int8_t precision);
 static int32_t smaSetKeepCfg(STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type);
+static int32_t rsmaRestore(SSma *pSma);
 
 #define SMA_SET_KEEP_CFG(l)                                                                       \
   do {                                                                                            \
@@ -100,6 +101,9 @@ int32_t smaOpen(SVnode *pVnode) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
   }
+
+  pVnode->pSma = pSma;
+
   pSma->pVnode = pVnode;
   taosThreadMutexInit(&pSma->mutex, NULL);
   pSma->locked = false;
@@ -117,17 +121,22 @@ int32_t smaOpen(SVnode *pVnode) {
         ASSERT(0);
       }
     }
+
+    // restore the rsma
+#if 0
+    if (rsmaRestore(pSma) < 0) {
+      goto _err;
+    }
+#endif
   }
 
-  pVnode->pSma = pSma;
   return 0;
 _err:
-  taosMemoryFreeClear(pSma);
   return -1;
 }
 
 int32_t smaCloseEnv(SSma *pSma) {
-  if(pSma) {
+  if (pSma) {
     SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
     SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));
   }
@@ -153,13 +162,12 @@ int32_t smaClose(SSma *pSma) {
 
 /**
  * @brief rsma env restore
- * 
+ *
- * @param pSma 
+ * @param pSma
- * @return int32_t 
+ * @return int32_t
  */
-int32_t smaRestore(SSma *pSma) {
+static int32_t rsmaRestore(SSma *pSma) {
-  if (!pSma) return 0;
+  ASSERT(VND_IS_RSMA(pSma->pVnode));
-  // iterate all stables to restore the rsma env
+
-  
+  return tdProcessRSmaRestoreImpl(pSma);
-  return TSDB_CODE_SUCCESS;
 }

source/dnode/vnode/src/sma/smaRollup.c

@@ -15,10 +15,14 @@
 
 #include "sma.h"
 
-#define RSMA_QTASK_PERSIST_MS 7200000
+#define RSMA_QTASKINFO_PERSIST_MS 7200000
+#define RSMA_QTASKINFO_BUFSIZE    32768
 typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T;
 static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
 
+typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
+typedef struct SRSmaQTaskFIter    SRSmaQTaskFIter;
+
 static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
 static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
 static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle,
@@ -27,6 +31,13 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
                                  tb_uid_t suid, int8_t level);
 static void    tdRSmaFetchTrigger(void *param, void *tmrId);
 static void    tdRSmaPersistTrigger(void *param, void *tmrId);
+static void   *tdRSmaPersistExec(void *param);
+static void    tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName);
+
+static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile);
+static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish);
+static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd);
+static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem);
 
 struct SRSmaInfoItem {
   SRSmaInfo *pRsmaInfo;
@@ -45,14 +56,38 @@ struct SRSmaInfo {
   SRSmaInfoItem items[TSDB_RETENTION_L2];
 };
 
+struct SRSmaQTaskInfoItem {
+  int32_t len;
+  int8_t  type;
+  int64_t suid;
+  void   *qTaskInfo;
+};
+
+struct SRSmaQTaskFIter {
+  STFile *pTFile;
+  int64_t offset;
+  int64_t fsize;
+  int32_t nBytes;
+  int32_t nAlloc;
+  char   *buf;
+  // ------------
+  int32_t nBufPos;
+};
+
+static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) {
+  return lenWithHead - sizeof(int32_t) - sizeof(int8_t) - sizeof(int64_t);
+}
+
+static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskFIter *pIter) { taosMemoryFreeClear(pIter->buf); }
+
 static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) {
   // Note: free/kill may in RC
   qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle);
   if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) {
-    smaDebug("vgId:%d, %s:%d free qTaskInfo_t %p of level %d", vgId, __func__, __LINE__, otaskHandle, level);
+    smaDebug("vgId:%d, free qTaskInfo_t %p of level %d", vgId, otaskHandle, level);
     qDestroyTask(otaskHandle);
   } else {
-    smaDebug("vgId:%d, %s:%d not free qTaskInfo_t %p of level %d", vgId, __func__, __LINE__, otaskHandle, level);
+    smaDebug("vgId:%d, not free qTaskInfo_t %p of level %d", vgId, otaskHandle, level);
   }
 }
 
@@ -89,7 +124,7 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) {
   return TSDB_CODE_SUCCESS;
 }
 
-static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
+static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
   SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
   SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
   SRSmaInfo *pRSmaInfo = NULL;
@@ -230,26 +265,21 @@ _err:
 }
 
 /**
- * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam.
+ * @brief for rsam create or restore
  *
- * @param pTsdb
+ * @param pSma
- * @param pMeta
+ * @param param
- * @param pReq
+ * @param suid
+ * @param tbName
  * @return int32_t
  */
-int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
+int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName) {
-  SSma *pSma = pVnode->pSma;
+  SVnode *pVnode = pSma->pVnode;
-  if (!pReq->rollup) {
+  SMeta  *pMeta = pVnode->pMeta;
-    smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
+  SMsgCb *pMsgCb = &pVnode->msgCb;
-    return TSDB_CODE_SUCCESS;
-  }
-
-  SMeta      *pMeta = pVnode->pMeta;
-  SMsgCb     *pMsgCb = &pVnode->msgCb;
-  SRSmaParam *param = &pReq->pRSmaParam;
 
   if ((param->qmsgLen[0] == 0) && (param->qmsgLen[1] == 0)) {
-    smaWarn("vgId:%d, no qmsg1/qmsg2 for rollup stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
+    smaDebug("vgId:%d, no qmsg1/qmsg2 for rollup table %s %" PRIi64, SMA_VID(pSma), tbName, suid);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -262,10 +292,10 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
   SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
   SRSmaInfo *pRSmaInfo = NULL;
 
-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t));
+  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
   if (pRSmaInfo) {
     ASSERT(0);  // TODO: free original pRSmaInfo is exists abnormally
-    smaWarn("vgId:%d, rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
+    smaDebug("vgId:%d, rsma info already exists for table %s, %" PRIi64, SMA_VID(pSma), tbName, suid);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -289,14 +319,14 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
       .vnode = pVnode,
   };
 
-  STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), pReq->suid, -1);
+  STSchema *pTSchema = metaGetTbTSchema(SMA_META(pSma), suid, -1);
   if (!pTSchema) {
     terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
     goto _err;
   }
   pRSmaInfo->pTSchema = pTSchema;
   pRSmaInfo->pSma = pSma;
-  pRSmaInfo->suid = pReq->suid;
+  pRSmaInfo->suid = suid;
 
   if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) {
     goto _err;
@@ -306,16 +336,16 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
     goto _err;
   }
 
-  if (taosHashPut(RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) {
+  if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) {
     goto _err;
   } else {
-    smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid);
+    smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid);
   }
 
   // start the persist timer
   if (TASK_TRIGGER_STAT_INIT ==
       atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) {
-    taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASK_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat));
+    taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat));
   }
 
   return TSDB_CODE_SUCCESS;
@@ -325,6 +355,24 @@ _err:
   return TSDB_CODE_FAILED;
 }
 
+/**
+ * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam.
+ *
+ * @param pTsdb
+ * @param pMeta
+ * @param pReq
+ * @return int32_t
+ */
+int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) {
+  SSma *pSma = pVnode->pSma;
+  if (!pReq->rollup) {
+    smaTrace("vgId:%d, return directly since no rollup for stable %s %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  return tdProcessRSmaCreateImpl(pSma, &pReq->rsmaParam, pReq->suid, pReq->name);
+}
+
 /**
  * @brief store suid/[uids], prefer to use array and then hash
  *
@@ -487,7 +535,7 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
 #if 1
     char flag[10] = {0};
     snprintf(flag, 10, "level %" PRIi8, pItem->level);
-    blockDebugShowData(pResult, flag);
+    blockDebugShowDataBlocks(pResult, flag);
 #endif
     STsdb      *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
     SSubmitReq *pReq = NULL;
@@ -502,8 +550,10 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
     }
 
     taosMemoryFreeClear(pReq);
+  } else if (terrno == 0) {
+    smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level);
   } else {
-    smaDebug("vgId:%d, no rsma %" PRIi8 " data generated since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno));
+    smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno));
   }
 
   tdDestroySDataBlockArray(pResult);
@@ -526,16 +576,16 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
 
   int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
   if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) {
-    smaDebug("vgId:%d, %s:%d level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma),
+    smaDebug("vgId:%d, level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma),
-             __func__, __LINE__, pItem->level, pItem->pRsmaInfo->suid);
+             pItem->level, pItem->pRsmaInfo->suid);
     return;
   }
 
   int8_t fetchTriggerStat =
       atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
   if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) {
-    smaDebug("vgId:%d, %s:%d level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), __func__,
+    smaDebug("vgId:%d, level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), pItem->level,
-             __LINE__, pItem->level, pItem->pRsmaInfo->suid);
+             pItem->pRsmaInfo->suid);
 
     tdRefSmaStat(pSma, (SSmaStat *)pStat);
 
@@ -546,13 +596,13 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
     tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
 
   } else {
-    smaDebug("vgId:%d, %s:%d level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), __func__,
+    smaDebug("vgId:%d, level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), pItem->level,
-             __LINE__, pItem->level, pItem->pRsmaInfo->suid);
+             pItem->pRsmaInfo->suid);
   }
 }
 
-static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem,
+static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid,
-                                              tb_uid_t suid, int8_t level) {
+                                 int8_t level) {
   if (!pItem || !pItem->taskInfo) {
     smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
     return TSDB_CODE_SUCCESS;
@@ -568,7 +618,7 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3
 
   tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK);
   atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
-  smaDebug("vgId:%d, %s:%d process rsma insert", SMA_VID(pSma), __func__, __LINE__);
+  smaDebug("vgId:%d, process rsma insert", SMA_VID(pSma));
 
   SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
   SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat);
@@ -645,7 +695,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
   return TSDB_CODE_SUCCESS;
 }
 
-void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) {
+static void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) {
   tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName);
 }
 
@@ -661,18 +711,6 @@ static void *tdRSmaPersistExec(void *param) {
     goto _end;
   }
 
-#if 0
-  SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
-  if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) {
-    ASSERT(0);
-  } else {
-    for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) {
-      tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
-      smaDebug("suid [%d] is %" PRIi64, i, suid);
-    }
-  }
-#endif
-
   void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL);
   if (!infoHash) {
     goto _end;
@@ -700,7 +738,7 @@ static void *tdRSmaPersistExec(void *param) {
       }
       char   *pOutput = NULL;
       int32_t len = 0;
-      int8_t  type = 0;
+      int8_t  type = (int8_t)(i + 1);
       if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) {
         smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1,
                  terrstr(terrno));
@@ -736,8 +774,9 @@ static void *tdRSmaPersistExec(void *param) {
 
         isFileCreated = true;
       }
-      len += (sizeof(len) + sizeof(pRSmaInfo->suid));
+      len += (sizeof(len) + sizeof(type) + sizeof(pRSmaInfo->suid));
       tdAppendTFile(&tFile, &len, sizeof(len), &toffset);
+      tdAppendTFile(&tFile, &type, sizeof(type), &toffset);
       tdAppendTFile(&tFile, &pRSmaInfo->suid, sizeof(pRSmaInfo->suid), &toffset);
       tdAppendTFile(&tFile, pOutput, len, &toffset);
 
@@ -833,25 +872,275 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
       if (TASK_TRIGGER_STAT_CANCELLED != atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
                                                                        TASK_TRIGGER_STAT_CANCELLED,
                                                                        TASK_TRIGGER_STAT_FINISHED)) {
-        smaDebug("%s:%d rsma persistence start since active", __func__, __LINE__);
+        smaDebug("rsma persistence start since active");
+
+        // start persist task
         tdRSmaPersistTask(pRSmaStat);
-        taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASK_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId);
+
+        taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle,
+                     &pRSmaStat->tmrId);
       } else {
         atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
       }
     } break;
     case TASK_TRIGGER_STAT_CANCELLED: {
       atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED);
-      smaDebug("%s:%d rsma persistence not start since cancelled and finished", __func__, __LINE__);
+      smaDebug("rsma persistence not start since cancelled and finished");
     } break;
     case TASK_TRIGGER_STAT_INACTIVE: {
-      smaDebug("%s:%d rsma persistence not start since inactive", __func__, __LINE__);
+      smaDebug("rsma persistence not start since inactive");
     } break;
     case TASK_TRIGGER_STAT_INIT: {
-      smaDebug("%s:%d rsma persistence not start since init", __func__, __LINE__);
+      smaDebug("rsma persistence not start since init");
     } break;
     default: {
-      smaWarn("%s:%d rsma persistence not start since unknown stat %" PRIi8, __func__, __LINE__, tmrStat);
+      smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat);
+      ASSERT(0);
     } break;
   }
-}
+}
+
+int32_t tdProcessRSmaRestoreImpl(SSma *pSma) {
+  SVnode *pVnode = pSma->pVnode;
+
+  // step 1: iterate all stables to restore the rsma env
+
+  SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
+  if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) {
+    smaError("vgId:%d, failed to restore rsma since get stb id list error: %s", TD_VID(pVnode), terrstr());
+    return TSDB_CODE_FAILED;
+  }
+
+  if (taosArrayGetSize(suidList) == 0) {
+    smaDebug("vgId:%d no need to restore rsma since empty stb id list", TD_VID(pVnode));
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SMetaReader mr = {0};
+  metaReaderInit(&mr, SMA_META(pSma), 0);
+  for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) {
+    tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
+    smaDebug("suid [%d] is %" PRIi64, i, suid);
+    if (metaGetTableEntryByUid(&mr, suid) < 0) {
+      smaError("vgId:%d failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr());
+      goto _err;
+    }
+    ASSERT(mr.me.type == TSDB_SUPER_TABLE);
+    ASSERT(mr.me.uid == suid);
+    if (TABLE_IS_ROLLUP(mr.me.flags)) {
+      SRSmaParam *param = &mr.me.stbEntry.rsmaParam;
+      for (int i = 0; i < 2; ++i) {
+        smaDebug("vgId: %d table:%" PRIi64 " maxdelay[%d]:%" PRIi64 " watermark[%d]:%" PRIi64, TD_VID(pSma->pVnode),
+                 suid, i, param->maxdelay[i], i, param->watermark[i]);
+      }
+      if (tdProcessRSmaCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) {
+        smaError("vgId:%d failed to retore rsma env for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr());
+        goto _err;
+      }
+    }
+  }
+
+  // step 2: retrieve qtaskinfo object from the rsma/qtaskinfo file and restore
+  STFile tFile = {0};
+  char   qTaskInfoFName[TSDB_FILENAME_LEN];
+
+  tdRSmaQTaskGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName);
+  if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) {
+    goto _err;
+  }
+  if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) {
+    goto _err;
+  }
+  SRSmaQTaskFIter fIter = {0};
+  if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
+    goto _err;
+  }
+  SRSmaQTaskInfoItem infoItem = {0};
+  bool               isEnd = false;
+  int32_t            code = 0;
+  while ((code = tdRSmaQTaskInfoIterNext(&fIter, &infoItem, &isEnd)) == 0) {
+    if (isEnd) {
+      break;
+    }
+    if ((code = tdRSmaQTaskInfoItemRestore(pSma, &infoItem)) < 0) {
+      break;
+    }
+  }
+  tdRSmaQTaskInfoIterDestroy(&fIter);
+
+  if (code < 0) {
+    goto _err;
+  }
+
+  metaReaderClear(&mr);
+  taosArrayDestroy(suidList);
+  return TSDB_CODE_SUCCESS;
+_err:
+  ASSERT(0);
+  metaReaderClear(&mr);
+  taosArrayDestroy(suidList);
+  smaError("failed to restore rsma info since %s", terrstr());
+  return TSDB_CODE_FAILED;
+}
+
+static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem) {
+  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv);
+  SRSmaInfo *pRSmaInfo = NULL;
+  void      *qTaskInfo = NULL;
+
+  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &infoItem->suid, sizeof(infoItem->suid));
+
+  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+    smaDebug("vgId:%d, no restore as no rsma info for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (infoItem->type == 1) {
+    qTaskInfo = pRSmaInfo->items[0].taskInfo;
+  } else if (infoItem->type == 2) {
+    qTaskInfo = pRSmaInfo->items[1].taskInfo;
+  } else {
+    ASSERT(0);
+  }
+
+  if (!qTaskInfo) {
+    smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for suid:%" PRIu64, SMA_VID(pSma), infoItem->suid);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (qDeserializeTaskStatus(qTaskInfo, infoItem->qTaskInfo, infoItem->len) < 0) {
+    smaError("vgId:%d, restore rsma failed for suid:%" PRIi64 " level %d since %s", SMA_VID(pSma), infoItem->suid,
+             infoItem->type, terrstr(terrno));
+    return TSDB_CODE_FAILED;
+  }
+  smaDebug("vgId:%d, restore rsma success for suid:%" PRIi64 " level %d", SMA_VID(pSma), infoItem->suid,
+           infoItem->type);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskFIter *pIter, STFile *pTFile) {
+  memset(pIter, 0, sizeof(*pIter));
+  pIter->pTFile = pTFile;
+  pIter->offset = TD_FILE_HEAD_SIZE;
+
+  if (tdGetTFileSize(pTFile, &pIter->fsize) < 0) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if ((pIter->fsize - TD_FILE_HEAD_SIZE) < RSMA_QTASKINFO_BUFSIZE) {
+    pIter->nAlloc = pIter->fsize - TD_FILE_HEAD_SIZE;
+  } else {
+    pIter->nAlloc = RSMA_QTASKINFO_BUFSIZE;
+  }
+
+  if (pIter->nAlloc < TD_FILE_HEAD_SIZE) {
+    pIter->nAlloc = TD_FILE_HEAD_SIZE;
+  }
+
+  pIter->buf = taosMemoryMalloc(pIter->nAlloc);
+  if (!pIter->buf) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return TSDB_CODE_FAILED;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskFIter *pIter, bool *isFinish) {
+  STFile *pTFile = pIter->pTFile;
+  int64_t nBytes = RSMA_QTASKINFO_BUFSIZE;
+
+  if (pIter->offset >= pIter->fsize) {
+    *isFinish = true;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if ((pIter->fsize - pIter->offset) < RSMA_QTASKINFO_BUFSIZE) {
+    nBytes = pIter->fsize - pIter->offset;
+  }
+
+  if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) {
+    ASSERT(0);
+    return TSDB_CODE_FAILED;
+  }
+
+  if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) {
+    ASSERT(0);
+    return TSDB_CODE_FAILED;
+  }
+
+  int32_t infoLen = 0;
+  taosDecodeFixedI32(pIter->buf, &infoLen);
+  if (infoLen > nBytes) {
+    ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE);
+    pIter->nAlloc = infoLen;
+    void *pBuf = taosMemoryRealloc(pIter->buf, infoLen);
+    if (!pBuf) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      return TSDB_CODE_FAILED;
+    }
+    pIter->buf = pBuf;
+    nBytes = infoLen;
+
+    if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) {
+      ASSERT(0);
+      return TSDB_CODE_FAILED;
+    }
+
+    if (tdReadTFile(pTFile, pIter->buf, nBytes) != nBytes) {
+      ASSERT(0);
+      return TSDB_CODE_FAILED;
+    }
+  }
+
+  pIter->offset += nBytes;
+  pIter->nBytes = nBytes;
+  pIter->nBufPos = 0;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t tdRSmaQTaskInfoIterNext(SRSmaQTaskFIter *pIter, SRSmaQTaskInfoItem *pItem, bool *isEnd) {
+  while (1) {
+    // block iter
+    bool isFinish = false;
+    if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) {
+      ASSERT(0);
+      return TSDB_CODE_FAILED;
+    }
+    if (isFinish) {
+      *isEnd = true;
+      return TSDB_CODE_SUCCESS;
+    }
+
+    // consume the block
+    int32_t qTaskInfoLenWithHead = 0;
+    pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead);
+    if (qTaskInfoLenWithHead < 0) {
+      terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+      return TSDB_CODE_FAILED;
+    }
+    while (1) {
+      if ((pIter->nBufPos + qTaskInfoLenWithHead) <= pIter->nBytes) {
+        pIter->buf = taosDecodeFixedI8(pIter->buf, &pItem->type);
+        pIter->buf = taosDecodeFixedI64(pIter->buf, &pItem->suid);
+        pItem->qTaskInfo = pIter->buf;
+        pItem->len = tdRSmaQTaskInfoContLen(qTaskInfoLenWithHead);
+        // do the restore job
+        printf("%s:%d ###### restore the qtask info offset:%" PRIi64 "\n", __func__, __LINE__, pIter->offset);
+
+        pIter->buf = POINTER_SHIFT(pIter->buf, pItem->len);
+        pIter->nBufPos += qTaskInfoLenWithHead;
+
+        pIter->buf = taosDecodeFixedI32(pIter->buf, &qTaskInfoLenWithHead);
+        continue;
+      }
+      // prepare and load next block in the file
+      pIter->offset -= (pIter->nBytes - pIter->nBufPos);
+      break;
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}

source/dnode/vnode/src/sma/smaUtil.c

@@ -17,8 +17,6 @@
 
 // smaFileUtil ================
 
-#define TD_FILE_HEAD_SIZE 512
-
 #define TD_FILE_STATE_OK  0
 #define TD_FILE_STATE_BAD 1
 
@@ -71,6 +69,11 @@ int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) {
   return loffset;
 }
 
+int64_t tdGetTFileSize(STFile *pTFile, int64_t *size) {
+  ASSERT(TD_FILE_OPENED(pTFile));
+  return taosFStatFile(pTFile->pFile, size, NULL);
+}
+
 int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) {
   ASSERT(TD_FILE_OPENED(pTFile));
 

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -2886,6 +2886,9 @@ int32_t tsdbGetCtbIdList(SMeta* pMeta, int64_t suid, SArray* list) {
  */
 int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list) {
   SMStbCursor* pCur = metaOpenStbCursor(pMeta, suid);
+  if(!pCur) {
+    return TSDB_CODE_FAILED;
+  }
 
   while (1) {
     tb_uid_t id = metaStbCursorNext(pCur);

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -172,9 +172,9 @@ void vnodeClose(SVnode *pVnode) {
     vnodeSyncClose(pVnode);
     vnodeQueryClose(pVnode);
     walClose(pVnode->pWal);
-    smaCloseEx(pVnode->pSma);
     tqClose(pVnode->pTq);
     if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
+    smaCloseEx(pVnode->pSma);
     metaClose(pVnode->pMeta);
     vnodeCloseBufPool(pVnode);
     // destroy handle

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -299,7 +299,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, int64_t version, SRpcMsg *pMsg, SRp
 void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
   // TODO
 
-  blockDebugShowData(data, __func__);
+  blockDebugShowDataBlocks(data, __func__);
   tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data);
 }
 

source/libs/executor/inc/executorimpl.h

@@ -730,10 +730,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo*
 SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo);
-
+SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
-SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams, SSDataBlock* pInputBlock,
-                                                   SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pColMatchColInfo,
-                                                   SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols,
@@ -811,7 +808,7 @@ int32_t getMaximumIdleDurationSec();
  * nOptrWithVal: *nOptrWithVal save the number of optr with value
  * return:  result code, 0 means success
  */
-int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length);
+int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length, int32_t *nOptrWithVal);
 
 /*
  * ops:    root operator, created by caller

source/libs/executor/inc/tsort.h

@@ -136,6 +136,13 @@ bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colId);
  */
 void* tsortGetValue(STupleHandle* pVHandle, int32_t colId);
 
+/**
+ *
+ * @param pVHandle
+ * @return
+ */
+uint64_t tsortGetGroupId(STupleHandle* pVHandle);
+
 /**
  *
  * @param pSortHandle

source/libs/executor/src/executil.c

@@ -159,7 +159,12 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) {
 }
 
 SArray* createSortInfo(SNodeList* pNodeList) {
-  size_t  numOfCols = LIST_LENGTH(pNodeList);
+  size_t numOfCols = 0;
+  if (pNodeList != NULL) {
+    numOfCols = LIST_LENGTH(pNodeList);
+  } else {
+    numOfCols = 0;
+  }
   SArray* pList = taosArrayInit(numOfCols, sizeof(SBlockOrderInfo));
   if (pList == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;

source/libs/executor/src/executorMain.c

@@ -222,7 +222,13 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) {
     return TSDB_CODE_INVALID_PARA;
   }
 
-  return encodeOperator(pTaskInfo->pRoot, pOutput, len);
+  int32_t nOptrWithVal = 0;
+  int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
+  if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) {
+    taosMemoryFreeClear(*pOutput);
+    *len = 0;
+  }
+  return code;
 }
 
 int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) {

source/libs/executor/src/executorimpl.c

@@ -4269,17 +4269,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     pOptr = createGroupSortOperatorInfo(ops[0], (SGroupSortPhysiNode*)pPhyNode, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) {
     SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode;
-
+    pOptr = createMultiwayMergeOperatorInfo(ops, size, pMergePhyNode, pTaskInfo);
-    SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc;
-    SSDataBlock*        pResBlock = createResDataBlock(pDescNode);
-
-    SArray* sortInfo = createSortInfo(pMergePhyNode->pMergeKeys);
-    int32_t numOfOutputCols = 0;
-    SArray* pColList =
-        extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
-    SPhysiNode*  pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
-    SSDataBlock* pInputDataBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc);
-    pOptr = createMultiwaySortMergeOperatorInfo(ops, size, pInputDataBlock, pResBlock, sortInfo, pColList, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION == type) {
     SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
 
@@ -4472,12 +4462,12 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa
   return 0;
 }
 
-int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) {
+int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) {
   int32_t code = TDB_CODE_SUCCESS;
   char*   pCurrent = NULL;
   int32_t currLength = 0;
   if (ops->fpSet.encodeResultRow) {
-    if (result == NULL || length == NULL) {
+    if (result == NULL || length == NULL || nOptrWithVal == NULL) {
       return TSDB_CODE_TSC_INVALID_INPUT;
     }
     code = ops->fpSet.encodeResultRow(ops, &pCurrent, &currLength);
@@ -4488,8 +4478,13 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) {
         *result = NULL;
       }
       return code;
+    } else if (currLength == 0) {
+      ASSERT(!pCurrent);
+      goto _downstream;
     }
-    
+
+    ++(*nOptrWithVal);
+
     ASSERT(currLength >= 0);
 
     if (*result == NULL) {
@@ -4516,8 +4511,10 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) {
     taosMemoryFree(pCurrent);
     *length = *(int32_t*)(*result);
   }
+
+_downstream:
   for (int32_t i = 0; i < ops->numOfDownstream; ++i) {
-    code = encodeOperator(ops->pDownstream[i], result, length);
+    code = encodeOperator(ops->pDownstream[i], result, length, nOptrWithVal);
     if (code != TDB_CODE_SUCCESS) {
       return code;
     }

source/libs/executor/src/scanoperator.c

@@ -2051,7 +2051,7 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
     qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId);
     return TSDB_CODE_SUCCESS;
   }
-  code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pPartitionTags);
+  code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pGroupTags);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -2455,7 +2455,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   if (pInfo == NULL || pOperator == NULL) {
     goto _error;
   }
-  if (pTableScanNode->pPartitionTags) {
+  if (pTableScanNode->pGroupTags) {
     taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid);
   }
 

source/libs/executor/src/sortoperator.c

@@ -142,7 +142,8 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
 
 SSDataBlock* loadNextDataBlock(void* param) {
   SOperatorInfo* pOperator = (SOperatorInfo*)param;
-  return pOperator->fpSet.getNextFn(pOperator);
+  SSDataBlock*   pBlock = pOperator->fpSet.getNextFn(pOperator);
+  return pBlock;
 }
 
 // todo refactor: merged with fetch fp
@@ -442,7 +443,7 @@ void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) {
 SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
                                            SExecTaskInfo* pTaskInfo) {
   SGroupSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupSortOperatorInfo));
-  SOperatorInfo*     pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  SOperatorInfo*          pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
   if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) {
     goto _error;
   }
@@ -474,8 +475,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
   pOperator->exprSupp.numOfExprs = numOfCols;
   pOperator->pTaskInfo = pTaskInfo;
 
-  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, NULL,
+  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo,
-                                         NULL, getGroupSortExplainExecInfo);
+                                         NULL, NULL, getGroupSortExplainExecInfo);
 
   int32_t code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
@@ -493,7 +494,7 @@ _error:
 
 //=====================================================================================
 // Multiway Sort Merge operator
-typedef struct SMultiwaySortMergeOperatorInfo {
+typedef struct SMultiwayMergeOperatorInfo {
   SOptrBasicInfo binfo;
 
   int32_t  bufPageSize;
@@ -503,14 +504,17 @@ typedef struct SMultiwaySortMergeOperatorInfo {
   SSortHandle* pSortHandle;
   SArray*      pColMatchInfo;  // for index map from table scan output
 
-  SSDataBlock* pInputBlock;
+  SSDataBlock*  pInputBlock;
-  int64_t      startTs;  // sort start time
+  int64_t       startTs;  // sort start time
-  uint64_t     groupId;
+  bool          groupSort;
-} SMultiwaySortMergeOperatorInfo;
+  bool          hasGroupId;
+  uint64_t      groupId;
+  STupleHandle* prefetchedTuple;
+} SMultiwayMergeOperatorInfo;
 
-int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
+int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
-  SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
+  SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
-  SExecTaskInfo*                  pTaskInfo = pOperator->pTaskInfo;
+  SExecTaskInfo*              pTaskInfo = pOperator->pTaskInfo;
 
   if (OPTR_IS_OPENED(pOperator)) {
     return TSDB_CODE_SUCCESS;
@@ -524,8 +528,9 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
                                              pInfo->pInputBlock, pTaskInfo->id.str);
 
   tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL);
-  tsortSetCompareGroupId(pInfo->pSortHandle, true);
+
-  
+  tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort);
+
   for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) {
     SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
     ps->param = pOperator->pDownstream[i];
@@ -547,8 +552,8 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) {
 
 SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
                                         SArray* pColMatchInfo, SOperatorInfo* pOperator) {
-  SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
+  SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
-  SExecTaskInfo*                  pTaskInfo = pOperator->pTaskInfo;
+  SExecTaskInfo*              pTaskInfo = pOperator->pTaskInfo;
 
   blockDataCleanup(pDataBlock);
 
@@ -560,18 +565,47 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
   blockDataEnsureCapacity(p, capacity);
 
   while (1) {
-    STupleHandle* pTupleHandle = tsortNextTuple(pHandle);
+    STupleHandle* pTupleHandle = NULL;
+    if (pInfo->groupSort) {
+      if (pInfo->prefetchedTuple == NULL) {
+        pTupleHandle = tsortNextTuple(pHandle);
+      } else {
+        pTupleHandle = pInfo->prefetchedTuple;
+        pInfo->groupId = tsortGetGroupId(pTupleHandle);
+        pInfo->prefetchedTuple = NULL;
+      }
+    }
+    else {
+      pTupleHandle = tsortNextTuple(pHandle);
+      pInfo->groupId = 0;
+    }
+
     if (pTupleHandle == NULL) {
       break;
     }
 
-    appendOneRowToDataBlock(p, pTupleHandle);
+    if (pInfo->groupSort)
+    {
+      uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
+      if (!pInfo->hasGroupId) {
+        pInfo->groupId = tupleGroupId;
+        pInfo->hasGroupId = true;
+        appendOneRowToDataBlock(p, pTupleHandle);
+      } else if (pInfo->groupId == tupleGroupId) {
+        appendOneRowToDataBlock(p, pTupleHandle);
+      } else {
+        pInfo->prefetchedTuple = pTupleHandle;
+        break;
+      }
+    } else {
+      appendOneRowToDataBlock(p, pTupleHandle);
+    }
     if (p->info.rows >= capacity) {
       break;
     }
   }
 
-  if (p->info.rows > 0) {// todo extract method
+  if (p->info.rows > 0) {  // todo extract method
     blockDataEnsureCapacity(pDataBlock, p->info.rows);
     int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
     for (int32_t i = 0; i < numOfCols; ++i) {
@@ -593,13 +627,13 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
   return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
 }
 
-SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
+SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) {
   if (pOperator->status == OP_EXEC_DONE) {
     return NULL;
   }
 
-  SExecTaskInfo*                  pTaskInfo = pOperator->pTaskInfo;
+  SExecTaskInfo*              pTaskInfo = pOperator->pTaskInfo;
-  SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info;
+  SMultiwayMergeOperatorInfo* pInfo = pOperator->info;
 
   int32_t code = pOperator->fpSet._openFn(pOperator);
   if (code != TSDB_CODE_SUCCESS) {
@@ -608,7 +642,6 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
 
   SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes,
                                                    pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pOperator);
-
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
   } else {
@@ -617,8 +650,8 @@ SSDataBlock* doMultiwaySortMerge(SOperatorInfo* pOperator) {
   return pBlock;
 }
 
-void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) {
+void destroyMultiwayMergeOperatorInfo(void* param, int32_t numOfOutput) {
-  SMultiwaySortMergeOperatorInfo* pInfo = (SMultiwaySortMergeOperatorInfo*)param;
+  SMultiwayMergeOperatorInfo* pInfo = (SMultiwayMergeOperatorInfo*)param;
   pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
   pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock);
 
@@ -626,11 +659,11 @@ void destroyMultiwaySortMergeOperatorInfo(void* param, int32_t numOfOutput) {
   taosArrayDestroy(pInfo->pColMatchInfo);
 }
 
-int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
+int32_t getMultiwayMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
   ASSERT(pOptr != NULL);
   SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo));
 
-  SMultiwaySortMergeOperatorInfo* pOperatorInfo = (SMultiwaySortMergeOperatorInfo*)pOptr->info;
+  SMultiwayMergeOperatorInfo* pOperatorInfo = (SMultiwayMergeOperatorInfo*)pOptr->info;
 
   *pInfo = tsortGetSortExecInfo(pOperatorInfo->pSortHandle);
   *pOptrExplain = pInfo;
@@ -638,24 +671,35 @@ int32_t getMultiwaySortMergeExplainExecInfo(SOperatorInfo* pOptr, void** pOptrEx
   return TSDB_CODE_SUCCESS;
 }
 
-SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, int32_t numStreams,
+SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size_t numStreams,
-                                                   SSDataBlock* pInputBlock, SSDataBlock* pResBlock, SArray* pSortInfo,
+                                                SMergePhysiNode* pMergePhyNode, SExecTaskInfo* pTaskInfo) {
-                                                   SArray* pColMatchColInfo, SExecTaskInfo* pTaskInfo) {
+  SPhysiNode* pPhyNode = (SPhysiNode*)pMergePhyNode;
-  SMultiwaySortMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwaySortMergeOperatorInfo));
+
-  SOperatorInfo*                  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  SMultiwayMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwayMergeOperatorInfo));
-  int32_t                         rowSize = pResBlock->info.rowSize;
+  SOperatorInfo*              pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  SDataBlockDescNode*         pDescNode = pPhyNode->pOutputDataBlockDesc;
+  SSDataBlock*                pResBlock = createResDataBlock(pDescNode);
+
+  int32_t rowSize = pResBlock->info.rowSize;
 
   if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) {
     goto _error;
   }
 
+  SArray* pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys);
+  int32_t numOfOutputCols = 0;
+  SArray* pColMatchColInfo =
+      extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
+  SPhysiNode*  pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
+  SSDataBlock* pInputBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc);
   initResultSizeInfo(pOperator, 1024);
 
+  pInfo->groupSort = pMergePhyNode->groupSort;
   pInfo->binfo.pRes = pResBlock;
   pInfo->pSortInfo = pSortInfo;
   pInfo->pColMatchInfo = pColMatchColInfo;
   pInfo->pInputBlock = pInputBlock;
-  pOperator->name = "MultiwaySortMerge";
+  pOperator->name = "MultiwayMerge";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE;
   pOperator->blocking = false;
   pOperator->status = OP_NOT_OPENED;
@@ -667,9 +711,8 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams,
   // one additional is reserved for merged result.
   pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1);
 
-  pOperator->fpSet =
+  pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, NULL,
-      createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL,
+                                         destroyMultiwayMergeOperatorInfo, NULL, NULL, getMultiwayMergeExplainExecInfo);
-                          destroyMultiwaySortMergeOperatorInfo, NULL, NULL, getMultiwaySortMergeExplainExecInfo);
 
   int32_t code = appendDownstream(pOperator, downStreams, numStreams);
   if (code != TSDB_CODE_SUCCESS) {

source/libs/executor/src/tsort.c

@@ -752,6 +752,10 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) {
   }
 }
 
+uint64_t tsortGetGroupId(STupleHandle* pVHandle) {
+  return pVHandle->pBlock->info.groupId;
+}
+
 SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) {
   SSortExecInfo info = {0};
 

source/libs/function/src/udfd.c

@@ -110,7 +110,7 @@ static void    udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
 static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf);
 static int32_t udfdConnectToMnode();
 static int32_t udfdLoadUdf(char *udfName, SUdf *udf);
-static bool    udfdRpcRfp(int32_t code);
+static bool    udfdRpcRfp(int32_t code, tmsg_t msgType);
 static int     initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
 static int32_t udfdOpenClientRpc();
 static int32_t udfdCloseClientRpc();
@@ -546,9 +546,12 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
   }
   return 0;
 }
-static bool udfdRpcRfp(int32_t code) {
+static bool udfdRpcRfp(int32_t code, tmsg_t msgType) {
   if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
       code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
+    if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) {
+      return false;
+    } 
     return true;
   } else {
     return false;

source/libs/nodes/src/nodesCloneFuncs.c

@@ -351,7 +351,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
   COPY_SCALAR_FIELD(watermark);
   COPY_SCALAR_FIELD(tsColId);
   COPY_SCALAR_FIELD(filesFactor);
-  CLONE_NODE_LIST_FIELD(pPartTags);
+  CLONE_NODE_LIST_FIELD(pGroupTags);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -401,6 +401,7 @@ static int32_t logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst
   CLONE_NODE_LIST_FIELD(pInputs);
   COPY_SCALAR_FIELD(numOfChannels);
   COPY_SCALAR_FIELD(srcGroupId);
+  COPY_SCALAR_FIELD(groupSort);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -436,6 +437,7 @@ static int32_t logicFillCopy(const SFillLogicNode* pSrc, SFillLogicNode* pDst) {
 static int32_t logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) {
   COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
   CLONE_NODE_LIST_FIELD(pSortKeys);
+  COPY_SCALAR_FIELD(groupSort);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -500,7 +502,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy
   COPY_SCALAR_FIELD(ratio);
   COPY_SCALAR_FIELD(dataRequired);
   CLONE_NODE_LIST_FIELD(pDynamicScanFuncs);
-  CLONE_NODE_LIST_FIELD(pPartitionTags);
+  CLONE_NODE_LIST_FIELD(pGroupTags);
   COPY_SCALAR_FIELD(interval);
   COPY_SCALAR_FIELD(offset);
   COPY_SCALAR_FIELD(sliding);

source/libs/nodes/src/nodesCodeFuncs.c

@@ -234,6 +234,8 @@ const char* nodesNodeName(ENodeType type) {
       return "PhysiMerge";
     case QUERY_NODE_PHYSICAL_PLAN_SORT:
       return "PhysiSort";
+    case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
+      return "PhysiGroupSort";
     case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
       return "PhysiHashInterval";
     case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:
@@ -539,7 +541,7 @@ static const char* jkScanLogicPlanScanPseudoCols = "ScanPseudoCols";
 static const char* jkScanLogicPlanTableId = "TableId";
 static const char* jkScanLogicPlanTableType = "TableType";
 static const char* jkScanLogicPlanTagCond = "TagCond";
-static const char* jkScanLogicPlanPartTags = "PartTags";
+static const char* jkScanLogicPlanGroupTags = "GroupTags";
 
 static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
   const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
@@ -561,7 +563,7 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
     code = tjsonAddObject(pJson, jkScanLogicPlanTagCond, nodeToJson, pNode->pTagCond);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = nodeListToJson(pJson, jkScanLogicPlanPartTags, pNode->pPartTags);
+    code = nodeListToJson(pJson, jkScanLogicPlanGroupTags, pNode->pGroupTags);
   }
 
   return code;
@@ -588,7 +590,7 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
     code = jsonToNodeObject(pJson, jkScanLogicPlanTagCond, &pNode->pTagCond);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = jsonToNodeList(pJson, jkScanLogicPlanPartTags, &pNode->pPartTags);
+    code = jsonToNodeList(pJson, jkScanLogicPlanGroupTags, &pNode->pGroupTags);
   }
 
   return code;
@@ -1430,7 +1432,8 @@ static const char* jkTableScanPhysiPlanTriggerType = "triggerType";
 static const char* jkTableScanPhysiPlanWatermark = "watermark";
 static const char* jkTableScanPhysiPlanTsColId = "tsColId";
 static const char* jkTableScanPhysiPlanFilesFactor = "FilesFactor";
-static const char* jkTableScanPhysiPlanPartitionTags = "PartitionTags";
+static const char* jkTableScanPhysiPlanGroupTags = "GroupTags";
+static const char* jkTableScanPhysiPlanGroupSort = "GroupSort";
 
 static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
   const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj;
@@ -1485,7 +1488,10 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
     code = tjsonAddDoubleToObject(pJson, jkTableScanPhysiPlanFilesFactor, pNode->filesFactor);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = nodeListToJson(pJson, jkTableScanPhysiPlanPartitionTags, pNode->pPartitionTags);
+    code = nodeListToJson(pJson, jkTableScanPhysiPlanGroupTags, pNode->pGroupTags);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanGroupSort, pNode->groupSort);
   }
 
   return code;
@@ -1544,7 +1550,10 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
     code = tjsonGetDoubleValue(pJson, jkTableScanPhysiPlanFilesFactor, &pNode->filesFactor);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = jsonToNodeList(pJson, jkTableScanPhysiPlanPartitionTags, &pNode->pPartitionTags);
+    code = jsonToNodeList(pJson, jkTableScanPhysiPlanGroupTags, &pNode->pGroupTags);
+  }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanGroupSort, &pNode->groupSort);
   }
 
   return code;
@@ -1725,6 +1734,7 @@ static const char* jkMergePhysiPlanMergeKeys = "MergeKeys";
 static const char* jkMergePhysiPlanTargets = "Targets";
 static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels";
 static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId";
+static const char* jkMergePhysiPlanGroupSort = "GroupSort";
 
 static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) {
   const SMergePhysiNode* pNode = (const SMergePhysiNode*)pObj;
@@ -1742,6 +1752,9 @@ static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkMergePhysiPlanSrcGroupId, pNode->srcGroupId);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkMergePhysiPlanGroupSort, pNode->groupSort);
+  }
 
   return code;
 }
@@ -1762,6 +1775,9 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetIntValue(pJson, jkMergePhysiPlanSrcGroupId, &pNode->srcGroupId);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkMergePhysiPlanGroupSort, &pNode->groupSort);
+  }
 
   return code;
 }
@@ -3369,6 +3385,7 @@ static const char* jkSlotDescSlotId = "SlotId";
 static const char* jkSlotDescDataType = "DataType";
 static const char* jkSlotDescReserve = "Reserve";
 static const char* jkSlotDescOutput = "Output";
+static const char* jkSlotDescName = "Name";
 
 static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) {
   const SSlotDescNode* pNode = (const SSlotDescNode*)pObj;
@@ -3383,6 +3400,9 @@ static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddBoolToObject(pJson, jkSlotDescOutput, pNode->output);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddStringToObject(pJson, jkSlotDescName, pNode->name);
+  }
 
   return code;
 }
@@ -3400,6 +3420,9 @@ static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetBoolValue(pJson, jkSlotDescOutput, &pNode->output);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetStringValue(pJson, jkSlotDescName, pNode->name);
+  }
 
   return code;
 }
@@ -4137,6 +4160,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
     case QUERY_NODE_PHYSICAL_PLAN_MERGE:
       return physiMergeNodeToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_PLAN_SORT:
+    case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
       return physiSortNodeToJson(pObj, pJson);
     case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:
@@ -4280,6 +4304,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
     case QUERY_NODE_PHYSICAL_PLAN_MERGE:
       return jsonToPhysiMergeNode(pJson, pObj);
     case QUERY_NODE_PHYSICAL_PLAN_SORT:
+    case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
       return jsonToPhysiSortNode(pJson, pObj);
     case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
     case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:

source/libs/nodes/src/nodesTraverseFuncs.c

@@ -500,7 +500,8 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk
       }
       break;
     }
-    case QUERY_NODE_PHYSICAL_PLAN_SORT: {
+    case QUERY_NODE_PHYSICAL_PLAN_SORT:
+    case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: {
       SSortPhysiNode* pSort = (SSortPhysiNode*)pNode;
       res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext);
       if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {

source/libs/nodes/src/nodesUtilFuncs.c

@@ -288,6 +288,8 @@ SNode* nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SMergePhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_SORT:
       return makeNode(type, sizeof(SSortPhysiNode));
+    case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT:
+      return makeNode(type, sizeof(SGroupSortPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL:
       return makeNode(type, sizeof(SIntervalPhysiNode));
     case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL:
@@ -709,7 +711,7 @@ void nodesDestroyNode(SNode* pNode) {
       nodesDestroyNode(pLogicNode->pTagCond);
       nodesDestroyNode(pLogicNode->pTagIndexCond);
       taosArrayDestroy(pLogicNode->pSmaIndexes);
-      nodesDestroyList(pLogicNode->pPartTags);
+      nodesDestroyList(pLogicNode->pGroupTags);
       break;
     }
     case QUERY_NODE_LOGIC_PLAN_JOIN: {
@@ -813,7 +815,7 @@ void nodesDestroyNode(SNode* pNode) {
       STableScanPhysiNode* pPhyNode = (STableScanPhysiNode*)pNode;
       destroyScanPhysiNode((SScanPhysiNode*)pNode);
       nodesDestroyList(pPhyNode->pDynamicScanFuncs);
-      nodesDestroyList(pPhyNode->pPartitionTags);
+      nodesDestroyList(pPhyNode->pGroupTags);
       break;
     }
     case QUERY_NODE_PHYSICAL_PLAN_PROJECT: {
@@ -850,7 +852,8 @@ void nodesDestroyNode(SNode* pNode) {
       nodesDestroyList(pPhyNode->pTargets);
       break;
     }
-    case QUERY_NODE_PHYSICAL_PLAN_SORT: {
+    case QUERY_NODE_PHYSICAL_PLAN_SORT:
+    case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: {
       SSortPhysiNode* pPhyNode = (SSortPhysiNode*)pNode;
       destroyPhysiNode((SPhysiNode*)pPhyNode);
       nodesDestroyList(pPhyNode->pExprs);

source/libs/parser/src/parTranslater.c

@@ -2456,90 +2456,6 @@ static int32_t rewriteUniqueStmt(STranslateContext* pCxt, SSelectStmt* pSelect)
   return cxt.pTranslateCxt->errCode;
 }
 
-typedef struct SRwriteTailCxt {
-  STranslateContext* pTranslateCxt;
-  int64_t            limit;
-  int64_t            offset;
-} SRwriteTailCxt;
-
-static EDealRes rewriteTailFunc(SNode** pNode, void* pContext) {
-  SRwriteTailCxt* pCxt = pContext;
-  if (QUERY_NODE_FUNCTION == nodeType(*pNode)) {
-    SFunctionNode* pFunc = (SFunctionNode*)*pNode;
-    if (FUNCTION_TYPE_TAIL == pFunc->funcType) {
-      pCxt->limit = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 1))->datum.i;
-      if (3 == LIST_LENGTH(pFunc->pParameterList)) {
-        pCxt->offset = ((SValueNode*)nodesListGetNode(pFunc->pParameterList, 2))->datum.i;
-      }
-      SNode* pExpr = nodesListGetNode(pFunc->pParameterList, 0);
-      strcpy(((SExprNode*)pExpr)->aliasName, ((SExprNode*)*pNode)->aliasName);
-      NODES_CLEAR_LIST(pFunc->pParameterList);
-      nodesDestroyNode(*pNode);
-      *pNode = pExpr;
-      return DEAL_RES_IGNORE_CHILD;
-    }
-  }
-  return DEAL_RES_CONTINUE;
-}
-
-static int32_t createLimieNode(SRwriteTailCxt* pCxt, SLimitNode** pOutput) {
-  *pOutput = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT);
-  if (NULL == *pOutput) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  (*pOutput)->limit = pCxt->limit;
-  (*pOutput)->offset = pCxt->offset;
-  return TSDB_CODE_SUCCESS;
-}
-
-static SNode* createOrderByExpr(STranslateContext* pCxt) {
-  SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
-  if (NULL == pOrder) {
-    return NULL;
-  }
-  pCxt->errCode = createPrimaryKeyCol(pCxt, &pOrder->pExpr);
-  if (TSDB_CODE_SUCCESS != pCxt->errCode) {
-    nodesDestroyNode((SNode*)pOrder);
-    return NULL;
-  }
-  pOrder->order = ORDER_DESC;
-  pOrder->nullOrder = NULL_ORDER_FIRST;
-  return (SNode*)pOrder;
-}
-
-/* case 1:
- * in:  select tail(expr, k, f) from t where_clause
- * out: select expr from t where_clause order by _rowts desc limit k offset f
- *
- * case 2:
- * in:  select tail(expr, k, f) from t where_clause partition_by_clause
- * out: select expr from t where_clause partition_by_clause sort by _rowts desc limit k offset f
- *
- * case 3:
- * in:  select tail(expr, k, f) from t where_clause order_by_clause limit_clause
- * out: select expr from (
- *        select expr from t where_clause order by _rowts desc limit k offset f
- *      ) order_by_clause limit_clause
- *
- * case 4:
- * in:  select tail(expr, k, f) from t where_clause partition_by_clause limit_clause
- * out:
- */
-static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) {
-  if (!pSelect->hasTailFunc) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  SRwriteTailCxt cxt = {.pTranslateCxt = pCxt, .limit = -1, .offset = -1};
-  nodesRewriteExprs(pSelect->pProjectionList, rewriteTailFunc, &cxt);
-  int32_t code = nodesListMakeStrictAppend(&pSelect->pOrderByList, createOrderByExpr(pCxt));
-  if (TSDB_CODE_SUCCESS == code) {
-    code = createLimieNode(&cxt, &pSelect->pLimit);
-  }
-  pSelect->hasIndefiniteRowsFunc = false;
-  return code;
-}
-
 typedef struct SReplaceOrderByAliasCxt {
   STranslateContext* pTranslateCxt;
   SNodeList*         pProjectionList;
@@ -2616,9 +2532,6 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect
   if (TSDB_CODE_SUCCESS == code) {
     code = rewriteUniqueStmt(pCxt, pSelect);
   }
-  // if (TSDB_CODE_SUCCESS == code) {
-  //   code = rewriteTailStmt(pCxt, pSelect);
-  // }
   if (TSDB_CODE_SUCCESS == code) {
     code = rewriteTimelineFunc(pCxt, pSelect);
   }

source/libs/parser/src/parUtil.c

@@ -337,7 +337,11 @@ int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
 static bool isValidateTag(char* input) {
   if (!input) return false;
   for (size_t i = 0; i < strlen(input); ++i) {
+  #ifdef WINDOWS
+    if (input[i] < 0x20 || input[i] > 0x7E) return false;
+  #else
     if (isprint(input[i]) == 0) return false;
+  #endif
   }
   return true;
 }
@@ -377,6 +381,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, SMs
 
     char* jsonKey = item->string;
     if (!isValidateTag(jsonKey)) {
+      fprintf(stdout,"%s(%d) %s %08" PRId64 "\n", __FILE__, __LINE__,__func__,taosGetSelfPthreadId());fflush(stdout);
       retCode = buildSyntaxErrMsg(pMsgBuf, "json key not validate", jsonKey);
       goto end;
     }

source/libs/planner/src/planLogicCreater.c

@@ -507,6 +507,8 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc;
+
   // indefinite rows functions and _select_values functions
   int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs);
   if (TSDB_CODE_SUCCESS == code) {
@@ -733,6 +735,8 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  pSort->groupSort = pSelect->groupSort;
+
   int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets);
   if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) {
     code = nodesListMakeStrictAppend(&pSort->node.pTargets,

source/libs/planner/src/planPhysiCreater.c

@@ -41,8 +41,12 @@ typedef struct SPhysiPlanContext {
 static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
   if (QUERY_NODE_COLUMN == nodeType(pNode)) {
     SColumnNode* pCol = (SColumnNode*)pNode;
-    if (NULL != pStmtName && '\0' != pStmtName[0]) {
+    if (NULL != pStmtName) {
-      return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName);
+      if ('\0' != pStmtName[0]) {
+        return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName);
+      } else {
+        return sprintf(pKey, "%s", pCol->node.aliasName);
+      }
     }
     if ('\0' == pCol->tableAlias[0]) {
       return sprintf(pKey, "%s", pCol->colName);
@@ -56,11 +60,13 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
   return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName);
 }
 
-static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId, bool output, bool reserve) {
+static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const char* pName, const SNode* pNode, int16_t slotId,
+                             bool output, bool reserve) {
   SSlotDescNode* pSlot = (SSlotDescNode*)nodesMakeNode(QUERY_NODE_SLOT_DESC);
   if (NULL == pSlot) {
     return NULL;
   }
+  strcpy(pSlot->name, pName);
   pSlot->slotId = slotId;
   pSlot->dataType = ((SExprNode*)pNode)->resType;
   pSlot->reserve = reserve;
@@ -99,10 +105,8 @@ static int32_t putSlotToHashImpl(int16_t dataBlockId, int16_t slotId, const char
   return taosHashPut(pHash, pName, len, &index, sizeof(SSlotIndex));
 }
 
-static int32_t putSlotToHash(int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) {
+static int32_t putSlotToHash(const char* pName, int16_t dataBlockId, int16_t slotId, SNode* pNode, SHashObj* pHash) {
-  char    name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
+  return putSlotToHashImpl(dataBlockId, slotId, pName, strlen(pName), pHash);
-  int32_t len = getSlotKey(pNode, NULL, name);
-  return putSlotToHashImpl(dataBlockId, slotId, name, len, pHash);
 }
 
 static int32_t createDataBlockDescHash(SPhysiPlanContext* pCxt, int32_t capacity, int16_t dataBlockId,
@@ -131,9 +135,11 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD
   int16_t slotId = 0;
   SNode*  pNode = NULL;
   FOREACH(pNode, pList) {
-    code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, slotId, true, false));
+    char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN];
+    getSlotKey(pNode, NULL, name);
+    code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pNode, slotId, true, false));
     if (TSDB_CODE_SUCCESS == code) {
-      code = putSlotToHash(pDataBlockDesc->dataBlockId, slotId, pNode, pHash);
+      code = putSlotToHash(name, pDataBlockDesc->dataBlockId, slotId, pNode, pHash);
     }
     if (TSDB_CODE_SUCCESS == code) {
       pDataBlockDesc->totalRowSize += ((SExprNode*)pNode)->resType.bytes;
@@ -196,7 +202,8 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList,
     int32_t     len = getSlotKey(pExpr, pStmtName, name);
     SSlotIndex* pIndex = taosHashGet(pHash, name, len);
     if (NULL == pIndex) {
-      code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pExpr, nextSlotId, output, reserve));
+      code =
+          nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, name, pExpr, nextSlotId, output, reserve));
       if (TSDB_CODE_SUCCESS == code) {
         code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash);
       }
@@ -513,12 +520,13 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp
   tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName);
   pTableScan->dataRequired = pScanLogicNode->dataRequired;
   pTableScan->pDynamicScanFuncs = nodesCloneList(pScanLogicNode->pDynamicScanFuncs);
-  pTableScan->pPartitionTags = nodesCloneList(pScanLogicNode->pPartTags);
+  pTableScan->pGroupTags = nodesCloneList(pScanLogicNode->pGroupTags);
   if ((NULL != pScanLogicNode->pDynamicScanFuncs && NULL == pTableScan->pDynamicScanFuncs) ||
-      (NULL != pScanLogicNode->pPartTags && NULL == pTableScan->pPartitionTags)) {
+      (NULL != pScanLogicNode->pGroupTags && NULL == pTableScan->pGroupTags)) {
     nodesDestroyNode((SNode*)pTableScan);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pTableScan->groupSort = pScanLogicNode->groupSort;
   pTableScan->interval = pScanLogicNode->interval;
   pTableScan->offset = pScanLogicNode->offset;
   pTableScan->sliding = pScanLogicNode->sliding;
@@ -1170,8 +1178,9 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr
 
 static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SSortLogicNode* pSortLogicNode,
                                    SPhysiNode** pPhyNode) {
-  SSortPhysiNode* pSort =
+  SSortPhysiNode* pSort = (SSortPhysiNode*)makePhysiNode(
-      (SSortPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pSortLogicNode, QUERY_NODE_PHYSICAL_PLAN_SORT);
+      pCxt, (SLogicNode*)pSortLogicNode,
+      pSortLogicNode->groupSort ? QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT : QUERY_NODE_PHYSICAL_PLAN_SORT);
   if (NULL == pSort) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
@@ -1185,7 +1194,7 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
   if (TSDB_CODE_SUCCESS == code && NULL != pPrecalcExprs) {
     code = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs, &pSort->pExprs);
     if (TSDB_CODE_SUCCESS == code) {
-      code = addDataBlockSlots(pCxt, pSort->pExprs, pChildTupe);
+      code = pushdownDataBlockSlots(pCxt, pSort->pExprs, pChildTupe);
     }
   }
 
@@ -1322,6 +1331,7 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM
 
   pMerge->numOfChannels = pMergeLogicNode->numOfChannels;
   pMerge->srcGroupId = pMergeLogicNode->srcGroupId;
+  pMerge->groupSort = pMergeLogicNode->groupSort;
 
   int32_t code = addDataBlockSlots(pCxt, pMergeLogicNode->pInputs, pMerge->node.pOutputDataBlockDesc);
 

source/libs/planner/src/planSpliter.c

@@ -362,7 +362,7 @@ static int32_t stbSplGetNumOfVgroups(SLogicNode* pNode) {
 }
 
 static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SLogicNode* pSplitNode,
-                                     SNodeList* pMergeKeys, SLogicNode* pPartChild) {
+                                     SNodeList* pMergeKeys, SLogicNode* pPartChild, bool groupSort) {
   SMergeLogicNode* pMerge = (SMergeLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE);
   if (NULL == pMerge) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -371,6 +371,7 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla
   pMerge->srcGroupId = pCxt->groupId;
   pMerge->node.precision = pPartChild->precision;
   pMerge->pMergeKeys = pMergeKeys;
+  pMerge->groupSort = groupSort;
 
   int32_t code = TSDB_CODE_SUCCESS;
   pMerge->pInputs = nodesCloneList(pPartChild->pTargets);
@@ -430,7 +431,7 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo
     SNodeList* pMergeKeys = NULL;
     code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, &pMergeKeys);
     if (TSDB_CODE_SUCCESS == code) {
-      code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow);
+      code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, false);
     }
     if (TSDB_CODE_SUCCESS != code) {
       nodesDestroyList(pMergeKeys);
@@ -497,12 +498,16 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo
   return code;
 }
 
-static void splSetTableScanType(SLogicNode* pNode, EScanType scanType) {
+static void stbSplSetTableMergeScan(SLogicNode* pNode) {
   if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
-    ((SScanLogicNode*)pNode)->scanType = scanType;
+    SScanLogicNode* pScan = (SScanLogicNode*)pNode;
+    pScan->scanType = SCAN_TYPE_TABLE_MERGE;
+    if (NULL != pScan->pGroupTags) {
+      pScan->groupSort = true;
+    }
   } else {
     if (1 == LIST_LENGTH(pNode->pChildren)) {
-      splSetTableScanType((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), scanType);
+      stbSplSetTableMergeScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0));
     }
   }
 }
@@ -515,7 +520,7 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl
   int32_t    code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys);
 
   if (TSDB_CODE_SUCCESS == code) {
-    code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild);
+    code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild, true);
   }
 
   if (TSDB_CODE_SUCCESS == code) {
@@ -524,13 +529,10 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl
   }
 
   if (TSDB_CODE_SUCCESS == code) {
-    splSetTableScanType(pChild, SCAN_TYPE_TABLE_MERGE);
+    stbSplSetTableMergeScan(pChild);
-    ++(pCxt->groupId);
-  }
-
-  if (TSDB_CODE_SUCCESS == code) {
     pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
     SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT);
+    ++(pCxt->groupId);
   } else {
     nodesDestroyList(pMergeKeys);
   }
@@ -560,7 +562,7 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
 
 static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) {
   if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
-    return ((SScanLogicNode*)pNode)->pPartTags;
+    return ((SScanLogicNode*)pNode)->pGroupTags;
   } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
     return ((SPartitionLogicNode*)pNode)->pPartitionKeys;
   } else {
@@ -775,6 +777,7 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut
     pPartSort->node.pChildren = pChildren;
     splSetParent((SLogicNode*)pPartSort);
     pPartSort->pSortKeys = pSortKeys;
+    pPartSort->groupSort = pSort->groupSort;
     code = stbSplCreateMergeKeys(pPartSort->pSortKeys, pPartSort->node.pTargets, &pMergeKeys);
   }
 
@@ -789,12 +792,29 @@ static int32_t stbSplCreatePartSortNode(SSortLogicNode* pSort, SLogicNode** pOut
   return code;
 }
 
+static void stbSplSetScanPartSort(SLogicNode* pNode) {
+  if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
+    SScanLogicNode* pScan = (SScanLogicNode*)pNode;
+    if (NULL != pScan->pGroupTags) {
+      pScan->groupSort = true;
+    }
+  } else {
+    if (1 == LIST_LENGTH(pNode->pChildren)) {
+      stbSplSetScanPartSort((SLogicNode*)nodesListGetNode(pNode->pChildren, 0));
+    }
+  }
+}
+
 static int32_t stbSplSplitSortNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
   SLogicNode* pPartSort = NULL;
   SNodeList*  pMergeKeys = NULL;
+  bool        groupSort = ((SSortLogicNode*)pInfo->pSplitNode)->groupSort;
   int32_t     code = stbSplCreatePartSortNode((SSortLogicNode*)pInfo->pSplitNode, &pPartSort, &pMergeKeys);
   if (TSDB_CODE_SUCCESS == code) {
-    code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort);
+    code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, pMergeKeys, pPartSort, groupSort);
+  }
+  if (TSDB_CODE_SUCCESS == code && groupSort) {
+    stbSplSetScanPartSort(pPartSort);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
@@ -829,7 +849,7 @@ static int32_t stbSplSplitScanNodeForJoin(SSplitContext* pCxt, SLogicSubplan* pS
   SNodeList* pMergeKeys = NULL;
   int32_t    code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pScan), &pMergeKeys);
   if (TSDB_CODE_SUCCESS == code) {
-    code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan);
+    code = stbSplCreateMergeNode(pCxt, pSubplan, (SLogicNode*)pScan, pMergeKeys, (SLogicNode*)pScan, false);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = nodesListMakeStrictAppend(&pSubplan->pChildren,

source/libs/planner/test/planBasicTest.cpp

@@ -67,6 +67,14 @@ TEST_F(PlanBasicTest, tailFunc) {
   run("SELECT TAIL(c1, 10) FROM t1");
 
   run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10");
+
+  run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1");
+
+  run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 ORDER BY 1");
+
+  run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 LIMIT 5");
+
+  run("SELECT TAIL(c2 + 10, 10, 80) FROM t1 WHERE c1 > 10 PARTITION BY c1 LIMIT 5");
 }
 
 TEST_F(PlanBasicTest, interpFunc) {

source/libs/planner/test/planTestMain.cpp

@@ -76,6 +76,7 @@ static void parseArg(int argc, char* argv[]) {
   static struct option long_options[] = {
     {"dump", optional_argument, NULL, 'd'},
     {"skipSql", required_argument, NULL, 's'},
+    {"limitSql", required_argument, NULL, 'i'},
     {"log", required_argument, NULL, 'l'},
     {0, 0, 0, 0}
   };
@@ -88,6 +89,9 @@ static void parseArg(int argc, char* argv[]) {
       case 's':
         setSkipSqlNum(optarg);
         break;
+      case 'i':
+        setLimitSqlNum(optarg);
+        break;
       case 'l':
         setLogLevel(optarg);
         break;

source/libs/planner/test/planTestUtil.cpp

@@ -51,6 +51,7 @@ enum DumpModule {
 
 DumpModule g_dumpModule = DUMP_MODULE_NOTHING;
 int32_t    g_skipSql = 0;
+int32_t    g_limitSql = 0;
 int32_t    g_logLevel = 131;
 
 void setDumpModule(const char* pModule) {
@@ -76,28 +77,33 @@ void setDumpModule(const char* pModule) {
 }
 
 void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(pNum); }
-
+void setLimitSqlNum(const char* pNum) { g_limitSql = stoi(pNum); }
 void setLogLevel(const char* pLogLevel) { g_logLevel = stoi(pLogLevel); }
 
 int32_t getLogLevel() { return g_logLevel; }
 
 class PlannerTestBaseImpl {
  public:
-  PlannerTestBaseImpl() : sqlNo_(0) {}
+  PlannerTestBaseImpl() : sqlNo_(0), sqlNum_(0) {}
 
   void useDb(const string& user, const string& db) {
     caseEnv_.acctId_ = 0;
     caseEnv_.user_ = user;
     caseEnv_.db_ = db;
-    caseEnv_.nsql_ = g_skipSql;
+    caseEnv_.numOfSkipSql_ = g_skipSql;
+    caseEnv_.numOfLimitSql_ = g_limitSql;
   }
 
   void run(const string& sql) {
     ++sqlNo_;
-    if (caseEnv_.nsql_ > 0) {
+    if (caseEnv_.numOfSkipSql_ > 0) {
-      --(caseEnv_.nsql_);
+      --(caseEnv_.numOfSkipSql_);
       return;
     }
+    if (caseEnv_.numOfLimitSql_ > 0 && caseEnv_.numOfLimitSql_ == sqlNum_) {
+      return;
+    }
+    ++sqlNum_;
 
     reset();
     try {
@@ -134,7 +140,7 @@ class PlannerTestBaseImpl {
   }
 
   void prepare(const string& sql) {
-    if (caseEnv_.nsql_ > 0) {
+    if (caseEnv_.numOfSkipSql_ > 0) {
       return;
     }
 
@@ -148,7 +154,7 @@ class PlannerTestBaseImpl {
   }
 
   void bindParams(TAOS_MULTI_BIND* pParams, int32_t colIdx) {
-    if (caseEnv_.nsql_ > 0) {
+    if (caseEnv_.numOfSkipSql_ > 0) {
       return;
     }
 
@@ -161,8 +167,8 @@ class PlannerTestBaseImpl {
   }
 
   void exec() {
-    if (caseEnv_.nsql_ > 0) {
+    if (caseEnv_.numOfSkipSql_ > 0) {
-      --(caseEnv_.nsql_);
+      --(caseEnv_.numOfSkipSql_);
       return;
     }
 
@@ -197,9 +203,10 @@ class PlannerTestBaseImpl {
     int32_t acctId_;
     string  user_;
     string  db_;
-    int32_t nsql_;
+    int32_t numOfSkipSql_;
+    int32_t numOfLimitSql_;
 
-    caseEnv() : nsql_(0) {}
+    caseEnv() : numOfSkipSql_(0) {}
   };
 
   struct stmtEnv {
@@ -401,6 +408,7 @@ class PlannerTestBaseImpl {
   stmtEnv stmtEnv_;
   stmtRes res_;
   int32_t sqlNo_;
+  int32_t sqlNum_;
 };
 
 PlannerTestBase::PlannerTestBase() : impl_(new PlannerTestBaseImpl()) {}

source/libs/planner/test/planTestUtil.h

@@ -43,6 +43,7 @@ class PlannerTestBase : public testing::Test {
 
 extern void    setDumpModule(const char* pModule);
 extern void    setSkipSqlNum(const char* pNum);
+extern void    setLimitSqlNum(const char* pNum);
 extern void    setLogLevel(const char* pLogLevel);
 extern int32_t getLogLevel();
 

source/libs/sync/inc/syncRaftEntry.h

@@ -56,6 +56,31 @@ void syncEntryPrint2(char* s, const SSyncRaftEntry* pObj);
 void syncEntryLog(const SSyncRaftEntry* pObj);
 void syncEntryLog2(char* s, const SSyncRaftEntry* pObj);
 
+//-----------------------------------
+typedef struct SRaftEntryCache {
+  SHashObj*     pEntryHash;
+  int32_t       maxCount;
+  int32_t       currentCount;
+  TdThreadMutex mutex;
+  SSyncNode*    pSyncNode;
+} SRaftEntryCache;
+
+SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount);
+void             raftCacheDestroy(SRaftEntryCache* pCache);
+int32_t          raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry);
+int32_t          raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry);
+int32_t          raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry);
+int32_t          raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index);
+int32_t          raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry);
+int32_t          raftCacheClear(struct SRaftEntryCache* pCache);
+
+cJSON* raftCache2Json(SRaftEntryCache* pObj);
+char*  raftCache2Str(SRaftEntryCache* pObj);
+void   raftCachePrint(SRaftEntryCache* pObj);
+void   raftCachePrint2(char* s, SRaftEntryCache* pObj);
+void   raftCacheLog(SRaftEntryCache* pObj);
+void   raftCacheLog2(char* s, SRaftEntryCache* pObj);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/sync/inc/syncSnapshot.h

@@ -35,12 +35,13 @@ extern "C" {
 
 #define SYNC_SNAPSHOT_RETRY_MS 5000
 
+//---------------------------------------------------
 typedef struct SSyncSnapshotSender {
   bool       start;
   int32_t    seq;
   int32_t    ack;
-  void *     pReader;
+  void      *pReader;
-  void *     pCurrentBlock;
+  void      *pCurrentBlock;
   int32_t    blockLen;
   SSnapshot  snapshot;
   SSyncCfg   lastConfig;
@@ -59,33 +60,36 @@ void                 snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot
 void                 snapshotSenderStop(SSyncSnapshotSender *pSender);
 int32_t              snapshotSend(SSyncSnapshotSender *pSender);
 int32_t              snapshotReSend(SSyncSnapshotSender *pSender);
-cJSON *              snapshotSender2Json(SSyncSnapshotSender *pSender);
-char *               snapshotSender2Str(SSyncSnapshotSender *pSender);
-char *               snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event);
 
+cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender);
+char  *snapshotSender2Str(SSyncSnapshotSender *pSender);
+char  *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event);
+
+//---------------------------------------------------
 typedef struct SSyncSnapshotReceiver {
-  bool start;
+  bool       start;
-
+  int32_t    ack;
-  int32_t   ack;
+  void      *pWriter;
-  void *    pWriter;
+  SyncTerm   term;
-  SyncTerm  term;
+  SyncTerm   privateTerm;
-  SyncTerm  privateTerm;
+  SSnapshot  snapshot;
-  SSnapshot snapshot;
-
-  SSyncNode *pSyncNode;
   SRaftId    fromId;
+  SSyncNode *pSyncNode;
 
 } SSyncSnapshotReceiver;
 
 SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId);
 void                   snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver);
-void   snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg);
+void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg);
-bool   snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver);
+bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver);
-void   snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply);
+void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply);
-cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver);
-char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver);
-char * snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event);
 
+cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver);
+char  *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver);
+char  *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event);
+
+//---------------------------------------------------
+// on message
 int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg);
 int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg);
 

source/libs/sync/src/syncMain.c

@@ -1355,11 +1355,16 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
   int32_t userStrLen = strlen(str);
 
   SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
-  if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
+  if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
     pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
   }
-  SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
+
-  SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore);
+  SyncIndex logLastIndex = SYNC_INDEX_INVALID;
+  SyncIndex logBeginIndex = SYNC_INDEX_INVALID;
+  if (pSyncNode->pLogStore != NULL) {
+    logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
+    logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore);
+  }
 
   char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg));
   char* printStr = "";

source/libs/sync/src/syncRaftEntry.c

@@ -180,3 +180,247 @@ void syncEntryLog2(char* s, const SSyncRaftEntry* pObj) {
   sTrace("syncEntryLog2 | len:%zu | %s | %s", strlen(serialized), s, serialized);
   taosMemoryFree(serialized);
 }
+
+//-----------------------------------
+SRaftEntryCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) {
+  SRaftEntryCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryCache));
+  if (pCache == NULL) {
+    sError("vgId:%d raft cache create error", pSyncNode->vgId);
+    return NULL;
+  }
+
+  pCache->pEntryHash =
+      taosHashInit(sizeof(SyncIndex), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  if (pCache->pEntryHash == NULL) {
+    sError("vgId:%d raft cache create hash error", pSyncNode->vgId);
+    return NULL;
+  }
+
+  taosThreadMutexInit(&(pCache->mutex), NULL);
+  pCache->maxCount = maxCount;
+  pCache->currentCount = 0;
+  pCache->pSyncNode = pSyncNode;
+
+  return pCache;
+}
+
+void raftCacheDestroy(SRaftEntryCache* pCache) {
+  if (pCache != NULL) {
+    taosThreadMutexLock(&(pCache->mutex));
+    taosHashCleanup(pCache->pEntryHash);
+    taosThreadMutexUnlock(&(pCache->mutex));
+    taosThreadMutexDestroy(&(pCache->mutex));
+    taosMemoryFree(pCache);
+  }
+}
+
+// success, return 1
+// max count, return 0
+// error, return -1
+int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry) {
+  taosThreadMutexLock(&(pCache->mutex));
+
+  if (pCache->currentCount >= pCache->maxCount) {
+    taosThreadMutexUnlock(&(pCache->mutex));
+    return 0;
+  }
+
+  taosHashPut(pCache->pEntryHash, &(pEntry->index), sizeof(pEntry->index), pEntry, pEntry->bytes);
+  ++(pCache->currentCount);
+
+  do {
+    char eventLog[128];
+    snprintf(eventLog, sizeof(eventLog), "raft cache add, type:%s,%d, type2:%s,%d, index:%ld, bytes:%d",
+             TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType,
+             pEntry->index, pEntry->bytes);
+    syncNodeEventLog(pCache->pSyncNode, eventLog);
+  } while (0);
+
+  taosThreadMutexUnlock(&(pCache->mutex));
+  return 1;
+}
+
+// success, return 0
+// error, return -1
+// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST
+int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) {
+  if (ppEntry == NULL) {
+    return -1;
+  }
+  *ppEntry = NULL;
+
+  taosThreadMutexLock(&(pCache->mutex));
+  void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index));
+  if (pTmp != NULL) {
+    SSyncRaftEntry* pEntry = pTmp;
+    *ppEntry = taosMemoryMalloc(pEntry->bytes);
+    memcpy(*ppEntry, pTmp, pEntry->bytes);
+
+    do {
+      char eventLog[128];
+      snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld",
+               TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType),
+               (*ppEntry)->originalRpcType, (*ppEntry)->index);
+      syncNodeEventLog(pCache->pSyncNode, eventLog);
+    } while (0);
+
+    taosThreadMutexUnlock(&(pCache->mutex));
+    return 0;
+  }
+
+  taosThreadMutexUnlock(&(pCache->mutex));
+  terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
+  return -1;
+}
+
+// success, return 0
+// error, return -1
+// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST
+int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) {
+  if (ppEntry == NULL) {
+    return -1;
+  }
+  *ppEntry = NULL;
+
+  taosThreadMutexLock(&(pCache->mutex));
+  void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index));
+  if (pTmp != NULL) {
+    SSyncRaftEntry* pEntry = pTmp;
+    *ppEntry = pEntry;
+
+    do {
+      char eventLog[128];
+      snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld",
+               TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType),
+               (*ppEntry)->originalRpcType, (*ppEntry)->index);
+      syncNodeEventLog(pCache->pSyncNode, eventLog);
+    } while (0);
+
+    taosThreadMutexUnlock(&(pCache->mutex));
+    return 0;
+  }
+
+  taosThreadMutexUnlock(&(pCache->mutex));
+  terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
+  return -1;
+}
+
+int32_t raftCacheDelEntry(struct SRaftEntryCache* pCache, SyncIndex index) {
+  taosThreadMutexLock(&(pCache->mutex));
+  taosHashRemove(pCache->pEntryHash, &index, sizeof(index));
+  --(pCache->currentCount);
+  taosThreadMutexUnlock(&(pCache->mutex));
+  return 0;
+}
+
+int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) {
+  if (ppEntry == NULL) {
+    return -1;
+  }
+  *ppEntry = NULL;
+
+  taosThreadMutexLock(&(pCache->mutex));
+  void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index));
+  if (pTmp != NULL) {
+    SSyncRaftEntry* pEntry = pTmp;
+    *ppEntry = taosMemoryMalloc(pEntry->bytes);
+    memcpy(*ppEntry, pTmp, pEntry->bytes);
+
+    do {
+      char eventLog[128];
+      snprintf(eventLog, sizeof(eventLog), "raft cache get-and-del, type:%s,%d, type2:%s,%d, index:%ld",
+               TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType),
+               (*ppEntry)->originalRpcType, (*ppEntry)->index);
+      syncNodeEventLog(pCache->pSyncNode, eventLog);
+    } while (0);
+
+    taosHashRemove(pCache->pEntryHash, &index, sizeof(index));
+    --(pCache->currentCount);
+
+    taosThreadMutexUnlock(&(pCache->mutex));
+    return 0;
+  }
+
+  taosThreadMutexUnlock(&(pCache->mutex));
+  terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
+  return -1;
+}
+
+int32_t raftCacheClear(struct SRaftEntryCache* pCache) {
+  taosThreadMutexLock(&(pCache->mutex));
+  taosHashClear(pCache->pEntryHash);
+  pCache->currentCount = 0;
+  taosThreadMutexUnlock(&(pCache->mutex));
+  return 0;
+}
+
+//-----------------------------------
+cJSON* raftCache2Json(SRaftEntryCache* pCache) {
+  char   u64buf[128] = {0};
+  cJSON* pRoot = cJSON_CreateObject();
+
+  if (pCache != NULL) {
+    taosThreadMutexLock(&(pCache->mutex));
+
+    snprintf(u64buf, sizeof(u64buf), "%p", pCache->pSyncNode);
+    cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
+    cJSON_AddNumberToObject(pRoot, "currentCount", pCache->currentCount);
+    cJSON_AddNumberToObject(pRoot, "maxCount", pCache->maxCount);
+    cJSON* pEntries = cJSON_CreateArray();
+    cJSON_AddItemToObject(pRoot, "entries", pEntries);
+
+    SSyncRaftEntry* pIter = (SSyncRaftEntry*)taosHashIterate(pCache->pEntryHash, NULL);
+    if (pIter != NULL) {
+      SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter;
+      cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry));
+    }
+    while (pIter) {
+      pIter = taosHashIterate(pCache->pEntryHash, pIter);
+      if (pIter != NULL) {
+        SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pIter;
+        cJSON_AddItemToArray(pEntries, syncEntry2Json(pEntry));
+      }
+    }
+
+    taosThreadMutexUnlock(&(pCache->mutex));
+  }
+
+  cJSON* pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SRaftEntryCache", pRoot);
+  return pJson;
+}
+
+char* raftCache2Str(SRaftEntryCache* pCache) {
+  cJSON* pJson = raftCache2Json(pCache);
+  char*  serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+void raftCachePrint(SRaftEntryCache* pCache) {
+  char* serialized = raftCache2Str(pCache);
+  printf("raftCachePrint | len:%lu | %s \n", strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void raftCachePrint2(char* s, SRaftEntryCache* pCache) {
+  char* serialized = raftCache2Str(pCache);
+  printf("raftCachePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void raftCacheLog(SRaftEntryCache* pCache) {
+  char* serialized = raftCache2Str(pCache);
+  sTrace("raftCacheLog | len:%lu | %s", strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void raftCacheLog2(char* s, SRaftEntryCache* pCache) {
+  if (gRaftDetailLog) {
+    char* serialized = raftCache2Str(pCache);
+    sTraceLong("raftCacheLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+    taosMemoryFree(serialized);
+  }
+}

source/libs/sync/src/syncRespMgr.c

@@ -32,11 +32,13 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) {
 }
 
 void syncRespMgrDestroy(SSyncRespMgr *pObj) {
-  taosThreadMutexLock(&(pObj->mutex));
+  if (pObj != NULL) {
-  taosHashCleanup(pObj->pRespHash);
+    taosThreadMutexLock(&(pObj->mutex));
-  taosThreadMutexUnlock(&(pObj->mutex));
+    taosHashCleanup(pObj->pRespHash);
-  taosThreadMutexDestroy(&(pObj->mutex));
+    taosThreadMutexUnlock(&(pObj->mutex));
-  taosMemoryFree(pObj);
+    taosThreadMutexDestroy(&(pObj->mutex));
+    taosMemoryFree(pObj);
+  }
 }
 
 int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) {

source/libs/sync/src/syncSnapshot.c

@@ -21,6 +21,7 @@
 #include "syncUtil.h"
 #include "wal.h"
 
+//----------------------------------
 static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm,
                                     SyncSnapshotSend *pBeginMsg);
 
@@ -49,7 +50,7 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI
     pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot));
     pSender->finish = false;
   } else {
-    sError("snapshotSenderCreate cannot create sender");
+    sError("vgId:%d cannot create snapshot sender", pSyncNode->vgId);
   }
 
   return pSender;
@@ -57,39 +58,59 @@ SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaI
 
 void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
   if (pSender != NULL) {
+    // free current block
     if (pSender->pCurrentBlock != NULL) {
       taosMemoryFree(pSender->pCurrentBlock);
+      pSender->pCurrentBlock = NULL;
     }
+
+    // close reader
+    if (pSender->pReader != NULL) {
+      int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader);
+      ASSERT(ret == 0);
+      pSender->pReader = NULL;
+    }
+
+    // free sender
     taosMemoryFree(pSender);
   }
 }
 
 bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; }
 
-// begin send snapshot (current term, seq begin)
+// begin send snapshot by snapshot, pReader
 void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void *pReader) {
   ASSERT(!snapshotSenderIsStart(pSender));
 
-  pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN;
-  pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID;
-
   // init snapshot and reader
   ASSERT(pSender->pReader == NULL);
   pSender->pReader = pReader;
   pSender->snapshot = snapshot;
 
+  // init current block
   if (pSender->pCurrentBlock != NULL) {
     taosMemoryFree(pSender->pCurrentBlock);
   }
   pSender->blockLen = 0;
 
+  // update term
+  pSender->term = pSender->pSyncNode->pRaftStore->currentTerm;
+  ++(pSender->privateTerm);
+
+  // update state
+  pSender->finish = false;
+  pSender->start = true;
+  pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN;
+  pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID;
+
+  // init last config
   if (pSender->snapshot.lastConfigIndex != SYNC_INDEX_INVALID) {
     int32_t         code = 0;
     SSyncRaftEntry *pEntry = NULL;
+    bool            getLastConfig = false;
+
     code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore,
                                                           pSender->snapshot.lastConfigIndex, &pEntry);
-
-    bool getLastConfig = false;
     if (code == 0) {
       ASSERT(pEntry != NULL);
 
@@ -112,29 +133,29 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void
       }
     }
 
+    // last config not found in wal, update to -1
     if (!getLastConfig) {
-      char logBuf[128];
+      SyncIndex oldLastConfigIndex = pSender->snapshot.lastConfigIndex;
-      snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to -1",
+      SyncIndex newLastConfigIndex = SYNC_INDEX_INVALID;
-               pSender->snapshot.lastConfigIndex);
+      pSender->snapshot.lastConfigIndex = SYNC_INDEX_INVALID;
-      pSender->snapshot.lastConfigIndex = -1;
-
-      char *eventLog = snapshotSender2SimpleStr(pSender, logBuf);
-      syncNodeEventLog(pSender->pSyncNode, eventLog);
-      taosMemoryFree(eventLog);
-
       memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg));
+
+      // event log
+      do {
+        char logBuf[128];
+        snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to %ld", oldLastConfigIndex,
+                 newLastConfigIndex);
+        char *eventLog = snapshotSender2SimpleStr(pSender, logBuf);
+        syncNodeEventLog(pSender->pSyncNode, eventLog);
+        taosMemoryFree(eventLog);
+      } while (0);
     }
 
   } else {
+    // no last config
     memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg));
   }
 
-  pSender->sendingMS = SYNC_SNAPSHOT_RETRY_MS;
-  pSender->term = pSender->pSyncNode->pRaftStore->currentTerm;
-  ++(pSender->privateTerm);
-  pSender->finish = false;
-  pSender->start = true;
-
   // build begin msg
   SyncSnapshotSend *pMsg = syncSnapshotSendBuild(0, pSender->pSyncNode->vgId);
   pMsg->srcId = pSender->pSyncNode->myRaftId;
@@ -151,40 +172,39 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void
   SRpcMsg rpcMsg;
   syncSnapshotSend2RpcMsg(pMsg, &rpcMsg);
   syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg);
-
-  char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender send");
-  syncNodeEventLog(pSender->pSyncNode, eventLog);
-  taosMemoryFree(eventLog);
-
   syncSnapshotSendDestroy(pMsg);
+
+  // event log
+  do {
+    char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender send");
+    syncNodeEventLog(pSender->pSyncNode, eventLog);
+    taosMemoryFree(eventLog);
+  } while (0);
 }
 
 void snapshotSenderStop(SSyncSnapshotSender *pSender) {
+  // close reader
   if (pSender->pReader != NULL) {
     int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStopRead(pSender->pSyncNode->pFsm, pSender->pReader);
     ASSERT(ret == 0);
     pSender->pReader = NULL;
   }
 
+  // free current block
   if (pSender->pCurrentBlock != NULL) {
     taosMemoryFree(pSender->pCurrentBlock);
     pSender->pCurrentBlock = NULL;
     pSender->blockLen = 0;
   }
 
+  // update flag
   pSender->start = false;
-
-  if (gRaftDetailLog) {
-    char *s = snapshotSender2Str(pSender);
-    sInfo("snapshotSenderStop %s", s);
-    taosMemoryFree(s);
-  }
 }
 
-// when sender receiver ack, call this function to send msg from seq
+// when sender receive ack, call this function to send msg from seq
 // seq = ack + 1, already updated
 int32_t snapshotSend(SSyncSnapshotSender *pSender) {
-  // free memory last time (seq - 1)
+  // free memory last time (current seq - 1)
   if (pSender->pCurrentBlock != NULL) {
     taosMemoryFree(pSender->pCurrentBlock);
     pSender->pCurrentBlock = NULL;
@@ -198,7 +218,7 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) {
   if (pSender->blockLen > 0) {
     // has read data
   } else {
-    // read finish
+    // read finish, update seq to end
     pSender->seq = SYNC_SNAPSHOT_SEQ_END;
   }
 
@@ -219,25 +239,28 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) {
   SRpcMsg rpcMsg;
   syncSnapshotSend2RpcMsg(pMsg, &rpcMsg);
   syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg);
-
-  if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) {
-    char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender finish");
-    syncNodeEventLog(pSender->pSyncNode, eventLog);
-    taosMemoryFree(eventLog);
-
-  } else {
-    char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender sending");
-    syncNodeEventLog(pSender->pSyncNode, eventLog);
-    taosMemoryFree(eventLog);
-  }
-
   syncSnapshotSendDestroy(pMsg);
+
+  // event log
+  do {
+    char *eventLog = NULL;
+    if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) {
+      eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender finish");
+    } else {
+      eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender sending");
+    }
+    syncNodeEventLog(pSender->pSyncNode, eventLog);
+    taosMemoryFree(eventLog);
+  } while (0);
+
   return 0;
 }
 
 // send snapshot data from cache
 int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
-  if (pSender->pCurrentBlock != NULL) {
+  // send current block data
+  if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) {
+    // build msg
     SyncSnapshotSend *pMsg = syncSnapshotSendBuild(pSender->blockLen, pSender->pSyncNode->vgId);
     pMsg->srcId = pSender->pSyncNode->myRaftId;
     pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex];
@@ -249,16 +272,20 @@ int32_t snapshotReSend(SSyncSnapshotSender *pSender) {
     pMsg->seq = pSender->seq;
     memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
 
+    // send msg
     SRpcMsg rpcMsg;
     syncSnapshotSend2RpcMsg(pMsg, &rpcMsg);
     syncNodeSendMsgById(&(pMsg->destId), pSender->pSyncNode, &rpcMsg);
-
-    char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender resend");
-    syncNodeEventLog(pSender->pSyncNode, eventLog);
-    taosMemoryFree(eventLog);
-
     syncSnapshotSendDestroy(pMsg);
+
+    // event log
+    do {
+      char *eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender resend");
+      syncNodeEventLog(pSender->pSyncNode, eventLog);
+      taosMemoryFree(eventLog);
+    } while (0);
   }
+
   return 0;
 }
 
@@ -294,7 +321,6 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) {
     snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyTerm);
     cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf);
     cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot);
-
     snprintf(u64buf, sizeof(u64buf), "%lu", pSender->sendingMS);
     cJSON_AddStringToObject(pRoot, "sendingMS", u64buf);
     snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode);
@@ -314,17 +340,17 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) {
 
 char *snapshotSender2Str(SSyncSnapshotSender *pSender) {
   cJSON *pJson = snapshotSender2Json(pSender);
-  char * serialized = cJSON_Print(pJson);
+  char  *serialized = cJSON_Print(pJson);
   cJSON_Delete(pJson);
   return serialized;
 }
 
 char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) {
   int32_t len = 256;
-  char *  s = taosMemoryMalloc(len);
+  char   *s = taosMemoryMalloc(len);
 
   SRaftId  destId = pSender->pSyncNode->replicasId[pSender->replicaIndex];
-  char     host[128];
+  char     host[64];
   uint16_t port;
   syncUtilU642Addr(destId.addr, host, sizeof(host), &port);
 
@@ -355,12 +381,12 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from
     pReceiver->term = pSyncNode->pRaftStore->currentTerm;
     pReceiver->privateTerm = 0;
     pReceiver->snapshot.data = NULL;
-    pReceiver->snapshot.lastApplyIndex = -1;
+    pReceiver->snapshot.lastApplyIndex = SYNC_INDEX_INVALID;
     pReceiver->snapshot.lastApplyTerm = 0;
-    pReceiver->snapshot.lastConfigIndex = -1;
+    pReceiver->snapshot.lastConfigIndex = SYNC_INDEX_INVALID;
 
   } else {
-    sInfo("snapshotReceiverCreate cannot create receiver");
+    sError("vgId:%d cannot create snapshot receiver", pSyncNode->vgId);
   }
 
   return pReceiver;
@@ -368,29 +394,55 @@ SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId from
 
 void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) {
   if (pReceiver != NULL) {
+    // close writer
+    if (pReceiver->pWriter != NULL) {
+      int32_t ret =
+          pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false);
+      ASSERT(ret == 0);
+      pReceiver->pWriter = NULL;
+    }
+
+    // free receiver
     taosMemoryFree(pReceiver);
   }
 }
 
 bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; }
 
-// begin receive snapshot msg (current term, seq begin)
+// static do start
+// receive first snapshot data
+// privateTerm, pBeginMsg
 static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm,
                                     SyncSnapshotSend *pBeginMsg) {
+  // update state
   pReceiver->term = pReceiver->pSyncNode->pRaftStore->currentTerm;
   pReceiver->privateTerm = privateTerm;
   pReceiver->ack = SYNC_SNAPSHOT_SEQ_BEGIN;
   pReceiver->fromId = pBeginMsg->srcId;
 
+  // update snapshot
   pReceiver->snapshot.lastApplyIndex = pBeginMsg->lastIndex;
   pReceiver->snapshot.lastApplyTerm = pBeginMsg->lastTerm;
   pReceiver->snapshot.lastConfigIndex = pBeginMsg->lastConfigIndex;
 
+  // write data
   ASSERT(pReceiver->pWriter == NULL);
   int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStartWrite(pReceiver->pSyncNode->pFsm, &(pReceiver->pWriter));
   ASSERT(ret == 0);
 }
 
+static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) {
+  // force close, abandon incomplete data
+  if (pReceiver->pWriter != NULL) {
+    int32_t ret =
+        pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false);
+    ASSERT(ret == 0);
+    pReceiver->pWriter = NULL;
+  }
+
+  pReceiver->start = false;
+}
+
 // if receiver receive msg from seq = SYNC_SNAPSHOT_SEQ_BEGIN, start receiver
 // if already start, force close, start again
 void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTerm, SyncSnapshotSend *pBeginMsg) {
@@ -413,12 +465,6 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateTer
     snapshotReceiverDoStart(pReceiver, privateTerm, pBeginMsg);
     pReceiver->start = true;
   }
-
-  if (gRaftDetailLog) {
-    char *s = snapshotReceiver2Str(pReceiver);
-    sInfo("snapshotReceiverStart %s", s);
-    taosMemoryFree(s);
-  }
 }
 
 void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) {
@@ -434,12 +480,6 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply) {
   if (apply) {
     //    ++(pReceiver->privateTerm);
   }
-
-  if (gRaftDetailLog) {
-    char *s = snapshotReceiver2Str(pReceiver);
-    sInfo("snapshotReceiverStop %s", s);
-    taosMemoryFree(s);
-  }
 }
 
 cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) {
@@ -461,7 +501,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) {
     cJSON_AddStringToObject(pFromId, "addr", u64buf);
     {
       uint64_t u64 = pReceiver->fromId.addr;
-      cJSON *  pTmp = pFromId;
+      cJSON   *pTmp = pFromId;
       char     host[128] = {0};
       uint16_t port;
       syncUtilU642Addr(u64, host, sizeof(host), &port);
@@ -494,14 +534,14 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) {
 
 char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) {
   cJSON *pJson = snapshotReceiver2Json(pReceiver);
-  char * serialized = cJSON_Print(pJson);
+  char  *serialized = cJSON_Print(pJson);
   cJSON_Delete(pJson);
   return serialized;
 }
 
 char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event) {
   int32_t len = 256;
-  char *  s = taosMemoryMalloc(len);
+  char   *s = taosMemoryMalloc(len);
 
   SRaftId  fromId = pReceiver->fromId;
   char     host[128];

source/libs/sync/test/CMakeLists.txt

@@ -17,6 +17,7 @@ add_executable(syncVotesRespondTest "")
 add_executable(syncIndexMgrTest "")
 add_executable(syncLogStoreTest "")
 add_executable(syncEntryTest "")
+add_executable(syncEntryCacheTest "")
 add_executable(syncRequestVoteTest "")
 add_executable(syncRequestVoteReplyTest "")
 add_executable(syncAppendEntriesTest "")
@@ -129,6 +130,10 @@ target_sources(syncEntryTest
     PRIVATE
     "syncEntryTest.cpp"
 )
+target_sources(syncEntryCacheTest
+    PRIVATE
+    "syncEntryCacheTest.cpp"
+)
 target_sources(syncRequestVoteTest
     PRIVATE
     "syncRequestVoteTest.cpp"
@@ -362,6 +367,11 @@ target_include_directories(syncEntryTest
     "${TD_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
+target_include_directories(syncEntryCacheTest
+    PUBLIC
+    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
 target_include_directories(syncRequestVoteTest
     PUBLIC
     "${TD_SOURCE_DIR}/include/libs/sync"
@@ -610,6 +620,10 @@ target_link_libraries(syncEntryTest
     sync
     gtest_main
 )
+target_link_libraries(syncEntryCacheTest
+    sync
+    gtest_main
+)
 target_link_libraries(syncRequestVoteTest
     sync
     gtest_main

source/libs/sync/test/syncEntryCacheTest.cpp

@@ -0,0 +1,162 @@
+#include <stdio.h>
+#include "syncEnv.h"
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncRaftLog.h"
+#include "syncRaftStore.h"
+#include "syncUtil.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+SSyncRaftEntry* createEntry(int i) {
+  int32_t         dataLen = 20;
+  SSyncRaftEntry* pEntry = syncEntryBuild(dataLen);
+  assert(pEntry != NULL);
+  pEntry->msgType = 88;
+  pEntry->originalRpcType = 99;
+  pEntry->seqNum = 3;
+  pEntry->isWeak = true;
+  pEntry->term = 100 + i;
+  pEntry->index = i;
+  snprintf(pEntry->data, dataLen, "value%d", i);
+
+  return pEntry;
+}
+
+SSyncNode* createFakeNode() {
+  SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode));
+  ASSERT(pSyncNode != NULL);
+  memset(pSyncNode, 0, sizeof(SSyncNode));
+
+  return pSyncNode;
+}
+
+SRaftEntryCache* createCache(int maxCount) {
+  SSyncNode* pSyncNode = createFakeNode();
+  ASSERT(pSyncNode != NULL);
+
+  SRaftEntryCache* pCache = raftCacheCreate(pSyncNode, maxCount);
+  ASSERT(pCache != NULL);
+
+  return pCache;
+}
+
+void test1() {
+  int32_t          code = 0;
+  SRaftEntryCache* pCache = createCache(5);
+  for (int i = 0; i < 5; ++i) {
+    SSyncRaftEntry* pEntry = createEntry(i);
+    code = raftCachePutEntry(pCache, pEntry);
+    ASSERT(code == 1);
+    syncEntryDestory(pEntry);
+  }
+  raftCacheLog2((char*)"==test1 write 5 entries==", pCache);
+
+  SyncIndex index;
+  index = 1;
+  code = raftCacheDelEntry(pCache, index);
+  ASSERT(code == 0);
+  index = 3;
+  code = raftCacheDelEntry(pCache, index);
+  ASSERT(code == 0);
+  raftCacheLog2((char*)"==test1 delete 1,3==", pCache);
+
+  code = raftCacheClear(pCache);
+  ASSERT(code == 0);
+  raftCacheLog2((char*)"==clear all==", pCache);
+}
+
+void test2() {
+  int32_t          code = 0;
+  SRaftEntryCache* pCache = createCache(5);
+  for (int i = 0; i < 5; ++i) {
+    SSyncRaftEntry* pEntry = createEntry(i);
+    code = raftCachePutEntry(pCache, pEntry);
+    ASSERT(code == 1);
+    syncEntryDestory(pEntry);
+  }
+  raftCacheLog2((char*)"==test2 write 5 entries==", pCache);
+
+  SyncIndex index;
+  index = 1;
+  SSyncRaftEntry* pEntry;
+  code = raftCacheGetEntry(pCache, index, &pEntry);
+  ASSERT(code == 0);
+  syncEntryDestory(pEntry);
+  syncEntryLog2((char*)"==test2 get entry 1==", pEntry);
+
+  index = 2;
+  code = raftCacheGetEntryP(pCache, index, &pEntry);
+  ASSERT(code == 0);
+  syncEntryLog2((char*)"==test2 get entry pointer 2==", pEntry);
+
+  // not found
+  index = 8;
+  code = raftCacheGetEntry(pCache, index, &pEntry);
+  ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST);
+  sTrace("==test2 get entry 8 not found==");
+
+  // not found
+  index = 9;
+  code = raftCacheGetEntryP(pCache, index, &pEntry);
+  ASSERT(code == -1 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST);
+  sTrace("==test2 get entry pointer 9 not found==");
+}
+
+void test3() {
+  int32_t          code = 0;
+  SRaftEntryCache* pCache = createCache(5);
+  for (int i = 0; i < 5; ++i) {
+    SSyncRaftEntry* pEntry = createEntry(i);
+    code = raftCachePutEntry(pCache, pEntry);
+    ASSERT(code == 1);
+    syncEntryDestory(pEntry);
+  }
+  for (int i = 6; i < 10; ++i) {
+    SSyncRaftEntry* pEntry = createEntry(i);
+    code = raftCachePutEntry(pCache, pEntry);
+    ASSERT(code == 0);
+    syncEntryDestory(pEntry);
+  }
+  raftCacheLog2((char*)"==test3 write 10 entries, max count is 5==", pCache);
+}
+
+void test4() {
+  int32_t          code = 0;
+  SRaftEntryCache* pCache = createCache(5);
+  for (int i = 0; i < 5; ++i) {
+    SSyncRaftEntry* pEntry = createEntry(i);
+    code = raftCachePutEntry(pCache, pEntry);
+    ASSERT(code == 1);
+    syncEntryDestory(pEntry);
+  }
+  raftCacheLog2((char*)"==test4 write 5 entries==", pCache);
+
+  SyncIndex index;
+  index = 3;
+  SSyncRaftEntry* pEntry;
+  code = raftCacheGetAndDel(pCache, index, &pEntry);
+  ASSERT(code == 0);
+  syncEntryLog2((char*)"==test4 get-and-del entry 3==", pEntry);
+  raftCacheLog2((char*)"==test4 after get-and-del entry 3==", pCache);
+}
+
+int main(int argc, char** argv) {
+  gRaftDetailLog = true;
+  tsAsyncLog = 0;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE + DEBUG_DEBUG;
+
+  test1();
+  test2();
+  test3();
+  test4();
+
+  return 0;
+}

source/libs/transport/inc/transportInt.h

@@ -52,7 +52,7 @@ typedef struct {
   char     user[TSDB_UNI_LEN];  // meter ID
 
   void (*cfp)(void* parent, SRpcMsg*, SEpSet*);
-  bool (*retry)(int32_t code);
+  bool (*retry)(int32_t code, tmsg_t msgType);
   int index;
 
   void*         parent;

source/libs/transport/src/trans.c

@@ -164,11 +164,6 @@ void rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) {
   transSetDefaultAddr(thandle, ip, fqdn);
 }
 
-// void rpcSetMsgTraceId(SRpcMsg* pMsg, STraceId uid) {
-//  SRpcHandleInfo* pInfo = &pMsg->info;
-//  pInfo->traceId = uid;
-//}
-
 int32_t rpcInit() {
   // impl later
   return 0;

source/libs/transport/src/transCli.c

@@ -1030,7 +1030,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
    */
   STransConnCtx* pCtx = pMsg->ctx;
   int32_t        code = pResp->code;
-  if (pTransInst->retry != NULL && pTransInst->retry(code)) {
+  if (pTransInst->retry != NULL && pTransInst->retry(code, pResp->msgType - 1)) {
     pMsg->sent = 0;
     pCtx->retryCnt += 1;
     if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {

source/os/src/osSystem.c

@@ -18,6 +18,10 @@
 #include "os.h"
 
 #if defined(WINDOWS)
+BOOL WINAPI CtrlHandler(DWORD fdwCtrlType) {
+  printf("\n" TAOS_CONSOLE_PROMPT_HEADER);
+  return TRUE;
+}
 #elif defined(_TD_DARWIN_64)
 #else
 #include <dlfcn.h>
@@ -124,7 +128,7 @@ int taosSetConsoleEcho(bool on) {
 
 void taosSetTerminalMode() {
 #if defined(WINDOWS)
-  // assert(0);
+  SetConsoleCtrlHandler(CtrlHandler, TRUE);
 
 #else
   struct termios newtio;
@@ -158,7 +162,6 @@ void taosSetTerminalMode() {
 
 int32_t taosGetOldTerminalMode() {
 #if defined(WINDOWS)
-  // assert(0);
 #else
   /* Make sure stdin is a terminal. */
   if (!isatty(STDIN_FILENO)) {
@@ -176,7 +179,7 @@ int32_t taosGetOldTerminalMode() {
 
 void taosResetTerminalMode() {
 #if defined(WINDOWS)
-  // assert(0);
+  SetConsoleCtrlHandler(CtrlHandler, FALSE);
 #else
   if (tcsetattr(0, TCSANOW, &oldtio) != 0) {
     fprintf(stderr, "Fail to reset the terminal properties!\n");

tests/parallel_test/container_build.sh

@@ -52,7 +52,7 @@ fi
 
 docker run \
     -v $REP_MOUNT_PARAM \
-    --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_TOOLS=true;make -j $THREAD_COUNT"
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true;make -j $THREAD_COUNT"
 
 ret=$?
 exit $ret

tests/script/jenkins/basic.txt

@@ -22,15 +22,19 @@
 
 # ---- dnode
 ./test.sh -f tsim/dnode/balance_replica1.sim
+#./test.sh -f tsim/dnode/balance_replica3.sim
 ./test.sh -f tsim/dnode/create_dnode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim
-#./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
+./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim
-#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
+./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim
-#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
+./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim
-#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
+./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim
+./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim
 ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
+./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim
+./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim
 
 # ---- insert
 ./test.sh -f tsim/insert/basic0.sim

tests/script/test-all.bat

@@ -11,14 +11,17 @@ if not "%2" == "" (
 )
 for /F "usebackq tokens=*" %%i in (!caseFile!) do (
     set line=%%i
-    if "!line:~,9!" == "./test.sh" (
+    call :CheckSkipCase %%i
-        set /a a+=1
+    if !skipCase! == false (
-        echo !a! Processing %%i
+        if "!line:~,9!" == "./test.sh" (
-        call :GetTimeSeconds !time!
+            set /a a+=1
-        set time1=!_timeTemp!
+            echo !a! Processing %%i
-        echo Start at !time!
+            call :GetTimeSeconds !time!
-        call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt
+            set time1=!_timeTemp!
-        if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
+            echo Start at !time!
+            call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt
+            if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
+        )
     )
 )
 exit !exitNum!
@@ -56,3 +59,10 @@ for %%a in (%tt%) do (
 )
 set /a _timeTemp=(%hh%*60+%mm%)*60+%ss%
 goto :eof
+
+:CheckSkipCase
+set skipCase=false
+if "%*" == "./test.sh -f tsim/query/scalarFunction.sim" ( set skipCase=true )
+if "%*" == "./test.sh -f tsim/stream/distributeInterval0.sim" ( set skipCase=true )
+if "%*" == "./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim" ( set skipCase=true )
+:goto eof

tests/script/tsim/dnode/balance_replica1.sim

@@ -117,7 +117,6 @@ if $rows != 6 then
   return -1
 endi
 
-return
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 system sh/exec.sh -n dnode3 -s stop -x SIGINT

tests/script/tsim/dnode/balance_replica3.sim

@@ -0,0 +1,362 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql connect
+
+print =============== step1 create dnode2
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+sql create dnode $hostname port 7500
+
+$x = 0
+step1:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
+if $rows != 5 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+if $data(3)[4] != ready then
+  goto step1
+endi
+if $data(4)[4] != ready then
+  goto step1
+endi
+
+print =============== step2: create db
+sql create database d1 vgroups 4 replica 3
+
+print =============== step32 wait vgroup2
+$x = 0
+step32:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(2)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(2)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(2)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step32
+endi
+
+print =============== step33 wait vgroup3
+$x = 0
+step33:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(3)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(3)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(3)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step33
+endi
+
+print =============== step34 wait vgroup4
+$x = 0
+step34:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(4)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step34
+endi
+
+print =============== step35 wait vgroup5
+$x = 0
+step35:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(4)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step35
+endi
+
+print =============== step36: create table
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql create table d1.c2 using st tags(1)
+sql create table d1.c3 using st tags(1)
+sql create table d1.c4 using st tags(1)
+sql create table d1.c5 using st tags(1)
+sql create table d1.c6 using st tags(1)
+sql show d1.tables
+if $rows != 6 then
+  return -1
+endi
+
+print =============== step4: start dnode5
+system sh/exec.sh -n dnode5 -s start
+$x = 0
+step4:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
+if $rows != 5 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step4
+endi
+if $data(2)[4] != ready then
+  goto step4
+endi
+if $data(3)[4] != ready then
+  goto step4
+endi
+if $data(4)[4] != ready then
+  goto step4
+endi
+if $data(5)[4] != ready then
+  goto step4
+endi
+
+print =============== step5: balance
+sql balance vgroup
+
+print =============== step62 wait vgroup2
+$x = 0
+step62:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(2)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(2)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(2)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step62
+endi
+
+print =============== step63 wait vgroup3
+$x = 0
+step63:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(3)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(3)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(3)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step63
+endi
+
+print =============== step64 wait vgroup4
+$x = 0
+step64:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(4)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step64
+endi
+
+print =============== step65 wait vgroup5
+$x = 0
+step65:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(4)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step65
+endi
+
+
+
+print =============== step7: select data
+
+sql show d1.tables
+print rows $rows
+if $rows != 6 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT

tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim

@@ -28,7 +28,7 @@ step1:
 sql show dnodes
 print ===> $data00 $data01 $data02 $data03 $data04 $data05
 print ===> $data10 $data11 $data12 $data13 $data14 $data15
-if $rows != 3 then
+if $rows != 5 then
   return -1
 endi
 if $data(1)[4] != ready then
@@ -49,6 +49,128 @@ endi
 
 print =============== step3 create database
 sql create database d1 vgroups 4 replica 3
+
+print =============== step32 wait vgroup2
+$x = 0
+step32:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(2)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(2)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(2)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step32
+endi
+
+print =============== step33 wait vgroup3
+$x = 0
+step33:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(3)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(3)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(3)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step33
+endi
+
+print =============== step34 wait vgroup4
+$x = 0
+step34:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(4)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step34
+endi
+
+print =============== step35 wait vgroup5
+$x = 0
+step35:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(4)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step35
+endi
+
+print =============== step36: create table
 sql use d1
 sql create table d1.st (ts timestamp, i int) tags (j int)
 sql create table d1.c1 using st tags(1)
@@ -57,21 +179,6 @@ if $rows != 1 then
   return -1
 endi
 
-sql show d1.vgroups
-print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
-if $rows != 1 then
-  return -1
-endi
-if $data(2)[3] != 2 then
-  return -1
-endi
-if $data(2)[5] != 3 then
-  return -1
-endi
-if $data(2)[7] != 4 then
-  return -1
-endi
-
 print =============== step4: drop dnode 2
 system sh/exec.sh -n dnode5 -s start
 $x = 0
@@ -85,7 +192,7 @@ step4:
 sql show dnodes
 print ===> $data00 $data01 $data02 $data03 $data04 $data05
 print ===> $data10 $data11 $data12 $data13 $data14 $data15
-if $rows != 3 then
+if $rows != 5 then
   return -1
 endi
 if $data(1)[4] != ready then
@@ -115,21 +222,133 @@ if $rows != 4 then
   return -1
 endi
 
-print show d1.vgroups
+print =============== step62 wait vgroup2
+$x = 0
+step62:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
 sql show d1.vgroups
-print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
-if $rows != 1 then
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
   return -1
 endi
+if $data(2)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(2)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(2)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step62
+endi
 
+print =============== step63 wait vgroup3
+$x = 0
+step63:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(3)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(3)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(3)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step63
+endi
 
-print =============== step6: select data
+print =============== step64 wait vgroup4
+$x = 0
+step64:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(4)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step64
+endi
+
+print =============== step35 wait vgroup5
+$x = 0
+step65:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19
+print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29
+print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39
+if $rows != 4 then
+  return -1
+endi
+if $data(4)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(4)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step65
+endi
+
+print =============== step7: select data
+print ===> $data00 $data01 
 sql show d1.tables
 if $rows != 1 then
   return -1
 endi
 
-return
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 system sh/exec.sh -n dnode3 -s stop -x SIGINT

tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim

@@ -49,6 +49,50 @@ endi
 
 print =============== step3 create database
 sql create database d1 vgroups 1 replica 3
+$leaderExist = 0
+$leaderVnode = 0
+$follower1 = 0
+$follower2 = 0
+
+$x = 0
+step3:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $rows != 1 then
+  return -1
+endi
+if $data(2)[4] == leader then
+  $leaderExist = 1
+  $leaderVnode = 2
+  $follower1 = 3
+  $follower2 = 4
+endi
+if $data(2)[6] == leader then
+  $leaderExist = 1
+  $leaderVnode = 3
+  $follower1 = 2
+  $follower2 = 4
+endi
+if $data(2)[8] == leader then
+  $leaderExist = 1
+  $leaderVnode = 4
+  $follower1 = 2
+  $follower2 = 3
+endi
+if  $leaderExist != 1 then
+  goto step3
+endi
+
+print leader $leaderVnode
+print follower1 $follower1
+print follower2 $follower2
+
 sql use d1
 sql create table d1.st (ts timestamp, i int) tags (j int)
 sql create table d1.c1 using st tags(1)
@@ -72,13 +116,6 @@ if $data(2)[7] != 4 then
   return -1
 endi
 
-system sh/exec.sh -n dnode4 -s stop -x SIGINT
-system sh/exec.sh -n dnode3 -s stop -x SIGINT
-
-
-return
-
-
 print =============== step4: drop dnode 2
 system sh/exec.sh -n dnode5 -s start
 $x = 0
@@ -92,7 +129,7 @@ step4:
 sql show dnodes
 print ===> $data00 $data01 $data02 $data03 $data04 $data05
 print ===> $data10 $data11 $data12 $data13 $data14 $data15
-if $rows != 3 then
+if $rows != 5 then
   return -1
 endi
 if $data(1)[4] != ready then
@@ -116,8 +153,11 @@ sql drop dnode 2
 
 print show dnodes; 
 sql show dnodes;
-print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] 
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
-print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] 
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
 if $rows != 4 then 
   return -1
 endi
@@ -128,9 +168,9 @@ print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4]
 if $rows != 1 then
   return -1
 endi
-#if $data(2)[3] != 3 then
+if $data(2)[3] != 5 then
-#  return -1
+  return -1
-#endi
+endi
 
 print =============== step6: select data
 sql show d1.tables
@@ -138,7 +178,37 @@ if $rows != 1 then
   return -1
 endi
 
-return
+$leaderExist = 0
+$leaderVnode = 0
+$follower1 = 0
+$follower2 = 0
+
+$x = 0
+step6:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $rows != 1 then
+  return -1
+endi
+if $data(2)[4] == leader then
+  $leaderExist = 1
+endi
+if $data(2)[6] == leader then
+  $leaderExist = 1
+endi
+if $data(2)[8] == leader then
+  $leaderExist = 1
+endi
+if  $leaderExist != 1 then
+  goto step6
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 system sh/exec.sh -n dnode3 -s stop -x SIGINT

tests/script/tsim/dnode/redistribute_vgroup_replica1.sim

@@ -0,0 +1,178 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+sql connect
+sql create user u1 pass 'taosdata'
+
+print =============== step1 create dnode2
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+
+$x = 0
+step1:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+if $rows != 4 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+
+print =============== step2: create db
+sql create database d1 vgroups 1 replica 1
+
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+$x = 0
+step2:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+if $rows != 4 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step2
+endi
+if $data(2)[4] != ready then
+  goto step2
+endi
+if $data(3)[4] != ready then
+  goto step2
+endi
+if $data(4)[4] != ready then
+  goto step2
+endi
+
+print =============== step3: create database
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step40: 
+print redistribute vgroup 2 dnode 3
+sql redistribute vgroup 2 dnode 3
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step41: 
+print redistribute vgroup 2 dnode 4
+sql redistribute vgroup 2 dnode 4
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step42: 
+print redistribute vgroup 2 dnode 2
+sql redistribute vgroup 2 dnode 2
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step43:
+print redistribute vgroup 2 dnode 3
+sql redistribute vgroup 2 dnode 3
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step44:
+print redistribute vgroup 2 dnode 4
+sql redistribute vgroup 2 dnode 4
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step45:
+print redistribute vgroup 2 dnode 2
+sql redistribute vgroup 2 dnode 2
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step46: 
+print redistribute vgroup 2 dnode 3
+sql redistribute vgroup 2 dnode 3
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step47: 
+print redistribute vgroup 2 dnode 2
+sql redistribute vgroup 2 dnode 2
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+system sh/exec.sh -n dnode4 -s stop -x SIGINT

tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim

@@ -65,7 +65,7 @@ sql_error redistribute vgroup 3 dnode 6 dnode 3 dnode 4
 # vgroup not exist
 sql_error redistribute vgroup 3 dnode 5 dnode 3 dnode 4 
 # un changed
-sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4
+# sql_error redistribute vgroup 2 dnode 2 dnode 3 dnode 4
 # no enought vnodes
 sql_error redistribute vgroup 2 dnode 1 dnode 3 dnode 4 
 # offline vnodes
@@ -176,8 +176,6 @@ if $rows != 1 then
   return -1
 endi
 
-return
-
 print =============== step32: 
 print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 
 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 

tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim

@@ -0,0 +1,245 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+system sh/deploy.sh -n dnode6 -i 6
+system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql connect
+sql create user u1 pass 'taosdata'
+
+print =============== step1 create dnode2
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+sql create dnode $hostname port 7500
+sql create dnode $hostname port 7600
+
+$x = 0
+step1:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
+if $rows != 6 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+if $data(3)[4] != ready then
+  goto step1
+endi
+if $data(4)[4] != ready then
+  goto step1
+endi
+
+print =============== step2: create db
+sql create database d1 vgroups 1 replica 3
+
+system sh/exec.sh -n dnode5 -s start
+system sh/exec.sh -n dnode6 -s start
+$x = 0
+step2:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
+if $rows != 6 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step2
+endi
+if $data(2)[4] != ready then
+  goto step2
+endi
+if $data(3)[4] != ready then
+  goto step2
+endi
+if $data(4)[4] != ready then
+  goto step2
+endi
+if $data(5)[4] != ready then
+  goto step2
+endi
+if $data(6)[4] != ready then
+  goto step2
+endi
+
+print =============== step3: create database
+$leaderExist = 0
+$leaderVnode = 0
+$follower1 = 0
+$follower2 = 0
+
+$x = 0
+step3:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $rows != 1 then
+  return -1
+endi
+if $data(2)[4] == leader then
+  $leaderExist = 1
+  $leaderVnode = 2
+  $follower1 = 3
+  $follower2 = 4
+endi
+if $data(2)[6] == leader then
+  $leaderExist = 1
+  $leaderVnode = 3
+  $follower1 = 2
+  $follower2 = 4
+endi
+if $data(2)[8] == leader then
+  $leaderExist = 1
+  $leaderVnode = 4
+  $follower1 = 2
+  $follower2 = 3
+endi
+if  $leaderExist != 1 then
+  goto step3
+endi
+
+print leader $leaderVnode
+print follower1 $follower1
+print follower2 $follower2
+
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step40: 
+print redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6
+sql redistribute vgroup 2 dnode $leaderVnode dnode 5 dnode 6
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step41: 
+print redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1
+sql redistribute vgroup 2 dnode 5 dnode $follower2 dnode $follower1
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step42: 
+print redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode
+sql redistribute vgroup 2 dnode $follower2 dnode 6 dnode $leaderVnode
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step43:
+print redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6
+sql redistribute vgroup 2 dnode $follower1 dnode 5 dnode 6
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step44:
+print redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode
+sql redistribute vgroup 2 dnode $follower2 dnode 5 dnode $leaderVnode
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step45:
+print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6 
+sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 6 
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step46: 
+print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 
+sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5 
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step47: 
+print redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode
+sql redistribute vgroup 2 dnode $follower1 dnode 6 dnode $leaderVnode
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+system sh/exec.sh -n dnode5 -s stop -x SIGINT

tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim

@@ -0,0 +1,263 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+system sh/deploy.sh -n dnode5 -i 5
+system sh/deploy.sh -n dnode6 -i 6
+system sh/deploy.sh -n dnode7 -i 7
+system sh/deploy.sh -n dnode8 -i 8
+system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode5 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode6 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode7 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode8 -c transPullupInterval -v 1
+system sh/cfg.sh -n dnode1 -c supportVnodes -v 0
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode4 -s start
+sql connect
+sql create user u1 pass 'taosdata'
+
+print =============== step1 create dnode2
+sql create dnode $hostname port 7200
+sql create dnode $hostname port 7300
+sql create dnode $hostname port 7400
+sql create dnode $hostname port 7500
+sql create dnode $hostname port 7600
+sql create dnode $hostname port 7700
+sql create dnode $hostname port 7800
+
+$x = 0
+step1:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
+if $rows != 8 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+if $data(3)[4] != ready then
+  goto step1
+endi
+if $data(4)[4] != ready then
+  goto step1
+endi
+
+print =============== step2: create db
+sql create database d1 vgroups 1 replica 3
+
+system sh/exec.sh -n dnode5 -s start
+system sh/exec.sh -n dnode6 -s start
+system sh/exec.sh -n dnode7 -s start
+system sh/exec.sh -n dnode8 -s start
+$x = 0
+step2:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+print ===> $data20 $data21 $data22 $data23 $data24 $data25
+print ===> $data30 $data31 $data32 $data33 $data34 $data35
+print ===> $data40 $data41 $data42 $data43 $data44 $data45
+if $rows != 8 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step2
+endi
+if $data(2)[4] != ready then
+  goto step2
+endi
+if $data(3)[4] != ready then
+  goto step2
+endi
+if $data(4)[4] != ready then
+  goto step2
+endi
+if $data(5)[4] != ready then
+  goto step2
+endi
+if $data(6)[4] != ready then
+  goto step2
+endi
+if $data(7)[4] != ready then
+  goto step2
+endi
+if $data(8)[4] != ready then
+  goto step2
+endi
+
+print =============== step3: create database
+$leaderExist = 0
+$leaderVnode = 0
+$follower1 = 0
+$follower2 = 0
+
+$x = 0
+step3:
+	$x = $x + 1
+	sleep 1000
+	if $x == 60 then
+	  print ====> db not ready!
+		return -1
+	endi
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+if $rows != 1 then
+  return -1
+endi
+if $data(2)[4] == leader then
+  $leaderExist = 1
+  $leaderVnode = 2
+  $follower1 = 3
+  $follower2 = 4
+endi
+if $data(2)[6] == leader then
+  $leaderExist = 1
+  $leaderVnode = 3
+  $follower1 = 2
+  $follower2 = 4
+endi
+if $data(2)[8] == leader then
+  $leaderExist = 1
+  $leaderVnode = 4
+  $follower1 = 2
+  $follower2 = 3
+endi
+if  $leaderExist != 1 then
+  goto step3
+endi
+
+print leader $leaderVnode
+print follower1 $follower1
+print follower2 $follower2
+
+sql use d1
+sql create table d1.st (ts timestamp, i int) tags (j int)
+sql create table d1.c1 using st tags(1)
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step40: 
+print redistribute vgroup 2 dnode 7 dnode 5 dnode 6
+sql redistribute vgroup 2 dnode 7 dnode 5 dnode 6
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step41: 
+print redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1
+sql redistribute vgroup 2 dnode $leaderVnode dnode $follower2 dnode $follower1
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step42: 
+print redistribute vgroup 2 dnode 7 dnode 6 dnode 8
+sql redistribute vgroup 2 dnode 7 dnode 6 dnode 8
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step43:
+print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5
+sql redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 5
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+
+return 
+
+
+print =============== step44:
+print redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode
+sql redistribute vgroup 2 dnode 6 dnode 7 dnode $leaderVnode
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step45:
+print redistribute vgroup 2 dnode $follower1 dnode $follower2 dnode 8
+sql redistribute vgroup 2dnode $follower1 dnode $follower2 dnode 8
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step46: 
+print redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5 
+sql redistribute vgroup 2 dnode $leaderVnode dnode 6 dnode 5 
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+print =============== step47: 
+print redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8
+sql redistribute vgroup 2 dnode $follower1 dnode 7 dnode 8
+sql show d1.vgroups
+print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show d1.tables
+if $rows != 1 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+system sh/exec.sh -n dnode5 -s stop -x SIGINT

tests/script/tsim/stream/distributeInterval0.sim

@@ -41,7 +41,7 @@ sql create table ts1 using st tags(1,1,1);
 sql create table ts2 using st tags(2,2,2);
 sql create table ts3 using st tags(3,2,2);
 sql create table ts4 using st tags(4,2,2);
-sql create stream stream_t1 trigger at_once into streamtST1 as select  _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b)  c4, min(c) c5 from st interval(10s);
+sql create stream stream_t1 trigger at_once watermark 1d into streamtST1 as select  _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b)  c4, min(c) c5 from st interval(10s);
 
 sleep 1000
 

tests/script/tsim/stream/distributeIntervalRetrive0.sim

@@ -0,0 +1,291 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+
+system sh/exec.sh -n dnode1 -s start 
+sleep 50
+sql connect
+
+sql create dnode $hostname2 port 7200
+
+system sh/exec.sh -n dnode2 -s start
+
+print ===== step1
+$x = 0
+step1:
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+	  print ====> dnode not ready!
+		return -1
+	endi
+sql show dnodes
+print ===> $data00 $data01 $data02 $data03 $data04 $data05
+print ===> $data10 $data11 $data12 $data13 $data14 $data15
+if $rows != 2 then
+  return -1
+endi
+if $data(1)[4] != ready then
+  goto step1
+endi
+if $data(2)[4] != ready then
+  goto step1
+endi
+
+print ===== step2
+
+sql create database test  vgroups 4;
+sql use test;
+sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create table ts2 using st tags(2,2,2);
+sql create table ts3 using st tags(3,2,2);
+sql create table ts4 using st tags(4,2,2);
+sql create stream stream_t1 trigger at_once into streamtST1 as select  _wstartts, count(*) c1, sum(a) c3 , max(b)  c4, min(c) c5 from st interval(10s);
+
+sleep 1000
+
+sql insert into ts1 values(1648791213001,1,12,3,1.0);
+sql insert into ts2 values(1648791213001,1,12,3,1.0);
+sql insert into ts1 values(1648791213002,NULL,NULL,NULL,NULL);
+sql insert into ts2 values(1648791213002,NULL,NULL,NULL,NULL);
+
+sql insert into ts1 values(1648791223002,2,2,3,1.1);
+sql insert into ts1 values(1648791233003,3,2,3,2.1);
+sql insert into ts2 values(1648791243004,4,2,43,73.1);
+
+sql insert into ts1 values(1648791213002,24,22,23,4.1) (1648791243005,4,20,3,3.1);
+
+sleep 1000
+
+sql insert into ts3 values(1648791213001,12,12,13,14.1) (1648791243005,14,30,30,30.1);
+
+$loop_count = 0
+loop1:
+sleep 1000
+sql select * from streamtST1;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+# row 0
+if $data01 != 5 then
+  print =====data01=$data01
+  goto loop1
+endi
+
+if $data02 != 14 then
+  print =====data02=$data02
+  goto loop1
+endi
+
+# row 1
+if $data11 != 1 then
+  print =====data11=$data11
+  goto loop1
+endi
+
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop1
+endi
+
+#row2
+if $data21 != 1 then
+  print =====data21=$data21
+  goto loop1
+endi
+
+if $data22 != 3 then
+  print =====data22=$data22
+  goto loop1
+endi
+
+#row 3
+if $data31 != 3 then
+  print =====data31=$data31
+  goto loop1
+endi
+
+if $data32 != 22 then
+  print =====data32=$data32
+  goto loop1
+endi
+
+print loop1 over
+
+sql insert into ts1 values(1648791223008,4,2,30,3.1) (1648791213009,4,2,3,3.1)  (1648791233010,4,2,3,3.1) (1648791243011,4,2,3,3.1)(1648791243012,34,32,33,3.1);
+
+$loop_count = 0
+loop2:
+sleep 1000
+sql select * from streamtST1;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+# row 0
+if $data01 != 6 then
+  print =====data01=$data01
+  goto loop2
+endi
+
+if $data02 != 18 then
+  print =====data02=$data02
+  goto loop2
+endi
+
+# row 1
+if $data11 != 2 then
+  print =====data11=$data11
+  goto loop2
+endi
+
+if $data12 != 6 then
+  print =====data12=$data12
+  goto loop2
+endi
+
+#row2
+if $data21 != 2 then
+  print =====data21=$data21
+  goto loop2
+endi
+
+if $data22 != 7 then
+  print =====data22=$data22
+  goto loop2
+endi
+
+#row 3
+if $data31 != 5 then
+  print =====data31=$data31
+  goto loop2
+endi
+
+if $data32 != 60 then
+  print =====data32=$data32
+  goto loop2
+endi
+
+print loop2 over
+
+sql insert into ts4 values(1648791223008,4,2,30,3.1) (1648791213009,4,2,3,3.1)  (1648791233010,4,2,3,3.1);
+
+$loop_count = 0
+loop3:
+sleep 1000
+sql select * from streamtST1;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+# row 0
+if $data01 != 7 then
+  print =====data01=$data01
+  goto loop3
+endi
+
+if $data02 != 22 then
+  print =====data02=$data02
+  goto loop3
+endi
+
+# row 1
+if $data11 != 3 then
+  print =====data11=$data11
+  goto loop3
+endi
+
+if $data12 != 10 then
+  print =====data12=$data12
+  goto loop3
+endi
+
+#row2
+if $data21 != 3 then
+  print =====data21=$data21
+  goto loop3
+endi
+
+if $data22 != 11 then
+  print =====data22=$data22
+  goto loop3
+endi
+
+#row 3
+if $data31 != 5 then
+  print =====data31=$data31
+  goto loop3
+endi
+
+if $data32 != 60 then
+  print =====data32=$data32
+  goto loop3
+endi
+
+print loop3 over
+
+$loop_count = 0
+loop4:
+sleep 1000
+sql select * from streamtST1;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+# row 0
+if $data01 != 7 then
+  print =====data01=$data01
+  goto loop4
+endi
+
+if $data02 != 22 then
+  print =====data02=$data02
+  goto loop4
+endi
+
+# row 1
+if $data11 != 3 then
+  print =====data11=$data11
+  goto loop4
+endi
+
+if $data12 != 10 then
+  print =====data12=$data12
+  goto loop4
+endi
+
+#row2
+if $data21 != 3 then
+  print =====data21=$data21
+  goto loop4
+endi
+
+if $data22 != 11 then
+  print =====data22=$data22
+  goto loop4
+endi
+
+#row 3
+if $data31 != 5 then
+  print =====data31=$data31
+  goto loop4
+endi
+
+if $data32 != 60 then
+  print =====data32=$data32
+  goto loop4
+endi
+
+print loop4 over
+
+system sh/stop_dnodes.sh

tests/system-test/2-query/abs.py

@@ -23,7 +23,6 @@ class TDTestCase:
         self.time_step = 1000
 
     def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ):
-    
         tdLog.info(" prepare datas for auto check abs function ")
 
         tdSql.execute(" create database test ")
@@ -36,7 +35,7 @@ class TDTestCase:
 
             ts = self.ts
             for row in range(rownums):
-                ts += time_step*row
+                ts = self.ts + time_step*row
                 c1 = random.randint(0,10000)
                 c2 = random.randint(0,100000)
                 c3 = random.randint(0,125)
@@ -538,25 +537,41 @@ class TDTestCase:
         # tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ")  #　taosd crash
         tdSql.query("select c1 ,t1 from stb1 where t1 =0 ")
         tdSql.checkRows(13)
-        # tdSql.query("select t1 from stb1 where t1 >0 ")
+        tdSql.query("select t1 from stb1 where t1 >0 ")
-        # tdSql.checkRows(3)
+        tdSql.checkRows(3)
+        tdSql.query("select t1 from stb1 where t1 =3 ")
+        tdSql.checkRows(1)
         # tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)")
         # tdSql.checkData(0,0,61)
         # tdSql.query("select distinct(c1) ,t1 from stb1")
         # tdSql.checkRows(20)
-        # tdSql.query("select max(t2) , t1 ,c1, t2 from stb1")
+        tdSql.query("select max(t2) , t1 ,c1, t2 from stb1")
-        # tdSql.checkData(0,3,33333)
+        tdSql.checkData(0,3,33333)
 
         # tag filter with abs function
-        # tdSql.query("select t1 from stb1 where abs(t1)=1")
+        tdSql.query("select t1 from stb1 where abs(t1)=1")
-        # tdSql.checkRows(1)
+        tdSql.checkRows(1)
         tdSql.query("select t1 from stb1 where abs(c1+t1)=1")
         tdSql.checkRows(1)
-        # tdSql.query("select t1 from stb1 where abs(t1+c1)=1")
+        tdSql.checkData(0,0,0)
-        # tdSql.checkRows(1)
+
         tdSql.query(
             "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1")
 
+    def support_super_table_test(self):
+        tdSql.execute(" use testdb ")
+        self.check_result_auto( " select c1 from stb1 order by ts " , "select abs(c1) from stb1 order by ts" )
+        self.check_result_auto( " select c1 from stb1 order by tbname " , "select abs(c1) from stb1 order by tbname" )
+        self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname  " , "select abs(c1) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto( " select c1 from stb1 where c1 > 0 order by tbname  " , "select abs(c1) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto( " select t1,c1 from stb1 order by ts " , "select t1, abs(c1) from stb1 order by ts" )
+        self.check_result_auto( " select t2,c1 from stb1 order by tbname " , "select t2 ,abs(c1) from stb1 order by tbname" )
+        self.check_result_auto( " select t3,c1 from stb1 where c1 > 0 order by tbname  " , "select t3 ,abs(c1) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto( " select t4,c1 from stb1 where c1 > 0 order by tbname  " , "select t4 , abs(c1) from stb1 where c1 > 0 order by tbname" )
+        pass
+
+
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
 
@@ -593,6 +608,10 @@ class TDTestCase:
 
         self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step)
 
+        tdLog.printNoPrefix("==========step8: check abs result of  stable query ============")
+
+        self.support_super_table_test()
+
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")

tests/system-test/2-query/arccos.py

@@ -479,6 +479,20 @@ class TDTestCase:
             tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2))
 
         self.check_result_auto_acos("select num1,num2 from tb3;" , "select acos(num1),acos(num2) from tb3")
+
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto_acos( " select c5 from stb1 order by ts " , "select acos(c5) from stb1 order by ts" )
+        self.check_result_auto_acos( " select c5 from stb1 order by tbname " , "select acos(c5) from stb1 order by tbname" )
+        self.check_result_auto_acos( " select c5 from stb1 where c1 > 0 order by tbname  " , "select acos(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_acos( " select c5 from stb1 where c1 > 0 order by tbname  " , "select acos(c5) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto_acos( " select t1,c5 from stb1 order by ts " , "select acos(t1), acos(c5) from stb1 order by ts" )
+        self.check_result_auto_acos( " select t1,c5 from stb1 order by tbname " , "select acos(t1) ,acos(c5) from stb1 order by tbname" )
+        self.check_result_auto_acos( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select acos(t1) ,acos(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_acos( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select acos(t1) , acos(c5) from stb1 where c1 > 0 order by tbname" )
+        pass
+    
     
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
@@ -512,6 +526,14 @@ class TDTestCase:
 
         self.abs_func_filter()
 
+        tdLog.printNoPrefix("==========step7: acos filter query ============") 
+
+        self.abs_func_filter()
+
+        tdLog.printNoPrefix("==========step8: check acos result of  stable query ============")
+
+        self.support_super_table_test()
+
 
     def stop(self):
         tdSql.close()

tests/system-test/2-query/arcsin.py

@@ -479,6 +479,20 @@ class TDTestCase:
             tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2))
 
         self.check_result_auto_asin("select num1,num2 from tb3;" , "select asin(num1),asin(num2) from tb3")
+
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto_asin( " select c5 from stb1 order by ts " , "select asin(c5) from stb1 order by ts" )
+        self.check_result_auto_asin( " select c5 from stb1 order by tbname " , "select asin(c5) from stb1 order by tbname" )
+        self.check_result_auto_asin( " select c5 from stb1 where c1 > 0 order by tbname  " , "select asin(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_asin( " select c5 from stb1 where c1 > 0 order by tbname  " , "select asin(c5) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto_asin( " select t1,c5 from stb1 order by ts " , "select asin(t1), asin(c5) from stb1 order by ts" )
+        self.check_result_auto_asin( " select t1,c5 from stb1 order by tbname " , "select asin(t1) ,asin(c5) from stb1 order by tbname" )
+        self.check_result_auto_asin( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select asin(t1) ,asin(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_asin( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select asin(t1) , asin(c5) from stb1 where c1 > 0 order by tbname" )
+        pass
+    
     
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
@@ -512,6 +526,10 @@ class TDTestCase:
 
         self.abs_func_filter()
 
+        tdLog.printNoPrefix("==========step8: check asin result of  stable query ============")
+
+        self.support_super_table_test()
+
 
     def stop(self):
         tdSql.close()

tests/system-test/2-query/arctan.py

@@ -476,6 +476,20 @@ class TDTestCase:
             tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2))
 
         self.check_result_auto_atan("select num1,num2 from tb3;" , "select atan(num1),atan(num2) from tb3")
+
+
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto_atan( " select c5 from stb1 order by ts " , "select atan(c5) from stb1 order by ts" )
+        self.check_result_auto_atan( " select c5 from stb1 order by tbname " , "select atan(c5) from stb1 order by tbname" )
+        self.check_result_auto_atan( " select c5 from stb1 where c1 > 0 order by tbname  " , "select atan(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_atan( " select c5 from stb1 where c1 > 0 order by tbname  " , "select atan(c5) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto_atan( " select t1,c5 from stb1 order by ts " , "select atan(t1), atan(c5) from stb1 order by ts" )
+        self.check_result_auto_atan( " select t1,c5 from stb1 order by tbname " , "select atan(t1) ,atan(c5) from stb1 order by tbname" )
+        self.check_result_auto_atan( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select atan(t1) ,atan(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_atan( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select atan(t1) , atan(c5) from stb1 where c1 > 0 order by tbname" )
+        pass
     
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
@@ -509,6 +523,10 @@ class TDTestCase:
 
         self.abs_func_filter()
 
+        tdLog.printNoPrefix("==========step8: check arctan result of  stable query ============")
+
+        self.support_super_table_test()
+
         
 
     def stop(self):

tests/system-test/2-query/ceil.py

@@ -427,6 +427,18 @@ class TDTestCase:
 
         self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select ceil(c1+1) ,ceil(c2) , ceil(c3*1) , ceil(c4/2), ceil(c5)/2, ceil(c6) from sub1_bound ")
         
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto( " select c5 from stb1 order by ts " , "select ceil(c5) from stb1 order by ts" )
+        self.check_result_auto( " select c5 from stb1 order by tbname " , "select ceil(c5) from stb1 order by tbname" )
+        self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname  " , "select ceil(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname  " , "select ceil(c5) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select ceil(t1), ceil(c5) from stb1 order by ts" )
+        self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select ceil(t1) ,ceil(c5) from stb1 order by tbname" )
+        self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select ceil(t1) ,ceil(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select ceil(t1) , ceil(c5) from stb1 where c1 > 0 order by tbname" )
+        pass
 
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
@@ -455,6 +467,10 @@ class TDTestCase:
 
         self.abs_func_filter()
 
+        tdLog.printNoPrefix("==========step7: check ceil result of  stable query ============")
+
+        self.support_super_table_test()
+
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")

tests/system-test/2-query/cos.py

@@ -476,6 +476,19 @@ class TDTestCase:
             tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2))
 
         self.check_result_auto_cos("select num1,num2 from tb3;" , "select cos(num1),cos(num2) from tb3")
+
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto_cos( " select c5 from stb1 order by ts " , "select cos(c5) from stb1 order by ts" )
+        self.check_result_auto_cos( " select c5 from stb1 order by tbname " , "select cos(c5) from stb1 order by tbname" )
+        self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname  " , "select cos(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname  " , "select cos(c5) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto_cos( " select t1,c5 from stb1 order by ts " , "select cos(t1), cos(c5) from stb1 order by ts" )
+        self.check_result_auto_cos( " select t1,c5 from stb1 order by tbname " , "select cos(t1) ,cos(c5) from stb1 order by tbname" )
+        self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select cos(t1) ,cos(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select cos(t1) , cos(c5) from stb1 where c1 > 0 order by tbname" )
+        pass
     
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
@@ -509,6 +522,10 @@ class TDTestCase:
 
         self.abs_func_filter()
 
+        tdLog.printNoPrefix("==========step8: check cos result of  stable query ============")
+
+        self.support_super_table_test()
+
 
     def stop(self):
         tdSql.close()

tests/system-test/2-query/floor.py

@@ -427,6 +427,19 @@ class TDTestCase:
 
         self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select floor(c1+1) ,floor(c2) , floor(c3*1) , floor(c4/2), floor(c5)/2, floor(c6) from sub1_bound ")
 
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto( " select c5 from stb1 order by ts " , "select floor(c5) from stb1 order by ts" )
+        self.check_result_auto( " select c5 from stb1 order by tbname " , "select floor(c5) from stb1 order by tbname" )
+        self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname  " , "select floor(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname  " , "select floor(c5) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select floor(t1), floor(c5) from stb1 order by ts" )
+        self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select floor(t1) ,floor(c5) from stb1 order by tbname" )
+        self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select floor(t1) ,floor(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select floor(t1) , floor(c5) from stb1 where c1 > 0 order by tbname" )
+        pass
+
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
 
@@ -454,6 +467,10 @@ class TDTestCase:
 
         self.abs_func_filter()
 
+        tdLog.printNoPrefix("==========step7: check floor result of  stable query ============")
+
+        self.support_super_table_test()
+
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")

tests/system-test/2-query/log.py

@@ -670,7 +670,19 @@ class TDTestCase:
         tdSql.checkData(0,2,math.log(32767.000000000,2))
         tdSql.checkData(0,3,math.log(63.500000000,2))
         tdSql.checkData(0,4,63.999401166)
-      
+
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto_log2( " select c5 from stb1 order by ts " , "select log(c5,2) from stb1 order by ts" )
+        self.check_result_auto_log2( " select c5 from stb1 order by tbname " , "select log(c5,2) from stb1 order by tbname" )
+        self.check_result_auto_log2( " select c5 from stb1 where c1 > 0 order by tbname  " , "select log(c5,2) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_log2( " select c5 from stb1 where c1 > 0 order by tbname  " , "select log(c5,2) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto_log2( " select t1,c5 from stb1 order by ts " , "select log(t1,2), log(c5,2) from stb1 order by ts" )
+        self.check_result_auto_log2( " select t1,c5 from stb1 order by tbname " , "select log(t1,2) ,log(c5,2) from stb1 order by tbname" )
+        self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select log(t1,2) ,log(c5,2) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select log(t1,2) , log(c5,2) from stb1 where c1 > 0 order by tbname" )
+        pass
     
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
@@ -707,7 +719,9 @@ class TDTestCase:
 
         self.abs_func_filter()
 
-        
+        tdLog.printNoPrefix("==========step9: check log result of  stable query ============")
+
+        self.support_super_table_test()            
 
     def stop(self):
         tdSql.close()

tests/system-test/2-query/pow.py

@@ -606,7 +606,23 @@ class TDTestCase:
         tdSql.checkData(0,3,math.pow(63.500000000,2))
         tdSql.checkData(0,5,None)
       
+
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto_pow2( " select c5 from stb1 order by ts " , "select pow(c5,2) from stb1 order by ts" )
+        self.check_result_auto_pow2( " select c5 from stb1 order by tbname " , "select pow(c5,2) from stb1 order by tbname" )
+        self.check_result_auto_pow2( " select c5 from stb1 where c1 > 0 order by tbname  " , "select pow(c5,2) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_pow2( " select c5 from stb1 where c1 > 0 order by tbname  " , "select pow(c5,2) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto_pow2( " select t1,c5 from stb1 order by ts " , "select pow(t1,2), pow(c5,2) from stb1 order by ts" )
+        self.check_result_auto_pow2( " select t1,c5 from stb1 order by tbname " , "select pow(t1,2) ,pow(c5,2) from stb1 order by tbname" )
+        self.check_result_auto_pow2( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select pow(t1,2) ,pow(c5,2) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_pow2( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select pow(t1,2) , pow(c5,2) from stb1 where c1 > 0 order by tbname" )
+        pass
     
+    
+
+
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
 
@@ -642,7 +658,9 @@ class TDTestCase:
 
         self.abs_func_filter()
 
-        
+        tdLog.printNoPrefix("==========step9: check pow result of  stable query ============")
+
+        self.support_super_table_test()    
 
     def stop(self):
         tdSql.close()

tests/system-test/2-query/round.py

@@ -432,6 +432,20 @@ class TDTestCase:
 
         self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select round(c1+1) ,round(c2) , round(c3*1) , round(c4/2), round(c5)/2, round(c6) from sub1_bound ")
 
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto( " select c5 from stb1 order by ts " , "select round(c5) from stb1 order by ts" )
+        self.check_result_auto( " select c5 from stb1 order by tbname " , "select round(c5) from stb1 order by tbname" )
+        self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname  " , "select round(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto( " select c5 from stb1 where c1 > 0 order by tbname  " , "select round(c5) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto( " select t1,c5 from stb1 order by ts " , "select round(t1), round(c5) from stb1 order by ts" )
+        self.check_result_auto( " select t1,c5 from stb1 order by tbname " , "select round(t1) ,round(c5) from stb1 order by tbname" )
+        self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select round(t1) ,round(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select round(t1) , round(c5) from stb1 where c1 > 0 order by tbname" )
+        pass
+    
+    
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
 
@@ -459,6 +473,10 @@ class TDTestCase:
 
         self.abs_func_filter()
 
+        tdLog.printNoPrefix("==========step7: check round result of  stable query ============")
+
+        self.support_super_table_test()
+
     def stop(self):
         tdSql.close()
         tdLog.success(f"{__file__} successfully executed")

tests/system-test/2-query/sin.py

@@ -476,6 +476,19 @@ class TDTestCase:
             tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2))
 
         self.check_result_auto_sin("select num1,num2 from tb3;" , "select sin(num1),sin(num2) from tb3")
+
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto_sin( " select c5 from stb1 order by ts " , "select sin(c5) from stb1 order by ts" )
+        self.check_result_auto_sin( " select c5 from stb1 order by tbname " , "select sin(c5) from stb1 order by tbname" )
+        self.check_result_auto_sin( " select c5 from stb1 where c1 > 0 order by tbname  " , "select sin(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_sin( " select c5 from stb1 where c1 > 0 order by tbname  " , "select sin(c5) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto_sin( " select t1,c5 from stb1 order by ts " , "select sin(t1), sin(c5) from stb1 order by ts" )
+        self.check_result_auto_sin( " select t1,c5 from stb1 order by tbname " , "select sin(t1) ,sin(c5) from stb1 order by tbname" )
+        self.check_result_auto_sin( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select sin(t1) ,sin(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_sin( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select sin(t1) , sin(c5) from stb1 where c1 > 0 order by tbname" )
+        pass
     
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
@@ -509,6 +522,10 @@ class TDTestCase:
 
         self.abs_func_filter()
 
+        tdLog.printNoPrefix("==========step8: check sin result of  stable query ============")
+
+        self.support_super_table_test()
+
         
 
     def stop(self):

tests/system-test/2-query/sqrt.py

@@ -505,7 +505,19 @@ class TDTestCase:
         tdSql.checkData(0,2,math.sqrt(32767.000000000))
         tdSql.checkData(0,3,math.sqrt(63.500000000))
       
-    
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto_sqrt( " select c5 from stb1 order by ts " , "select sqrt(c5) from stb1 order by ts" )
+        self.check_result_auto_sqrt( " select c5 from stb1 order by tbname " , "select sqrt(c5) from stb1 order by tbname" )
+        self.check_result_auto_sqrt( " select c5 from stb1 where c1 > 0 order by tbname  " , "select sqrt(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_sqrt( " select c5 from stb1 where c1 > 0 order by tbname  " , "select sqrt(c5) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto_sqrt( " select t1,c5 from stb1 order by ts " , "select sqrt(t1), sqrt(c5) from stb1 order by ts" )
+        self.check_result_auto_sqrt( " select t1,c5 from stb1 order by tbname " , "select sqrt(t1) ,sqrt(c5) from stb1 order by tbname" )
+        self.check_result_auto_sqrt( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select sqrt(t1) ,sqrt(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_sqrt( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select sqrt(t1) , sqrt(c5) from stb1 where c1 > 0 order by tbname" )
+        pass
+
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
 
@@ -541,6 +553,10 @@ class TDTestCase:
 
         self.abs_func_filter()
 
+        tdLog.printNoPrefix("==========step9: check sqrt result of  stable query ============")
+
+        self.support_super_table_test()            
+
         
 
     def stop(self):

tests/system-test/2-query/tail.py

@@ -188,8 +188,8 @@ class TDTestCase:
 
     def check_tail_table(self , tbname , col_name , tail_rows , offset):
         tail_sql = f"select tail({col_name} , {tail_rows} , {offset}) from {tbname}"
-        equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts"
+        #equal_sql = f"select {col_name} from (select ts , {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}) order by ts"
-        #equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}"
+        equal_sql = f"select {col_name} from {tbname} order by ts desc limit {tail_rows} offset {offset}"
         tdSql.query(tail_sql)
         tail_result = tdSql.queryResult
 
@@ -404,7 +404,7 @@ class TDTestCase:
                 f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )"
             )
         
-        tdSql.query("select tail(c2,2) from sub1_bound")
+        tdSql.query("select tail(c2,2) from sub1_bound order by 1 desc")
         tdSql.checkRows(2)
         tdSql.checkData(0,0,9223372036854775803)
 

tests/system-test/2-query/tan.py

@@ -476,6 +476,20 @@ class TDTestCase:
             tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2))
 
         self.check_result_auto_tan("select num1,num2 from tb3;" , "select tan(num1),tan(num2) from tb3")
+
+    def support_super_table_test(self):
+        tdSql.execute(" use db ")
+        self.check_result_auto_tan( " select c5 from stb1 order by ts " , "select tan(c5) from stb1 order by ts" )
+        self.check_result_auto_tan( " select c5 from stb1 order by tbname " , "select tan(c5) from stb1 order by tbname" )
+        self.check_result_auto_tan( " select c5 from stb1 where c1 > 0 order by tbname  " , "select tan(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_tan( " select c5 from stb1 where c1 > 0 order by tbname  " , "select tan(c5) from stb1 where c1 > 0 order by tbname" )
+
+        self.check_result_auto_tan( " select t1,c5 from stb1 order by ts " , "select tan(t1), tan(c5) from stb1 order by ts" )
+        self.check_result_auto_tan( " select t1,c5 from stb1 order by tbname " , "select tan(t1) ,tan(c5) from stb1 order by tbname" )
+        self.check_result_auto_tan( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select tan(t1) ,tan(c5) from stb1 where c1 > 0 order by tbname" )
+        self.check_result_auto_tan( " select t1,c5 from stb1 where c1 > 0 order by tbname  " , "select tan(t1) , tan(c5) from stb1 where c1 > 0 order by tbname" )
+        pass
+    
     
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdSql.prepare()
@@ -509,6 +523,10 @@ class TDTestCase:
 
         self.abs_func_filter()
 
+        tdLog.printNoPrefix("==========step8: check tan result of  stable query ============")
+
+        self.support_super_table_test()
+
 
     def stop(self):
         tdSql.close()

tests/system-test/7-tmq/stbTagFilter.py

@@ -0,0 +1,113 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+sys.path.append("./7-tmq")
+from tmqCommon import *
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+        #tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+
+    def tmqCase1(self):
+        tdLog.printNoPrefix("======== test case 1: ")
+        paraDict = {'dbName':     'db2',
+                    'dropFlag':   1,
+                    'event':      '',
+                    'vgroups':    4,
+                    'stbName':    'stb',
+                    'colPrefix':  'c',
+                    'tagPrefix':  't',
+                    'colSchema':   [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}],
+                    'tagSchema':   [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
+                    'ctbPrefix':  'ctb',
+                    'ctbStartIdx': 0,
+                    'ctbNum':     10,
+                    'rowsPerTbl': 1000,
+                    'batchNum':   10,
+                    'startTs':    1640966400000,  # 2022-01-01 00:00:00.000
+                    'pollDelay':  10,
+                    'showMsg':    1,
+                    'showRow':    1}
+
+        topicNameList = ['topic1']
+        expectRowsList = []
+        tmqCom.initConsumerTable()
+        tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1)
+        tdLog.info("create stb")
+        tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
+        tdLog.info("create ctb")
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data")
+        tmqCom.asyncInsertData(paraDict)
+
+        tdLog.info("create topics from stb with filter")
+        # queryString = "select ts, sin(c1), pow(c2,3) from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName'])
+        queryString = "select * from %s.%s where t2 == 'beijing' or t2 == 'changsha'" %(paraDict['dbName'], paraDict['stbName'])
+        sqlString = "create topic %s as %s" %(topicNameList[0], queryString)
+        tdLog.info("create topic sql: %s"%sqlString)
+        tdSql.execute(sqlString) 
+          
+        # start tmq consume processor
+        tdLog.info("insert consume info to consume processor")
+        consumerId   = 0
+        expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2
+        topicList    = topicNameList[0]
+        ifcheckdata  = 0
+        ifManualCommit = 1
+        keyList      = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:2000, auto.offset.reset:earliest'
+        tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
+
+        tdLog.info("start consume processor")
+        tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
+
+        # tmqCom.getStartCommitNotifyFromTmqsim()
+        tmqCom.getStartConsumeNotifyFromTmqsim()
+        tdLog.info("create some new ctb")
+        paraDict['ctbStartIdx'] = paraDict['ctbStartIdx'] + paraDict['ctbNum']
+        tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum'], ctbStartIdx=paraDict['ctbStartIdx'])
+        tdLog.info("insert data into new ctb")
+        pThread = tmqCom.asyncInsertData(paraDict)
+
+        pThread.join()
+        tdLog.info("wait insert end") 
+        tdSql.query(queryString)        
+        expectRowsList.append(tdSql.getRows())
+
+        tdLog.info("wait the consume result") 
+        expectRows = 1
+        resultList = tmqCom.selectConsumeResult(expectRows)
+        
+        if expectRowsList[0] != resultList[0]:
+            tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0]))
+            tdLog.exit("0 tmq consume rows error!")
+        
+        time.sleep(10)        
+        for i in range(len(topicNameList)):
+            tdSql.query("drop topic %s"%topicNameList[i])
+
+        tdLog.printNoPrefix("======== test case 1 end ...... ")
+
+    def run(self):
+        tdSql.prepare()
+        self.tmqCase1()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+event = threading.Event()
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/7-tmq/tmqCommon.py

@@ -159,7 +159,7 @@ class TMQCom:
         tdLog.debug("complete to create %s.%s" %(dbName, stbName))
         return
 
-    def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1):
+    def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0):
         tsql.execute("use %s" %dbName)
         pre_create = "create table"
         sql = pre_create
@@ -168,8 +168,10 @@ class TMQCom:
             tagValue = 'beijing'
             if (i % 2 == 0):
                 tagValue = 'shanghai'
+            elif (i % 3 == 0):
+                tagValue = 'changsha'
             
-            sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue)
+            sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i+ctbStartIdx,stbName,i+ctbStartIdx+1, tagValue)
             if (i > 0) and (i%100 == 0):
                 tsql.execute(sql)
                 sql = pre_create
@@ -235,7 +237,7 @@ class TMQCom:
         tdLog.debug("insert data ............ [OK]")
         return
 
-    def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs):
+    def insert_data_2(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,ctbStartIdx=0):
         tdLog.debug("start to insert data ............")
         tsql.execute("use %s" %dbName)
         pre_insert = "insert into "
@@ -245,7 +247,7 @@ class TMQCom:
         startTs = int(round(t * 1000))
         #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
         for i in range(ctbNum):
-            sql += " %s%d values "%(ctbPrefix,i)
+            sql += " %s%d values "%(ctbPrefix,i+ctbStartIdx)
             for j in range(rowsPerTbl):
                 if (j % 2 == 0):
                     sql += "(%d, %d, %d, 'tmqrow_%d', now) "%(startTs + j, j, j, j)
@@ -254,7 +256,7 @@ class TMQCom:
                 if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
                     tsql.execute(sql)
                     if j < rowsPerTbl - 1:
-                        sql = "insert into %s%d values " %(ctbPrefix,i)
+                        sql = "insert into %s%d values " %(ctbPrefix,i+ctbStartIdx)
                     else:
                         sql = "insert into "
         #end sql
@@ -354,7 +356,10 @@ class TMQCom:
     def threadFunctionForInsert(self, **paraDict):
         # create new connector for new tdSql instance in my thread
         newTdSql = tdCom.newTdSql()
-        self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
+        if 'ctbStartIdx' in paraDict.keys():
+            self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"],paraDict["ctbStartIdx"])
+        else:
+            self.insert_data_2(newTdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
         return
 
     def asyncInsertData(self, paraDict):

tests/system-test/fulltest.bat

@@ -1,102 +0,0 @@
-
-python3 .\test.py -f 0-others\taosShell.py
-python3 .\test.py -f 0-others\taosShellError.py
-python3 .\test.py -f 0-others\taosShellNetChk.py
-python3 .\test.py -f 0-others\telemetry.py
-python3 .\test.py -f 0-others\taosdMonitor.py
-python3 .\test.py -f 0-others\udfTest.py
-python3 .\test.py -f 0-others\udf_create.py
-python3 .\test.py -f 0-others\udf_restart_taosd.py
-python3 .\test.py -f 0-others\cachelast.py
-
-python3 .\test.py -f 0-others\user_control.py
-python3 .\test.py -f 0-others\fsync.py
-
-python3 .\test.py -f 1-insert\influxdb_line_taosc_insert.py
-python3 .\test.py -f 1-insert\opentsdb_telnet_line_taosc_insert.py
-python3 .\test.py -f 1-insert\opentsdb_json_taosc_insert.py
-@REM #python3 .\test.py -f 1-insert\test_stmt_muti_insert_query.py
-python3 .\test.py -f 1-insert\alter_stable.py
-python3 .\test.py -f 1-insert\alter_table.py
-python3 .\test.py -f 2-query\between.py
-python3 .\test.py -f 2-query\distinct.py
-python3 .\test.py -f 2-query\varchar.py
-python3 .\test.py -f 2-query\ltrim.py
-python3 .\test.py -f 2-query\rtrim.py
-python3 .\test.py -f 2-query\length.py
-python3 .\test.py -f 2-query\char_length.py
-python3 .\test.py -f 2-query\upper.py
-python3 .\test.py -f 2-query\lower.py
-python3 .\test.py -f 2-query\join.py
-python3 .\test.py -f 2-query\join2.py
-python3 .\test.py -f 2-query\cast.py
-python3 .\test.py -f 2-query\union.py
-python3 .\test.py -f 2-query\union1.py
-python3 .\test.py -f 2-query\concat.py
-python3 .\test.py -f 2-query\concat2.py
-python3 .\test.py -f 2-query\concat_ws.py
-python3 .\test.py -f 2-query\concat_ws2.py
-python3 .\test.py -f 2-query\check_tsdb.py
-python3 .\test.py -f 2-query\spread.py
-python3 .\test.py -f 2-query\hyperloglog.py
-
-python3 .\test.py -f 2-query\timezone.py
-python3 .\test.py -f 2-query\Now.py
-python3 .\test.py -f 2-query\Today.py
-python3 .\test.py -f 2-query\max.py
-python3 .\test.py -f 2-query\min.py
-python3 .\test.py -f 2-query\count.py
-python3 .\test.py -f 2-query\last.py
-python3 .\test.py -f 2-query\first.py
-python3 .\test.py -f 2-query\To_iso8601.py
-python3 .\test.py -f 2-query\To_unixtimestamp.py
-python3 .\test.py -f 2-query\timetruncate.py
-python3 .\test.py -f 2-query\diff.py
-python3 .\test.py -f 2-query\Timediff.py
-
-python3 .\test.py -f 2-query\top.py
-python3 .\test.py -f 2-query\bottom.py
-python3 .\test.py -f 2-query\percentile.py
-python3 .\test.py -f 2-query\apercentile.py
-python3 .\test.py -f 2-query\abs.py
-python3 .\test.py -f 2-query\ceil.py
-python3 .\test.py -f 2-query\floor.py
-python3 .\test.py -f 2-query\round.py
-python3 .\test.py -f 2-query\log.py
-python3 .\test.py -f 2-query\pow.py
-python3 .\test.py -f 2-query\sqrt.py
-python3 .\test.py -f 2-query\sin.py
-python3 .\test.py -f 2-query\cos.py
-python3 .\test.py -f 2-query\tan.py
-python3 .\test.py -f 2-query\arcsin.py
-python3 .\test.py -f 2-query\arccos.py
-python3 .\test.py -f 2-query\arctan.py
-python3 .\test.py -f 2-query\query_cols_tags_and_or.py
-@REM # python3 .\test.py -f 2-query\nestedQuery.py
-@REM # TD-15983 subquery output duplicate name column. 
-@REM # Please Xiangyang Guo modify the following script
-@REM # python3 .\test.py -f 2-query\nestedQuery_str.py
-
-python3 .\test.py -f 2-query\avg.py
-python3 .\test.py -f 2-query\elapsed.py
-python3 .\test.py -f 2-query\csum.py
-python3 .\test.py -f 2-query\mavg.py
-python3 .\test.py -f 2-query\diff.py
-python3 .\test.py -f 2-query\sample.py
-python3 .\test.py -f 2-query\function_diff.py
-python3 .\test.py -f 2-query\unique.py
-python3 .\test.py -f 2-query\stateduration.py
-python3 .\test.py -f 2-query\function_stateduration.py
-python3 .\test.py -f 2-query\statecount.py
-
-python3 .\test.py -f 7-tmq\basic5.py
-python3 .\test.py -f 7-tmq\subscribeDb.py
-python3 .\test.py -f 7-tmq\subscribeDb0.py
-python3 .\test.py -f 7-tmq\subscribeDb1.py
-python3 .\test.py -f 7-tmq\subscribeStb.py
-python3 .\test.py -f 7-tmq\subscribeStb0.py
-python3 .\test.py -f 7-tmq\subscribeStb1.py
-python3 .\test.py -f 7-tmq\subscribeStb2.py
-python3 .\test.py -f 7-tmq\subscribeStb3.py
-python3 .\test.py -f 7-tmq\subscribeStb4.py
-python3 .\test.py -f 7-tmq\db.py

tests/system-test/test-all.bat

@@ -6,19 +6,23 @@ if "%1" == "full" (
     echo Windows Taosd Full Test
     set /a exitNum=0
     del /Q /F failed.txt
-    set caseFile="fulltest.bat"
+    set caseFile="fulltest.sh"
     if not "%2" == "" (
         set caseFile="%2"
     )
     for /F "usebackq tokens=*" %%i in (!caseFile!) do (
-        for /f "tokens=1* delims= " %%a in ("%%i") do if not "%%a" == "@REM" (
+        call :CheckSkipCase %%i
-            set /a a+=1
+        if !skipCase! == false (
-            echo !a! Processing %%i
+            set line=%%i
-            call :GetTimeSeconds !time!
+            if "!line:~,7!" == "python3" (
-            set time1=!_timeTemp!
+                set /a a+=1
-            echo Start at !time!
+                echo !a! Processing %%i
-            call %%i ARG1 > result_!a!.txt 2>error_!a!.txt
+                call :GetTimeSeconds !time!
-            if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
+                set time1=!_timeTemp!
+                echo Start at !time!
+                call %%i ARG1 > result_!a!.txt 2>error_!a!.txt
+                if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. )
+            )
         )
     )
     exit !exitNum!
@@ -83,3 +87,8 @@ for %%a in (%tt%) do (
 )
 set /a _timeTemp=(%hh%*60+%mm%)*60+%ss%
 goto :eof
+
+:CheckSkipCase
+set skipCase=false
+if "%*" == "python3 ./test.py -f 1-insert/insertWithMoreVgroup.py" ( set skipCase=true )
+:goto eof

tools/shell/src/shellArguments.c

@@ -339,8 +339,8 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) {
   shell.info.clientVersion =
       "Welcome to the TDengine shell from %s, Client Version:%s\n"
       "Copyright (c) 2022 by TAOS Data, Inc. All rights reserved.\n\n";
-  shell.info.promptHeader = "taos> ";
+  shell.info.promptHeader = TAOS_CONSOLE_PROMPT_HEADER;
-  shell.info.promptContinue = "   -> ";
+  shell.info.promptContinue = TAOS_CONSOLE_PROMPT_CONTINUE;
   shell.info.promptSize = 6;
   snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version);
 

tools/shell/src/shellCommand.c

@@ -411,8 +411,9 @@ char taosGetConsoleChar() {
   static char mbStr[5];
   static unsigned long bufLen = 0;
   static uint16_t bufIndex = 0, mbStrIndex = 0, mbStrLen = 0;
-  if (bufLen == 0) {
+  while (bufLen == 0) {
     ReadConsoleW(console, buf, SHELL_INPUT_MAX_COMMAND_SIZE, &bufLen, NULL);
+    if (bufLen > 0 && buf[0] == 0) bufLen = 0;
     bufIndex = 0;
   }
   if (mbStrLen == 0){
@@ -466,6 +467,8 @@ int32_t shellReadCommand(char *command) {
     } else if (c < '\033') {
       // Ctrl keys.  TODO: Implement ctrl combinations
       switch (c) {
+        case 0:
+          break;
         case 1:  // ctrl A
           shellPositionCursorHome(&cmd);
           break;