Merge branch '3.0' into enh/TS-3737-3.0

include/common/tmsg.h

@@ -2366,20 +2366,31 @@ void    tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp* pRsp);
 typedef struct {
   int32_t dnodeId;
   char*   token;
-} SMArbUpdateGroupReqMember;
+} SMArbUpdateGroupMember;
 
 typedef struct {
-  int32_t vgId;
+  int32_t dnodeId;
-  int64_t dbUid;
+  char*   token;
-  SMArbUpdateGroupReqMember members[2];
+  int8_t  acked;
-  int8_t                    isSync;
+} SMArbUpdateGroupAssigned;
-  SMArbUpdateGroupReqMember assignedLeader;
-  int64_t                   version;
-} SMArbUpdateGroupReq;
 
-int32_t tSerializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq);
+typedef struct {
-int32_t tDeserializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq);
+  int32_t                  vgId;
-void    tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq* pReq);
+  int64_t                  dbUid;
+  SMArbUpdateGroupMember   members[2];
+  int8_t                   isSync;
+  int8_t                   assignedAcked;
+  SMArbUpdateGroupAssigned assignedLeader;
+  int64_t                  version;
+} SMArbUpdateGroup;
+
+typedef struct {
+  SArray* updateArray;  // SMArbUpdateGroup
+} SMArbUpdateGroupBatchReq;
+
+int32_t tSerializeSMArbUpdateGroupBatchReq(void* buf, int32_t bufLen, SMArbUpdateGroupBatchReq* pReq);
+int32_t tDeserializeSMArbUpdateGroupBatchReq(void* buf, int32_t bufLen, SMArbUpdateGroupBatchReq* pReq);
+void    tFreeSMArbUpdateGroupBatchReq(SMArbUpdateGroupBatchReq* pReq);
 
 typedef struct {
   char queryStrId[TSDB_QUERY_ID_LEN];

include/common/tmsgdef.h

@@ -388,7 +388,8 @@
   TD_NEW_MSG_SEG(TDMT_MND_ARB_MSG)  //9 << 8
   TD_DEF_MSG_TYPE(TDMT_MND_ARB_HEARTBEAT_TIMER, "mnd-arb-hb-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_ARB_CHECK_SYNC_TIMER, "mnd-arb-check-sync-tmr", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL) // no longer used
+  TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP_BATCH, "mnd-arb-update-group-batch", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_ARB_MAX_MSG, "mnd-arb-max", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_ARB_MSG)
 

include/dnode/mgmt/dnode.h

@@ -44,6 +44,11 @@ int32_t dmRun();
  */
 void dmStop();
 
+/**
+ * for tests
+ */
+bool dmReadyForTest();
+
 #ifdef __cplusplus
 }
 #endif

include/dnode/mnode/mnode.h

@@ -101,6 +101,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
  */
 int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad);
 
+int64_t mndGetRoleTimeMs(SMnode *pMnode);
+
 /**
  * @brief Process the rpc, sync request.
  *

include/libs/nodes/plannodes.h

@@ -183,6 +183,7 @@ typedef struct SProjectLogicNode {
   char       stmtName[TSDB_TABLE_NAME_LEN];
   bool       ignoreGroupId;
   bool       inputIgnoreGroup;
+  bool       isSetOpProj;
 } SProjectLogicNode;
 
 typedef struct SIndefRowsFuncLogicNode {

include/libs/nodes/querynodes.h

@@ -415,6 +415,7 @@ typedef struct SSelectStmt {
   int32_t       returnRows;  // EFuncReturnRows
   ETimeLineMode timeLineCurMode;
   ETimeLineMode timeLineResMode;
+  bool          timeLineFromOrderBy;
   bool          isEmptyResult;
   bool          isSubquery;
   bool          hasAggFuncs;
@@ -453,6 +454,7 @@ typedef struct SSetOperator {
   char             stmtName[TSDB_TABLE_NAME_LEN];
   uint8_t          precision;
   ETimeLineMode    timeLineResMode;  
+  bool             timeLineFromOrderBy;
   bool             joinContains;
 } SSetOperator;
 

include/util/taoserror.h

@@ -327,7 +327,6 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_DB_IN_CREATING            TAOS_DEF_ERROR_CODE(0, 0x0396) //
 #define TSDB_CODE_MND_INVALID_SYS_TABLENAME     TAOS_DEF_ERROR_CODE(0, 0x039A)
 #define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE  TAOS_DEF_ERROR_CODE(0, 0x039B)
-#define TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED  TAOS_DEF_ERROR_CODE(0, 0x039C)
 
 // mnode-node
 #define TSDB_CODE_MND_MNODE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03A0)
@@ -612,6 +611,16 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE    TAOS_DEF_ERROR_CODE(0, 0x0821)
 #define TSDB_CODE_GRANT_DUPLICATED_ACTIVE       TAOS_DEF_ERROR_CODE(0, 0x0822)
 #define TSDB_CODE_GRANT_VIEW_LIMITED            TAOS_DEF_ERROR_CODE(0, 0x0823)
+#define TSDB_CODE_GRANT_BASIC_EXPIRED           TAOS_DEF_ERROR_CODE(0, 0x0824)
+#define TSDB_CODE_GRANT_STREAM_EXPIRED          TAOS_DEF_ERROR_CODE(0, 0x0825)
+#define TSDB_CODE_GRANT_SUBSCRIPTION_EXPIRED    TAOS_DEF_ERROR_CODE(0, 0x0826)
+#define TSDB_CODE_GRANT_VIEW_EXPIRED            TAOS_DEF_ERROR_CODE(0, 0x0827)
+#define TSDB_CODE_GRANT_AUDIT_EXPIRED           TAOS_DEF_ERROR_CODE(0, 0x0828)
+#define TSDB_CODE_GRANT_CSV_EXPIRED             TAOS_DEF_ERROR_CODE(0, 0x0829)
+#define TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED   TAOS_DEF_ERROR_CODE(0, 0x082A)
+#define TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED  TAOS_DEF_ERROR_CODE(0, 0x082B)
+#define TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082C)
+#define TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED   TAOS_DEF_ERROR_CODE(0, 0x082D)
 
 // sync
 // #define TSDB_CODE_SYN_INVALID_CONFIG         TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x

source/client/src/clientHb.c

@@ -92,6 +92,7 @@ static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *bat
         }
         if (!pRsp) {
           releaseTscObj(pReq->connKey.tscRid);
+          taosHashCancelIterate(hbMgr->activeInfo, pReq);
           break;
         }
       }

source/common/src/systable.c

@@ -76,6 +76,7 @@ static const SSysDbTableSchema arbGroupsSchema[] = {
     {.name = "is_sync", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
     {.name = "assigned_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
     {.name = "assigned_token", .bytes = TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
+    {.name = "assigned_acked", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
 };
 
 static const SSysDbTableSchema clusterSchema[] = {

source/common/src/tdataformat.c

@@ -2907,7 +2907,7 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt
         }
       } else {
         if (varDataTLen(data + offset) > bytes) {
-          uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset),
+          uError("var data length invalid, varDataTLen(data + offset):%d >= bytes:%d", (int)varDataTLen(data + offset),
                  bytes);
           code = TSDB_CODE_PAR_VALUE_TOO_LONG;
           goto _exit;

source/common/src/tmsg.c

@@ -6441,21 +6441,33 @@ void tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp *pRsp) {
   taosMemoryFreeClear(pRsp->memberToken);
 }
 
-int32_t tSerializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) {
+int32_t tSerializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpdateGroupBatchReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
 
   if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
+
-  if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1;
+  int32_t sz = taosArrayGetSize(pReq->updateArray);
-  for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+  if (tEncodeI32(&encoder, sz) < 0) return -1;
-    if (tEncodeI32(&encoder, pReq->members[i].dnodeId) < 0) return -1;
+
-    if (tEncodeCStr(&encoder, pReq->members[i].token) < 0) return -1;
+  for (int32_t i = 0; i < sz; i++) {
+    SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i);
+    if (tEncodeI32(&encoder, pGroup->vgId) < 0) return -1;
+    if (tEncodeI64(&encoder, pGroup->dbUid) < 0) return -1;
+    for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+      if (tEncodeI32(&encoder, pGroup->members[i].dnodeId) < 0) return -1;
+      if (tEncodeCStr(&encoder, pGroup->members[i].token) < 0) return -1;
+    }
+    if (tEncodeI8(&encoder, pGroup->isSync) < 0) return -1;
+    if (tEncodeI32(&encoder, pGroup->assignedLeader.dnodeId) < 0) return -1;
+    if (tEncodeCStr(&encoder, pGroup->assignedLeader.token) < 0) return -1;
+    if (tEncodeI64(&encoder, pGroup->version) < 0) return -1;
+  }
+
+  for (int32_t i = 0; i < sz; i++) {
+    SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i);
+    if (tEncodeI8(&encoder, pGroup->assignedLeader.acked) < 0) return -1;
   }
-  if (tEncodeI8(&encoder, pReq->isSync) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->assignedLeader.dnodeId) < 0) return -1;
-  if (tEncodeCStr(&encoder, pReq->assignedLeader.token) < 0) return -1;
-  if (tEncodeI64(&encoder, pReq->version) < 0) return -1;
 
   tEndEncode(&encoder);
 
@@ -6464,23 +6476,44 @@ int32_t tSerializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGrou
   return tlen;
 }
 
-int32_t tDeserializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) {
+int32_t tDeserializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpdateGroupBatchReq *pReq) {
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
 
   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
+  int32_t sz = 0;
-  if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1;
+  if (tDecodeI32(&decoder, &sz) < 0) return -1;
-  for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+
-    if (tDecodeI32(&decoder, &pReq->members[i].dnodeId) < 0) return -1;
+  SArray *updateArray = taosArrayInit(sz, sizeof(SMArbUpdateGroup));
-    pReq->members[i].token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
+  if (!updateArray) return -1;
-    if (tDecodeCStrTo(&decoder, pReq->members[i].token) < 0) return -1;
+
+  for (int32_t i = 0; i < sz; i++) {
+    SMArbUpdateGroup group = {0};
+    if (tDecodeI32(&decoder, &group.vgId) < 0) return -1;
+    if (tDecodeI64(&decoder, &group.dbUid) < 0) return -1;
+    for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+      if (tDecodeI32(&decoder, &group.members[i].dnodeId) < 0) return -1;
+      group.members[i].token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
+      if (tDecodeCStrTo(&decoder, group.members[i].token) < 0) return -1;
+    }
+    if (tDecodeI8(&decoder, &group.isSync) < 0) return -1;
+    if (tDecodeI32(&decoder, &group.assignedLeader.dnodeId) < 0) return -1;
+    group.assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
+    if (tDecodeCStrTo(&decoder, group.assignedLeader.token) < 0) return -1;
+    if (tDecodeI64(&decoder, &group.version) < 0) return -1;
+    group.assignedLeader.acked = false;
+
+    taosArrayPush(updateArray, &group);
   }
-  if (tDecodeI8(&decoder, &pReq->isSync) < 0) return -1;
+
-  if (tDecodeI32(&decoder, &pReq->assignedLeader.dnodeId) < 0) return -1;
+  if (!tDecodeIsEnd(&decoder)) {
-  pReq->assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE);
+    for (int32_t i = 0; i < sz; i++) {
-  if (tDecodeCStrTo(&decoder, pReq->assignedLeader.token) < 0) return -1;
+      SMArbUpdateGroup *pGroup = taosArrayGet(updateArray, i);
-  if (tDecodeI64(&decoder, &pReq->version) < 0) return -1;
+      if (tDecodeI8(&decoder, &pGroup->assignedLeader.acked) < 0) return -1;
+    }
+  }
+
+  pReq->updateArray = updateArray;
 
   tEndDecode(&decoder);
 
@@ -6488,14 +6521,20 @@ int32_t tDeserializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGr
   return 0;
 }
 
-void tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq *pReq) {
+void tFreeSMArbUpdateGroupBatchReq(SMArbUpdateGroupBatchReq *pReq) {
-  if (NULL == pReq) {
+  if (NULL == pReq || NULL == pReq->updateArray) {
     return;
   }
-  for (int i = 0; i < 2; i++) {
+
-    taosMemoryFreeClear(pReq->members[i].token);
+  int32_t sz = taosArrayGetSize(pReq->updateArray);
+  for (int32_t i = 0; i < sz; i++) {
+    SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i);
+    for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+      taosMemoryFreeClear(pGroup->members[i].token);
+    }
+    taosMemoryFreeClear(pGroup->assignedLeader.token);
   }
-  taosMemoryFreeClear(pReq->assignedLeader.token);
+  taosArrayDestroy(pReq->updateArray);
 }
 
 // int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) {

source/dnode/mgmt/node_mgmt/src/dmEnv.c

@@ -415,3 +415,7 @@ void dmReportStartup(const char *pName, const char *pDesc) {
 }
 
 int64_t dmGetClusterId() { return globalDnode.data.clusterId; }
+
+bool dmReadyForTest() {
+  return dmInstance()->data.dnodeVer > 0;
+}

source/dnode/mgmt/test/sut/inc/server.h

@@ -20,10 +20,10 @@ class TestServer {
  public:
   bool Start();
   void Stop();
-bool runnning;
+  bool running;
   
  private:
   TdThread threadId;
 };
 
-#endif /* _TD_TEST_SERVER_H_ */
+#endif /* _TD_TEST_SERVER_H_ */

source/dnode/mgmt/test/sut/src/server.cpp

@@ -17,13 +17,11 @@
 
 void* serverLoop(void* param) {
   TestServer* server = (TestServer*)param;
-  server->runnning = false;
 
   if (dmInit() != 0) {
     return NULL;
   }
 
-  server->runnning = true;
   if (dmRun() != 0) {
     return NULL;
   }
@@ -33,13 +31,18 @@ void* serverLoop(void* param) {
 }
 
 bool TestServer::Start() {
+  tstrncpy(tsVersionName, "trial", strlen("trial"));
+  running = false;
   TdThreadAttr thAttr;
   taosThreadAttrInit(&thAttr);
   taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
   taosThreadCreate(&threadId, &thAttr, serverLoop, this);
   taosThreadAttrDestroy(&thAttr);
-  taosMsleep(2100);
+  while (!dmReadyForTest()) {
-  return runnning;
+    taosMsleep(500);
+  }
+  running = true;
+  return running;
 }
 
 void TestServer::Stop() {

source/dnode/mnode/impl/inc/mndDef.h

@@ -180,7 +180,7 @@ typedef struct {
   tmsg_t        originRpcType;
   char          dbname[TSDB_TABLE_FNAME_LEN];
   char          stbname[TSDB_TABLE_FNAME_LEN];
-  int32_t       arbGroupId;
+  SHashObj*     arbGroupIds;
   int32_t       startFunc;
   int32_t       stopFunc;
   int32_t       paramLen;
@@ -255,6 +255,7 @@ typedef struct {
 typedef struct {
   int32_t dnodeId;
   char    token[TSDB_ARB_TOKEN_SIZE];
+  int8_t  acked;
 } SArbAssignedLeader;
 
 typedef struct {

source/dnode/mnode/impl/inc/mndTrans.h

@@ -78,7 +78,7 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
 void    mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
 void    mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen);
 void    mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname);
-void    mndTransSetArbGroupId(STrans *pTrans, int32_t groupId);
+void    mndTransAddArbGroupId(STrans *pTrans, int32_t groupId);
 void    mndTransSetSerial(STrans *pTrans);
 void    mndTransSetParallel(STrans *pTrans);
 void    mndTransSetChangeless(STrans *pTrans);

source/dnode/mnode/impl/src/mndArbGroup.c

@@ -25,7 +25,7 @@
 #include "mndVgroup.h"
 
 #define ARBGROUP_VER_NUMBER   1
-#define ARBGROUP_RESERVE_SIZE 64
+#define ARBGROUP_RESERVE_SIZE 63
 
 static SHashObj *arbUpdateHash = NULL;
 
@@ -39,10 +39,11 @@ static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup);
 
 static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew);
 static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup);
+static int32_t mndPullupArbUpdateGroupBatch(SMnode *pMnode, SArray *newGroupArray);
 
 static int32_t mndProcessArbHbTimer(SRpcMsg *pReq);
 static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq);
-static int32_t mndProcessArbUpdateGroupReq(SRpcMsg *pReq);
+static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq);
 static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp);
 static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp);
 static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp);
@@ -68,7 +69,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) {
 
   mndSetMsgHandle(pMnode, TDMT_MND_ARB_HEARTBEAT_TIMER, mndProcessArbHbTimer);
   mndSetMsgHandle(pMnode, TDMT_MND_ARB_CHECK_SYNC_TIMER, mndProcessArbCheckSyncTimer);
-  mndSetMsgHandle(pMnode, TDMT_MND_ARB_UPDATE_GROUP, mndProcessArbUpdateGroupReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_ARB_UPDATE_GROUP_BATCH, mndProcessArbUpdateGroupBatchReq);
   mndSetMsgHandle(pMnode, TDMT_VND_ARB_HEARTBEAT_RSP, mndProcessArbHbRsp);
   mndSetMsgHandle(pMnode, TDMT_VND_ARB_CHECK_SYNC_RSP, mndProcessArbCheckSyncRsp);
   mndSetMsgHandle(pMnode, TDMT_SYNC_SET_ASSIGNED_LEADER_RSP, mndProcessArbSetAssignedLeaderRsp);
@@ -81,9 +82,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) {
   return sdbSetTable(pMnode->pSdb, table);
 }
 
-void mndCleanupArbGroup(SMnode *pMnode) {
+void mndCleanupArbGroup(SMnode *pMnode) { taosHashCleanup(arbUpdateHash); }
-  taosHashCleanup(arbUpdateHash);
-}
 
 SArbGroup *mndAcquireArbGroup(SMnode *pMnode, int32_t vgId) {
   SArbGroup *pGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &vgId);
@@ -130,6 +129,7 @@ SSdbRaw *mndArbGroupActionEncode(SArbGroup *pGroup) {
   SDB_SET_INT32(pRaw, dataPos, pLeader->dnodeId, _OVER)
   SDB_SET_BINARY(pRaw, dataPos, pLeader->token, TSDB_ARB_TOKEN_SIZE, _OVER)
   SDB_SET_INT64(pRaw, dataPos, pGroup->version, _OVER)
+  SDB_SET_INT8(pRaw, dataPos, pLeader->acked, _OVER)
 
   SDB_SET_RESERVE(pRaw, dataPos, ARBGROUP_RESERVE_SIZE, _OVER)
 
@@ -183,6 +183,7 @@ SSdbRow *mndArbGroupActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT32(pRaw, dataPos, &pLeader->dnodeId, _OVER)
   SDB_GET_BINARY(pRaw, dataPos, pLeader->token, TSDB_ARB_TOKEN_SIZE, _OVER)
   SDB_GET_INT64(pRaw, dataPos, &pGroup->version, _OVER)
+  SDB_GET_INT8(pRaw, dataPos, &pLeader->acked, _OVER)
 
   pGroup->mutexInited = false;
 
@@ -236,6 +237,7 @@ static int32_t mndArbGroupActionUpdate(SSdb *pSdb, SArbGroup *pOld, SArbGroup *p
   pOld->isSync = pNew->isSync;
   pOld->assignedLeader.dnodeId = pNew->assignedLeader.dnodeId;
   memcpy(pOld->assignedLeader.token, pNew->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
+  pOld->assignedLeader.acked = pNew->assignedLeader.acked;
   pOld->version++;
 
 _OVER:
@@ -541,6 +543,16 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
     return -1;
   }
 
+  int64_t roleTimeMs = mndGetRoleTimeMs(pMnode);
+  int64_t nowMs = taosGetTimestampMs();
+  if (nowMs - roleTimeMs < tsArbHeartBeatIntervalSec * 1000 * 2) {
+    mInfo("arb skip to check sync since mnd had just switch over, roleTime:%" PRId64 " now:%" PRId64, roleTimeMs,
+          nowMs);
+    return 0;
+  }
+
+  SArray *pUpdateArray = taosArrayInit(16, sizeof(SArbGroup));
+
   while (1) {
     pIter = sdbFetch(pSdb, SDB_ARBGROUP, pIter, (void **)&pArbGroup);
     if (pIter == NULL) break;
@@ -550,15 +562,14 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
     taosThreadMutexUnlock(&pArbGroup->mutex);
 
     int32_t vgId = arbGroupDup.vgId;
-    int64_t nowMs = taosGetTimestampMs();
 
     bool                member0IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 0, nowMs);
     bool                member1IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 1, nowMs);
     SArbAssignedLeader *pAssignedLeader = &arbGroupDup.assignedLeader;
     int32_t             currentAssignedDnodeId = pAssignedLeader->dnodeId;
 
-    // 1. has assigned && is sync => send req
+    // 1. has assigned && is sync && no response => send req
-    if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true) {
+    if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true && pAssignedLeader->acked == false) {
       (void)mndSendArbSetAssignedLeaderReq(pMnode, currentAssignedDnodeId, vgId, arbToken, term,
                                            pAssignedLeader->token);
       mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", vgId, currentAssignedDnodeId);
@@ -612,40 +623,27 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
     SArbGroup newGroup = {0};
     mndArbGroupDupObj(&arbGroupDup, &newGroup);
     mndArbGroupSetAssignedLeader(&newGroup, candidateIndex);
-    if (mndPullupArbUpdateGroup(pMnode, &newGroup) != 0) {
+    taosArrayPush(pUpdateArray, &newGroup);
-      mError("vgId:%d, arb failed to pullup set assigned leader to dnodeId:%d, since %s", vgId, pMember->info.dnodeId,
-             terrstr());
-      sdbRelease(pSdb, pArbGroup);
-      return -1;
-    }
-
-    mInfo("vgId:%d, arb pull up set assigned leader to dnodeId:%d", vgId, pMember->info.dnodeId);
 
     sdbRelease(pSdb, pArbGroup);
   }
 
+  (void)mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray);
+
+  taosArrayDestroy(pUpdateArray);
   return 0;
 }
 
-static void *mndBuildArbUpdateGroupReq(int32_t *pContLen, SArbGroup *pNewGroup) {
+static void *mndBuildArbUpdateGroupBatchReq(int32_t *pContLen, SArray *updateArray) {
-  SMArbUpdateGroupReq req = {0};
+  SMArbUpdateGroupBatchReq req = {0};
-  req.vgId = pNewGroup->vgId;
+  req.updateArray = updateArray;
-  req.dbUid = pNewGroup->dbUid;
-  for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
-    req.members[i].dnodeId = pNewGroup->members[i].info.dnodeId;
-    req.members[i].token = pNewGroup->members[i].state.token;
-  }
-  req.isSync = pNewGroup->isSync;
-  req.assignedLeader.dnodeId = pNewGroup->assignedLeader.dnodeId;
-  req.assignedLeader.token = pNewGroup->assignedLeader.token;
-  req.version = pNewGroup->version;
 
-  int32_t contLen = tSerializeSMArbUpdateGroupReq(NULL, 0, &req);
+  int32_t contLen = tSerializeSMArbUpdateGroupBatchReq(NULL, 0, &req);
   if (contLen <= 0) return NULL;
   SMsgHead *pHead = rpcMallocCont(contLen);
   if (pHead == NULL) return NULL;
 
-  if (tSerializeSMArbUpdateGroupReq(pHead, contLen, &req) <= 0) {
+  if (tSerializeSMArbUpdateGroupBatchReq(pHead, contLen, &req) <= 0) {
     rpcFreeCont(pHead);
     return NULL;
   }
@@ -653,60 +651,174 @@ static void *mndBuildArbUpdateGroupReq(int32_t *pContLen, SArbGroup *pNewGroup)
   return pHead;
 }
 
+static void mndInitArbUpdateGroup(SArbGroup *pGroup, SMArbUpdateGroup *outGroup) {
+  outGroup->vgId = pGroup->vgId;
+  outGroup->dbUid = pGroup->dbUid;
+  for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+    outGroup->members[i].dnodeId = pGroup->members[i].info.dnodeId;
+    outGroup->members[i].token = pGroup->members[i].state.token;  // just copy the pointer
+  }
+  outGroup->isSync = pGroup->isSync;
+  outGroup->assignedLeader.dnodeId = pGroup->assignedLeader.dnodeId;
+  outGroup->assignedLeader.token = pGroup->assignedLeader.token;  // just copy the pointer
+  outGroup->assignedLeader.acked = pGroup->assignedLeader.acked;
+  outGroup->version = pGroup->version;
+}
+
 static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup) {
   if (taosHashGet(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)) != NULL) {
     mInfo("vgId:%d, arb skip to pullup arb-update-group request, since it is in process", pNewGroup->vgId);
     return 0;
   }
 
-  int32_t contLen = 0;
+  int32_t ret = -1;
-  void   *pHead = mndBuildArbUpdateGroupReq(&contLen, pNewGroup);
-  if (!pHead) {
-    mError("vgId:%d, failed to build arb-update-group request", pNewGroup->vgId);
-    return -1;
-  }
-  SRpcMsg rpcMsg = {.msgType = TDMT_MND_ARB_UPDATE_GROUP, .pCont = pHead, .contLen = contLen, .info.noResp = true};
 
-  int32_t ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+  SMArbUpdateGroup newGroup = {0};
-  if (ret == 0) {
+  mndInitArbUpdateGroup(pNewGroup, &newGroup);
-    taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0);
+
+  SArray *pArray = taosArrayInit(1, sizeof(SMArbUpdateGroup));
+  taosArrayPush(pArray, &newGroup);
+
+  int32_t contLen = 0;
+  void   *pHead = mndBuildArbUpdateGroupBatchReq(&contLen, pArray);
+  if (!pHead) {
+    mError("failed to build arb-update-group request");
+    goto _OVER;
   }
+
+  SRpcMsg rpcMsg = {
+      .msgType = TDMT_MND_ARB_UPDATE_GROUP_BATCH, .pCont = pHead, .contLen = contLen, .info.noResp = true};
+  ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+  if (ret != 0) goto _OVER;
+
+  taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0);
+
+_OVER:
+  taosArrayDestroy(pArray);
   return ret;
 }
 
-static int32_t mndProcessArbUpdateGroupReq(SRpcMsg *pReq) {
+static int32_t mndPullupArbUpdateGroupBatch(SMnode *pMnode, SArray *newGroupArray) {
-  int ret = 0;
+  int32_t ret = -1;
 
-  SMArbUpdateGroupReq req = {0};
+  size_t  sz = taosArrayGetSize(newGroupArray);
-  tDeserializeSMArbUpdateGroupReq(pReq->pCont, pReq->contLen, &req);
+  SArray *pArray = taosArrayInit(sz, sizeof(SMArbUpdateGroup));
+  for (size_t i = 0; i < sz; i++) {
+    SArbGroup *pNewGroup = taosArrayGet(newGroupArray, i);
+    if (taosHashGet(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)) != NULL) {
+      mInfo("vgId:%d, arb skip to pullup arb-update-group request, since it is in process", pNewGroup->vgId);
+      continue;
+    }
 
-  SArbGroup newGroup = {0};
+    SMArbUpdateGroup newGroup = {0};
-  newGroup.vgId = req.vgId;
+    mndInitArbUpdateGroup(pNewGroup, &newGroup);
-  newGroup.dbUid = req.dbUid;
+
-  for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+    taosArrayPush(pArray, &newGroup);
-    newGroup.members[i].info.dnodeId = req.members[i].dnodeId;
+    taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0);
-    memcpy(newGroup.members[i].state.token, req.members[i].token, TSDB_ARB_TOKEN_SIZE);
   }
 
-  newGroup.isSync = req.isSync;
+  if (taosArrayGetSize(pArray) == 0) {
-  newGroup.assignedLeader.dnodeId = req.assignedLeader.dnodeId;
+    ret = 0;
-  memcpy(newGroup.assignedLeader.token, req.assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
+    goto _OVER;
-  newGroup.version = req.version;
-
-  SMnode    *pMnode = pReq->info.node;
-  SArbGroup *pOldGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &newGroup.vgId);
-  if (!pOldGroup) {
-    mInfo("vgId:%d, arb skip to update arbgroup, since no obj found", newGroup.vgId);
-    return 0;
-  }
-  sdbRelease(pMnode->pSdb, pOldGroup);
-
-  if (mndArbGroupUpdateTrans(pMnode, &newGroup) != 0) {
-    mError("vgId:%d, arb failed to update arbgroup, since %s", newGroup.vgId, terrstr());
-    ret = -1;
   }
 
-  tFreeSMArbUpdateGroupReq(&req);
+  int32_t contLen = 0;
+  void   *pHead = mndBuildArbUpdateGroupBatchReq(&contLen, pArray);
+  if (!pHead) {
+    mError("failed to build arb-update-group request");
+    goto _OVER;
+  }
+
+  SRpcMsg rpcMsg = {
+      .msgType = TDMT_MND_ARB_UPDATE_GROUP_BATCH, .pCont = pHead, .contLen = contLen, .info.noResp = true};
+  ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
+
+_OVER:
+  taosArrayDestroy(pArray);
+
+  if (ret != 0) {
+    for (size_t i = 0; i < sz; i++) {
+      SArbGroup *pNewGroup = taosArrayGet(newGroupArray, i);
+      taosHashRemove(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId));
+    }
+  }
+
+  return ret;
+}
+
+static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
+  int    ret = -1;
+  size_t sz = 0;
+
+  SMArbUpdateGroupBatchReq req = {0};
+  if (tDeserializeSMArbUpdateGroupBatchReq(pReq->pCont, pReq->contLen, &req) != 0) {
+    mError("arb failed to decode arb-update-group request");
+    return -1;
+  }
+
+  SMnode *pMnode = pReq->info.node;
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-arbgroup");
+  if (pTrans == NULL) {
+    mError("failed to update arbgroup in create trans, since %s", terrstr());
+    goto _OVER;
+  }
+
+  sz = taosArrayGetSize(req.updateArray);
+  for (size_t i = 0; i < sz; i++) {
+    SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i);
+    SArbGroup         newGroup = {0};
+    newGroup.vgId = pUpdateGroup->vgId;
+    newGroup.dbUid = pUpdateGroup->dbUid;
+    for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) {
+      newGroup.members[i].info.dnodeId = pUpdateGroup->members[i].dnodeId;
+      memcpy(newGroup.members[i].state.token, pUpdateGroup->members[i].token, TSDB_ARB_TOKEN_SIZE);
+    }
+
+    newGroup.isSync = pUpdateGroup->isSync;
+    newGroup.assignedLeader.dnodeId = pUpdateGroup->assignedLeader.dnodeId;
+    memcpy(newGroup.assignedLeader.token, pUpdateGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE);
+    newGroup.assignedLeader.acked = pUpdateGroup->assignedLeader.acked;
+    newGroup.version = pUpdateGroup->version;
+
+    SArbGroup *pOldGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &newGroup.vgId);
+    if (!pOldGroup) {
+      mInfo("vgId:%d, arb skip to update arbgroup, since no obj found", newGroup.vgId);
+      taosHashRemove(arbUpdateHash, &newGroup.vgId, sizeof(int32_t));
+      continue;
+    }
+
+    mndTransAddArbGroupId(pTrans, newGroup.vgId);
+
+    if (mndSetCreateArbGroupCommitLogs(pTrans, &newGroup) != 0) {
+      mError("failed to update arbgroup in set commit log, vgId:%d, trans:%d, since %s", newGroup.vgId, pTrans->id,
+             terrstr());
+      goto _OVER;
+    }
+
+    mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s][%d]",
+          pTrans->id, newGroup.vgId, newGroup.members[0].info.dnodeId, newGroup.members[0].state.token,
+          newGroup.members[1].info.dnodeId, newGroup.members[1].state.token, newGroup.isSync,
+          newGroup.assignedLeader.dnodeId, newGroup.assignedLeader.token, newGroup.assignedLeader.acked);
+
+    sdbRelease(pMnode->pSdb, pOldGroup);
+  }
+
+  if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
+
+  ret = 0;
+
+_OVER:
+  if (ret != 0) {
+    // failed to update arbgroup
+    for (size_t i = 0; i < sz; i++) {
+      SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i);
+      taosHashRemove(arbUpdateHash, &pUpdateGroup->vgId, sizeof(int32_t));
+    }
+  }
+
+  mndTransDrop(pTrans);
+  tFreeSMArbUpdateGroupBatchReq(&req);
   return ret;
 }
 
@@ -719,11 +831,13 @@ static void mndArbGroupSetAssignedLeader(SArbGroup *pGroup, int32_t index) {
 
   pGroup->assignedLeader.dnodeId = pMember->info.dnodeId;
   strncpy(pGroup->assignedLeader.token, pMember->state.token, TSDB_ARB_TOKEN_SIZE);
+  pGroup->assignedLeader.acked = false;
 }
 
 static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup) {
   pGroup->assignedLeader.dnodeId = 0;
   memset(pGroup->assignedLeader.token, 0, TSDB_ARB_TOKEN_SIZE);
+  pGroup->assignedLeader.acked = false;
 }
 
 static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
@@ -734,12 +848,12 @@ static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
     goto _OVER;
   }
 
-  mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s]",
+  mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s][%d]",
         pTrans->id, pNew->vgId, pNew->members[0].info.dnodeId, pNew->members[0].state.token,
         pNew->members[1].info.dnodeId, pNew->members[1].state.token, pNew->isSync, pNew->assignedLeader.dnodeId,
-        pNew->assignedLeader.token);
+        pNew->assignedLeader.token, pNew->assignedLeader.acked);
 
-  mndTransSetArbGroupId(pTrans, pNew->vgId);
+  mndTransAddArbGroupId(pTrans, pNew->vgId);
   if (mndTransCheckConflict(pMnode, pTrans) != 0) {
     ret = -1;
     goto _OVER;
@@ -816,10 +930,10 @@ _OVER:
 }
 
 static int32_t mndUpdateArbHeartBeat(SMnode *pMnode, int32_t dnodeId, SArray *memberArray) {
-  int     ret = 0;
   int64_t nowMs = taosGetTimestampMs();
+  size_t  size = taosArrayGetSize(memberArray);
+  SArray *pUpdateArray = taosArrayInit(size, sizeof(SArbGroup));
 
-  size_t size = taosArrayGetSize(memberArray);
   for (size_t i = 0; i < size; i++) {
     SVArbHbRspMember *pRspMember = taosArrayGet(memberArray, i);
 
@@ -832,17 +946,16 @@ static int32_t mndUpdateArbHeartBeat(SMnode *pMnode, int32_t dnodeId, SArray *me
 
     bool updateToken = mndUpdateArbGroupByHeartBeat(pGroup, pRspMember, nowMs, dnodeId, &newGroup);
     if (updateToken) {
-      ret = mndPullupArbUpdateGroup(pMnode, &newGroup);
+      taosArrayPush(pUpdateArray, &newGroup);
-      if (ret != 0) {
-        mInfo("failed to pullup update arb token, vgId:%d, since %s", pRspMember->vgId, terrstr());
-      }
     }
 
     sdbRelease(pMnode->pSdb, pGroup);
-    if (ret != 0) break;
   }
 
-  return ret;
+  (void)mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray);
+
+  taosArrayDestroy(pUpdateArray);
+  return 0;
 }
 
 bool mndUpdateArbGroupByCheckSync(SArbGroup *pGroup, int32_t vgId, char *member0Token, char *member1Token,
@@ -900,6 +1013,11 @@ static int32_t mndUpdateArbSync(SMnode *pMnode, int32_t vgId, char *member0Token
 }
 
 static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) {
+  if (pRsp->contLen == 0) {
+    mDebug("arb hb-rsp contLen is 0");
+    return 0;
+  }
+
   int32_t ret = -1;
 
   SMnode *pMnode = pRsp->info.node;
@@ -914,6 +1032,7 @@ static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) {
 
   SVArbHeartBeatRsp arbHbRsp = {0};
   if (tDeserializeSVArbHeartBeatRsp(pRsp->pCont, pRsp->contLen, &arbHbRsp) != 0) {
+    mInfo("arb hb-rsp des failed, since:%s", tstrerror(pRsp->code));
     terrno = TSDB_CODE_INVALID_MSG;
     return -1;
   }
@@ -934,6 +1053,11 @@ _OVER:
 }
 
 static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
+  if (pRsp->contLen == 0) {
+    mDebug("arb check-sync-rsp contLen is 0");
+    return 0;
+  }
+
   int32_t ret = -1;
 
   SMnode *pMnode = pRsp->info.node;
@@ -948,7 +1072,7 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
 
   SVArbCheckSyncRsp syncRsp = {0};
   if (tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp) != 0) {
-    mInfo("arb sync check failed, since:%s", tstrerror(pRsp->code));
+    mInfo("arb check-sync-rsp des failed, since:%s", tstrerror(pRsp->code));
     if (pRsp->code == TSDB_CODE_MND_ARB_TOKEN_MISMATCH) {
       terrno = TSDB_CODE_SUCCESS;
       return 0;
@@ -993,11 +1117,12 @@ bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char
     goto _OVER;
   }
 
-  if (pGroup->isSync) {
+  if (pGroup->assignedLeader.acked == false) {
     mndArbGroupDupObj(pGroup, pNewGroup);
     pNewGroup->isSync = false;
+    pNewGroup->assignedLeader.acked = true;
 
-    mInfo("vgId:%d, arb isSync is setting to false", vgId);
+    mInfo("vgId:%d, arb received assigned ack", vgId);
     updateAssigned = true;
     goto _OVER;
   }
@@ -1008,6 +1133,11 @@ _OVER:
 }
 
 static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
+  if (pRsp->contLen == 0) {
+    mDebug("arb set-assigned-rsp contLen is 0");
+    return 0;
+  }
+
   int32_t ret = -1;
 
   SMnode *pMnode = pRsp->info.node;
@@ -1022,8 +1152,8 @@ static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
 
   SVArbSetAssignedLeaderRsp setAssignedRsp = {0};
   if (tDeserializeSVArbSetAssignedLeaderRsp(pRsp->pCont, pRsp->contLen, &setAssignedRsp) != 0) {
+    mInfo("arb set-assigned-rsp des failed, since:%s", tstrerror(pRsp->code));
     terrno = TSDB_CODE_INVALID_MSG;
-    mInfo("arb set assigned failed, des failed since:%s", tstrerror(pRsp->code));
     return -1;
   }
 
@@ -1102,12 +1232,18 @@ static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
       STR_WITH_MAXSIZE_TO_VARSTR(token, pGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE);
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetVal(pColInfo, numOfRows, (const char *)token, false);
+
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataSetVal(pColInfo, numOfRows, (const char *)&pGroup->assignedLeader.acked, false);
     } else {
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetNULL(pColInfo, numOfRows);
 
       pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
       colDataSetNULL(pColInfo, numOfRows);
+
+      pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+      colDataSetNULL(pColInfo, numOfRows);
     }
 
     taosThreadMutexUnlock(&pGroup->mutex);

source/dnode/mnode/impl/src/mndDb.c

@@ -821,8 +821,7 @@ static int32_t mndCheckDbEncryptKey(SMnode *pMnode, SCreateDbReq *pReq) {
 
 #ifdef TD_ENTERPRISE
   if (pReq->encryptAlgorithm == TSDB_ENCRYPT_ALGO_NONE) goto _exit;
-  if (grantCheck(TSDB_GRANT_DB_ENCRYPTION) != 0) {
+  if ((code = grantCheck(TSDB_GRANT_DB_ENCRYPTION)) != 0) {
-    code = TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED;
     goto _exit;
   }
   if (tsEncryptionKeyStat != ENCRYPT_KEY_STAT_LOADED) {
@@ -1226,7 +1225,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
 _OVER:
   if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
     if (terrno != 0) code = terrno;
-    mError("db:%s, failed to alter since %s", alterReq.db, terrstr());
+    mError("db:%s, failed to alter since %s", alterReq.db, tstrerror(code));
   }
 
   mndReleaseDb(pMnode, pDb);

source/dnode/mnode/impl/src/mndMain.c

@@ -334,6 +334,8 @@ static int32_t minCronTime() {
   min = TMIN(min, tsStreamCheckpointInterval);
   min = TMIN(min, 6);  // checkpointRemain
   min = TMIN(min, tsStreamNodeCheckInterval);
+  min = TMIN(min, tsArbHeartBeatIntervalSec);
+  min = TMIN(min, tsArbCheckSyncIntervalSec);
 
   int64_t telemInt = TMIN(60, (tsTelemInterval - 1));
   min = TMIN(min, telemInt);
@@ -390,6 +392,18 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
   if (sec % tsUptimeInterval == 0) {
     mndIncreaseUpTime(pMnode);
   }
+
+  if (sec % (tsArbHeartBeatIntervalSec) == 0) {
+    if (mndPullupArbHeartbeat(pMnode) != 0) {
+      mError("failed to pullup arb heartbeat, since:%s", terrstr());
+    }
+  }
+
+  if (sec % (tsArbCheckSyncIntervalSec) == 0) {
+    if (mndPullupArbCheckSync(pMnode) != 0) {
+      mError("failed to pullup arb check sync, since:%s", terrstr());
+    }
+  }
 }
 void mndDoTimerCheckTask(SMnode *pMnode, int64_t sec) {
   if (sec % (tsStatusInterval * 5) == 0) {
@@ -421,18 +435,6 @@ static void *mndThreadFp(void *param) {
       continue;
     }
     mndDoTimerPullupTask(pMnode, sec);
-
-    if (sec % (tsArbHeartBeatIntervalSec) == 0) {
-      if (mndPullupArbHeartbeat(pMnode) != 0) {
-        mError("failed to pullup arb heartbeat, since:%s", terrstr());
-      }
-    }
-
-    if (sec % (tsArbCheckSyncIntervalSec) == 0) {
-      if (mndPullupArbCheckSync(pMnode) != 0) {
-        mError("failed to pullup arb check sync, since:%s", terrstr());
-      }
-    }
   }
 
   return NULL;
@@ -1076,6 +1078,11 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
   return 0;
 }
 
+int64_t mndGetRoleTimeMs(SMnode *pMnode) {
+  SSyncState state = syncGetState(pMnode->syncMgmt.sync);
+  return state.roleTimeMs;
+}
+
 void mndSetRestored(SMnode *pMnode, bool restored) {
   if (restored) {
     taosThreadRwlockWrlock(&pMnode->lock);

source/dnode/mnode/impl/src/mndStream.c

@@ -1639,11 +1639,11 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
   // info
   if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
     const char *sinkStr = "%.2fMiB";
-    sprintf(buf, sinkStr, pe->sinkDataSize);
+    snprintf(buf, tListLen(buf), sinkStr, pe->sinkDataSize);
   } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
     // offset info
     const char *offsetStr = "%" PRId64 " [%" PRId64 ", %" PRId64 "]";
-    sprintf(buf, offsetStr, pe->processedVer, pe->verRange.minVer, pe->verRange.maxVer);
+    snprintf(buf, tListLen(buf), offsetStr, pe->processedVer, pe->verRange.minVer, pe->verRange.maxVer);
   }
 
   STR_TO_VARSTR(vbuf, buf);
@@ -1847,8 +1847,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
   SMnode     *pMnode = pReq->info.node;
   SStreamObj *pStream = NULL;
 
-  if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) {
+  if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
-    terrno = TSDB_CODE_GRANT_EXPIRED;
     return -1;
   }
 

source/dnode/mnode/impl/src/mndStreamHb.c

@@ -225,7 +225,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
   SArray      *pFailedTasks = NULL;
   SArray      *pOrphanTasks = NULL;
 
-  if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) {
+  if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
     if (suspendAllStreams(pMnode, &pReq->info) < 0) {
       return -1;
     }

source/dnode/mnode/impl/src/mndTrans.c

@@ -26,7 +26,7 @@
 #define TRANS_VER1_NUMBER  1
 #define TRANS_VER2_NUMBER  2
 #define TRANS_ARRAY_SIZE   8
-#define TRANS_RESERVE_SIZE 48
+#define TRANS_RESERVE_SIZE 44
 
 static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
 static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld);
@@ -196,10 +196,21 @@ SSdbRaw *mndTransEncode(STrans *pTrans) {
   }
 
   SDB_SET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER)
+
+  int32_t arbGroupNum = taosHashGetSize(pTrans->arbGroupIds);
+  SDB_SET_INT32(pRaw, dataPos, arbGroupNum, _OVER)
+  void *pIter = NULL;
+  pIter = taosHashIterate(pTrans->arbGroupIds, NULL);
+  while (pIter) {
+    int32_t arbGroupId = *(int32_t *)pIter;
+    SDB_SET_INT32(pRaw, dataPos, arbGroupId, _OVER)
+    pIter = taosHashIterate(pTrans->arbGroupIds, pIter);
+  }
+
   SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER)
   SDB_SET_DATALEN(pRaw, dataPos, _OVER)
 
-  terrno = 0;
+      terrno = 0;
 
 _OVER:
   if (terrno != 0) {
@@ -279,6 +290,7 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) {
   int32_t  undoActionNum = 0;
   int32_t  commitActionNum = 0;
   int32_t  dataPos = 0;
+  int32_t  arbgroupIdNum = 0;
 
   if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
 
@@ -350,6 +362,16 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) {
   }
 
   SDB_GET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER);
+
+  pTrans->arbGroupIds = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
+
+  SDB_GET_INT32(pRaw, dataPos, &arbgroupIdNum, _OVER)
+  for (int32_t i = 0; i < arbgroupIdNum; ++i) {
+    int32_t arbGroupId = 0;
+    SDB_GET_INT32(pRaw, dataPos, &arbGroupId, _OVER)
+    taosHashPut(pTrans->arbGroupIds, &arbGroupId, sizeof(int32_t), NULL, 0);
+  }
+
   SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER)
 
   terrno = 0;
@@ -462,6 +484,9 @@ void mndTransDropData(STrans *pTrans) {
     mndTransDropActions(pTrans->commitActions);
     pTrans->commitActions = NULL;
   }
+  if (pTrans->arbGroupIds != NULL) {
+    taosHashCleanup(pTrans->arbGroupIds);
+  }
   if (pTrans->pRpcArray != NULL) {
     taosArrayDestroy(pTrans->pRpcArray);
     pTrans->pRpcArray = NULL;
@@ -581,6 +606,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict,
   pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
   pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
   pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
+  pTrans->arbGroupIds = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
   pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo));
   pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64();
   taosInitRWLatch(&pTrans->lockRpcArray);
@@ -733,7 +759,9 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname)
   }
 }
 
-void mndTransSetArbGroupId(STrans *pTrans, int32_t groupId) { pTrans->arbGroupId = groupId; }
+void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId) {
+  taosHashPut(pTrans->arbGroupIds, &groupId, sizeof(int32_t), NULL, 0);
+}
 
 void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; }
 
@@ -821,7 +849,16 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) {
     if (pNew->conflict == TRN_CONFLICT_ARBGROUP) {
       if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
       if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) {
-        if (pNew->arbGroupId == pTrans->arbGroupId) conflict = true;
+        void *pIter = taosHashIterate(pNew->arbGroupIds, NULL);
+        while (pIter != NULL) {
+          int32_t groupId = *(int32_t *)pIter;
+          if (taosHashGet(pTrans->arbGroupIds, &groupId, sizeof(int32_t)) != NULL) {
+            taosHashCancelIterate(pNew->arbGroupIds, pIter);
+            conflict = true;
+            break;
+          }
+          pIter = taosHashIterate(pNew->arbGroupIds, pIter);
+        }
       }
     }
 
@@ -1372,7 +1409,7 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr
   mInfo("trans:%d, execute %d actions serial, current redoAction:%d", pTrans->id, numOfActions, pTrans->actionPos);
 
   for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) {
-    STransAction *pAction = taosArrayGet(pActions, pTrans->actionPos);
+    STransAction *pAction = taosArrayGet(pActions, action);
 
     code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf);
     if (code == 0) {

source/dnode/mnode/impl/test/func/func.cpp

@@ -49,6 +49,7 @@ TEST_F(MndTestFunc, 01_Show_Func) {
 }
 
 TEST_F(MndTestFunc, 02_Create_Func) {
+#ifndef WINDOWS
   {
     SCreateFuncReq createReq = {0};
     strcpy(createReq.name, "");
@@ -159,9 +160,11 @@ TEST_F(MndTestFunc, 02_Create_Func) {
 
   test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", "");
   EXPECT_EQ(test.GetShowRows(), 1);
+#endif
 }
 
 TEST_F(MndTestFunc, 03_Retrieve_Func) {
+#ifndef WINDOWS
   {
     SRetrieveFuncReq retrieveReq = {0};
     retrieveReq.numOfFuncs = 1;
@@ -376,9 +379,11 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST);
   }
+#endif
 }
 
 TEST_F(MndTestFunc, 04_Drop_Func) {
+#ifndef WINDOWS
   {
     SDropFuncReq dropReq = {0};
     strcpy(dropReq.name, "");
@@ -441,9 +446,11 @@ TEST_F(MndTestFunc, 04_Drop_Func) {
 
   test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", "");
   EXPECT_EQ(test.GetShowRows(), 1);
+#endif
 }
 
 TEST_F(MndTestFunc, 05_Actual_code) {
+#ifndef WINDOWS
   {
     SCreateFuncReq createReq = {0};
     strcpy(createReq.name, "udf1");
@@ -507,4 +514,5 @@ TEST_F(MndTestFunc, 05_Actual_code) {
     }
     tFreeSRetrieveFuncRsp(&retrieveRsp);
   }
-}
+#endif
+}

source/dnode/vnode/src/inc/tsdb.h

@@ -355,6 +355,8 @@ typedef struct {
   int    flush_count;
 } SCacheFlushState;
 
+typedef struct SCompMonitor SCompMonitor;
+
 struct STsdb {
   char *               path;
   SVnode *             pVnode;
@@ -375,8 +377,11 @@ struct STsdb {
   TdThreadMutex        pgMutex;
   struct STFileSystem *pFS;  // new
   SRocksCache          rCache;
-  // compact monitor
+  SCompMonitor         *pCompMonitor;
-  struct SCompMonitor *pCompMonitor;
+  struct {
+    SVHashTable *ht;
+    SArray      *arr;
+  } *commitInfo;
 };
 
 struct TSDBKEY {

source/dnode/vnode/src/inc/vnd.h

@@ -49,31 +49,21 @@ int32_t vnodeEncodeConfig(const void* pObj, SJson* pJson);
 int32_t vnodeDecodeConfig(const SJson* pJson, void* pObj);
 
 // vnodeAsync.c
-typedef struct SVAsync SVAsync;
-
 typedef enum {
   EVA_PRIORITY_HIGH = 0,
   EVA_PRIORITY_NORMAL,
   EVA_PRIORITY_LOW,
 } EVAPriority;
 
-#define VNODE_ASYNC_VALID_CHANNEL_ID(channelId) ((channelId) > 0)
+int32_t vnodeAsyncOpen(int32_t numOfThreads);
-#define VNODE_ASYNC_VALID_TASK_ID(taskId)       ((taskId) > 0)
+int32_t vnodeAsyncClose();
-
+int32_t vnodeAChannelInit(int64_t async, SVAChannelID* channelID);
-int32_t vnodeAsyncInit(SVAsync** async, char* label);
+int32_t vnodeAChannelDestroy(SVAChannelID* channelID, bool waitRunning);
-int32_t vnodeAsyncDestroy(SVAsync** async);
+int32_t vnodeAsync(SVAChannelID* channelID, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*),
-int32_t vnodeAChannelInit(SVAsync* async, int64_t* channelId);
+                   void* arg, SVATaskID* taskID);
-int32_t vnodeAChannelDestroy(SVAsync* async, int64_t channelId, bool waitRunning);
+int32_t vnodeAWait(SVATaskID* taskID);
-int32_t vnodeAsync(SVAsync* async, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*), void* arg,
+int32_t vnodeACancel(SVATaskID* taskID);
-                   int64_t* taskId);
+int32_t vnodeAsyncSetWorkers(int64_t async, int32_t numWorkers);
-int32_t vnodeAsyncC(SVAsync* async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void*),
-                    void (*complete)(void*), void* arg, int64_t* taskId);
-int32_t vnodeAWait(SVAsync* async, int64_t taskId);
-int32_t vnodeACancel(SVAsync* async, int64_t taskId);
-int32_t vnodeAsyncSetWorkers(SVAsync* async, int32_t numWorkers);
-
-// vnodeModule.c
-extern SVAsync* vnodeAsyncHandle[2];
 
 // vnodeBufPool.c
 typedef struct SVBufPoolNode SVBufPoolNode;

source/dnode/vnode/src/inc/vnodeInt.h

@@ -237,9 +237,6 @@ int32_t tsdbCacheNewSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, int8_t
 int32_t tsdbCacheDropSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, bool hasPrimayKey);
 int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t col_type);
 int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey);
-int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo);
-int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync);
-int32_t tsdbS3Migrate(STsdb* tsdb, int64_t now, int32_t sync);
 int     tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg);
 int     tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp);
 int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows);
@@ -470,6 +467,16 @@ typedef struct SVMonitorObj {
   taos_counter_t* insertCounter;
 } SVMonitorObj;
 
+typedef struct {
+  int64_t async;
+  int64_t id;
+} SVAChannelID;
+
+typedef struct {
+  int64_t async;
+  int64_t id;
+} SVATaskID;
+
 struct SVnode {
   char*     path;
   SVnodeCfg config;
@@ -491,8 +498,8 @@ struct SVnode {
   SVBufPool*    onRecycle;
 
   // commit variables
-  int64_t commitChannel;
+  SVAChannelID commitChannel;
-  int64_t commitTask;
+  SVATaskID    commitTask;
 
   SMeta*        pMeta;
   SSma*         pSma;
@@ -598,6 +605,24 @@ struct SCompactInfo {
 
 void initStorageAPI(SStorageAPI* pAPI);
 
+// a simple hash table impl
+typedef struct SVHashTable SVHashTable;
+
+struct SVHashTable {
+  uint32_t (*hash)(const void*);
+  int32_t (*compare)(const void*, const void*);
+  int32_t              numEntries;
+  uint32_t             numBuckets;
+  struct SVHashEntry** buckets;
+};
+
+#define vHashNumEntries(ht) ((ht)->numEntries)
+int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*));
+int32_t vHashDestroy(SVHashTable** ht);
+int32_t vHashPut(SVHashTable* ht, void* obj);
+int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj);
+int32_t vHashDrop(SVHashTable* ht, const void* obj);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/sma/smaRollup.c

@@ -622,8 +622,8 @@ int32_t smaRetention(SSma *pSma, int64_t now) {
 
   for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
     if (pSma->pRSmaTsdb[i]) {
-      code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1);
+      // code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1);
-      if (code) goto _end;
+      // if (code) goto _end;
     }
   }
 

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -588,6 +588,13 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud
     tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud);
   }
 
+  for (uint8_t i = 0; i < pLastCol->rowKey.numOfPKs; ++i) {
+    SValue *pValue = &pLastCol->rowKey.pks[i];
+    if (IS_VAR_DATA_TYPE(pValue->type)) {
+      taosMemoryFree(pValue->pData);
+    }
+  }
+
   if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type) /* && pLastCol->colVal.value.nData > 0*/) {
     taosMemoryFree(pLastCol->colVal.value.pData);
   }
@@ -1072,6 +1079,8 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray
       SLastCol *PToFree = pLastCol;
 
       if (IS_LAST_KEY(idxKey->key) && !COL_VAL_IS_VALUE(pColVal)) {
+        taosMemoryFreeClear(PToFree);
+        rocksdb_free(values_list[i]);
         continue;
       }
 
@@ -1238,6 +1247,18 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo
   ctxArray = taosArrayInit(pBlockData->nColData, sizeof(SLastUpdateCtx));
 
   // 1. prepare last
+  STsdbRowKey tsdbRowKey = {0};
+  tsdbRowGetKey(&lRow, &tsdbRowKey);
+
+  {
+    SLastUpdateCtx updateCtx = {
+        .lflag = LFLAG_LAST,
+        .tsdbRowKey = tsdbRowKey,
+        .colVal = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, ((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP,
+                                                                       .val = lRow.pBlockData->aTSKEY[lRow.iRow]}))};
+    taosArrayPush(ctxArray, &updateCtx);
+  }
+
   TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0);
 
   for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) {
@@ -1263,9 +1284,6 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo
   }
 
   // 2. prepare last row
-  STsdbRowKey tsdbRowKey = {0};
-  tsdbRowGetKey(&lRow, &tsdbRowKey);
-
   STSDBRowIter iter = {0};
   tsdbRowIterOpen(&iter, &lRow, pTSchema);
   for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) {

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -17,13 +17,12 @@
 
 // extern dependencies
 typedef struct {
-  STsdb         *tsdb;
+  int32_t    fid;
-  TFileSetArray *fsetArr;
+  STFileSet *fset;
-  TFileOpArray   fopArray[1];
+} SFileSetCommitInfo;
-
-  // SSkmInfo skmTb[1];
-  // SSkmInfo skmRow[1];
 
+typedef struct {
+  STsdb  *tsdb;
   int32_t minutes;
   int8_t  precision;
   int32_t minRow;
@@ -32,34 +31,34 @@ typedef struct {
   int32_t sttTrigger;
   int32_t szPage;
   int64_t compactVersion;
+  int64_t cid;
+  int64_t now;
 
   struct {
-    int64_t    cid;
+    SFileSetCommitInfo *info;
-    int64_t    now;
+
-    TSKEY      nextKey;
+    int32_t expLevel;
-    int32_t    fid;
+    SDiskID did;
-    int32_t    expLevel;
+    TSKEY   minKey;
-    SDiskID    did;
+    TSKEY   maxKey;
-    TSKEY      minKey;
+    TABLEID tbid[1];
-    TSKEY      maxKey;
+    bool    hasTSData;
-    STFileSet *fset;
+
-    TABLEID    tbid[1];
+    bool      skipTsRow;
-    bool       hasTSData;
+    SHashObj *pColCmprObj;
-    bool       skipTsRow;
-    SHashObj  *pColCmprObj;
   } ctx[1];
 
   // reader
   TSttFileReaderArray sttReaderArray[1];
-
   // iter
   TTsdbIterArray dataIterArray[1];
   SIterMerger   *dataIterMerger;
   TTsdbIterArray tombIterArray[1];
   SIterMerger   *tombIterMerger;
-
   // writer
   SFSetWriter *writer;
+
+  TFileOpArray fopArray[1];
 } SCommitter2;
 
 static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
@@ -74,8 +73,8 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
       .maxRow = committer->maxRow,
       .szPage = committer->szPage,
       .cmprAlg = committer->cmprAlg,
-      .fid = committer->ctx->fid,
+      .fid = committer->ctx->info->fid,
-      .cid = committer->ctx->cid,
+      .cid = committer->cid,
       .did = committer->ctx->did,
       .level = 0,
   };
@@ -83,11 +82,11 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) {
   if (committer->sttTrigger == 1) {
     config.toSttOnly = false;
 
-    if (committer->ctx->fset) {
+    if (committer->ctx->info->fset) {
       for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ftype++) {
-        if (committer->ctx->fset->farr[ftype] != NULL) {
+        if (committer->ctx->info->fset->farr[ftype] != NULL) {
           config.files[ftype].exist = true;
-          config.files[ftype].file = committer->ctx->fset->farr[ftype]->f[0];
+          config.files[ftype].file = committer->ctx->info->fset->farr[ftype]->f[0];
         }
       }
     }
@@ -117,7 +116,6 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
 
   committer->ctx->tbid->suid = 0;
   committer->ctx->tbid->uid = 0;
-  
   for (SRowInfo *row; (row = tsdbIterMergerGetData(committer->dataIterMerger)) != NULL;) {
     if (row->uid != committer->ctx->tbid->uid) {
       committer->ctx->tbid->suid = row->suid;
@@ -132,7 +130,6 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) {
 
     int64_t ts = TSDBROW_TS(&row->row);
     if (ts > committer->ctx->maxKey) {
-      committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts);
       code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid);
       TSDB_CHECK_CODE(code, lino, _exit);
       continue;
@@ -152,7 +149,8 @@ _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
   } else {
-    tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->fid, numOfRow);
+    tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->info->fid,
+              numOfRow);
   }
   return code;
 }
@@ -168,7 +166,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
   }
 
   // do not need to write tomb data if there is no ts data
-  bool skip = (committer->ctx->fset == NULL && !committer->ctx->hasTSData);
+  bool skip = (committer->ctx->info->fset == NULL && !committer->ctx->hasTSData);
 
   committer->ctx->tbid->suid = 0;
   committer->ctx->tbid->uid = 0;
@@ -187,12 +185,8 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
     if (record->ekey < committer->ctx->minKey) {
       // do nothing
     } else if (record->skey > committer->ctx->maxKey) {
-      committer->ctx->nextKey = TMIN(record->skey, committer->ctx->nextKey);
+      // committer->ctx->nextKey = TMIN(record->skey, committer->ctx->nextKey);
     } else {
-      if (record->ekey > committer->ctx->maxKey) {
-        committer->ctx->nextKey = TMIN(committer->ctx->nextKey, committer->ctx->maxKey + 1);
-      }
-
       record->skey = TMAX(record->skey, committer->ctx->minKey);
       record->ekey = TMIN(record->ekey, committer->ctx->maxKey);
 
@@ -211,8 +205,8 @@ _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
   } else {
-    tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), committer->ctx->fid,
+    tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode),
-              numRecord);
+              committer->ctx->info->fid, numRecord);
   }
   return code;
 }
@@ -223,15 +217,15 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) {
 
   ASSERT(TARRAY2_SIZE(committer->sttReaderArray) == 0);
 
-  if (committer->ctx->fset == NULL                        //
+  if (committer->ctx->info->fset == NULL                        //
-      || committer->sttTrigger > 1                        //
+      || committer->sttTrigger > 1                              //
-      || TARRAY2_SIZE(committer->ctx->fset->lvlArr) == 0  //
+      || TARRAY2_SIZE(committer->ctx->info->fset->lvlArr) == 0  //
   ) {
     return 0;
   }
 
   SSttLvl *lvl;
-  TARRAY2_FOREACH(committer->ctx->fset->lvlArr, lvl) {
+  TARRAY2_FOREACH(committer->ctx->info->fset->lvlArr, lvl) {
     STFileObj *fobj = NULL;
     TARRAY2_FOREACH(lvl->fobjArr, fobj) {
       SSttFileReader *sttReader;
@@ -289,7 +283,7 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) {
   config.from->version = VERSION_MIN;
   config.from->key = (SRowKey){
       .ts = committer->ctx->minKey,
-      .numOfPKs = 0,  // TODO: support multiple primary keys
+      .numOfPKs = 0,
   };
 
   code = tsdbIterOpen(&config, &iter);
@@ -359,22 +353,15 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
   int32_t lino = 0;
   STsdb  *tsdb = committer->tsdb;
 
-  int32_t fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision);
-
   // check if can commit
-  tsdbFSCheckCommit(tsdb, fid);
+  tsdbFSCheckCommit(tsdb, committer->ctx->info->fid);
 
-  committer->ctx->fid = fid;
+  committer->ctx->expLevel = tsdbFidLevel(committer->ctx->info->fid, &tsdb->keepCfg, committer->now);
-  committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now);
+  tsdbFidKeyRange(committer->ctx->info->fid, committer->minutes, committer->precision, &committer->ctx->minKey,
-  tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey,
                   &committer->ctx->maxKey);
   code = tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did);
   TSDB_CHECK_CODE(code, lino, _exit);
   tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did);
-  STFileSet fset = {.fid = committer->ctx->fid};
-  committer->ctx->fset = &fset;
-  STFileSet **fsetPtr = TARRAY2_SEARCH(committer->fsetArr, &committer->ctx->fset, tsdbTFileSetCmprFn, TD_EQ);
-  committer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr;
   committer->ctx->tbid->suid = 0;
   committer->ctx->tbid->uid = 0;
 
@@ -391,15 +378,13 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) {
   code = tsdbCommitOpenWriter(committer);
   TSDB_CHECK_CODE(code, lino, _exit);
 
-  // reset nextKey
-  committer->ctx->nextKey = TSKEY_MAX;
-
 _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
   } else {
     tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode),
-              __func__, committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel);
+              __func__, committer->ctx->info->fid, committer->ctx->minKey, committer->ctx->maxKey,
+              committer->ctx->expLevel);
   }
   return code;
 }
@@ -421,7 +406,7 @@ _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
   } else {
-    tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid);
+    tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid);
   }
   return code;
 }
@@ -449,7 +434,220 @@ _exit:
   if (code) {
     TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code);
   } else {
-    tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid);
+    tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid);
+  }
+  return code;
+}
+
+static int32_t tFileSetCommitInfoCompare(const void *arg1, const void *arg2) {
+  SFileSetCommitInfo *info1 = (SFileSetCommitInfo *)arg1;
+  SFileSetCommitInfo *info2 = (SFileSetCommitInfo *)arg2;
+
+  if (info1->fid < info2->fid) {
+    return -1;
+  } else if (info1->fid > info2->fid) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+static int32_t tFileSetCommitInfoPCompare(const void *arg1, const void *arg2) {
+  return tFileSetCommitInfoCompare(*(SFileSetCommitInfo **)arg1, *(SFileSetCommitInfo **)arg2);
+}
+
+static uint32_t tFileSetCommitInfoHash(const void *arg) {
+  SFileSetCommitInfo *info = (SFileSetCommitInfo *)arg;
+  return MurmurHash3_32((const char *)&info->fid, sizeof(info->fid));
+}
+
+static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  if (pTsdb->commitInfo) {
+    for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {
+      SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i);
+      vHashDrop(pTsdb->commitInfo->ht, info);
+      tsdbTFileSetClear(&info->fset);
+      taosMemoryFree(info);
+    }
+
+    vHashDestroy(&pTsdb->commitInfo->ht);
+    taosArrayDestroy(pTsdb->commitInfo->arr);
+    pTsdb->commitInfo->arr = NULL;
+    taosMemoryFreeClear(pTsdb->commitInfo);
+  }
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+  }
+  return code;
+}
+
+static int32_t tsdbCommitInfoInit(STsdb *pTsdb) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  pTsdb->commitInfo = taosMemoryCalloc(1, sizeof(*pTsdb->commitInfo));
+  if (pTsdb->commitInfo == NULL) {
+    TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
+  }
+
+  code = vHashInit(&pTsdb->commitInfo->ht, tFileSetCommitInfoHash, tFileSetCommitInfoCompare);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  pTsdb->commitInfo->arr = taosArrayInit(0, sizeof(SFileSetCommitInfo *));
+  if (pTsdb->commitInfo->arr == NULL) {
+    TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
+  }
+
+_exit:
+  if (code) {
+    tsdbCommitInfoDestroy(pTsdb);
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+  }
+  return code;
+}
+
+static int32_t tsdbCommitInfoAdd(STsdb *tsdb, int32_t fid) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  SFileSetCommitInfo *tinfo;
+
+  if ((tinfo = taosMemoryMalloc(sizeof(*tinfo))) == NULL) {
+    TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
+  }
+  tinfo->fid = fid;
+  tinfo->fset = NULL;
+
+  code = vHashPut(tsdb->commitInfo->ht, tinfo);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) {
+    TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
+  }
+  taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare);
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code));
+  }
+  return code;
+}
+
+static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STFileSet  *fset = NULL;
+  SRBTreeIter iter;
+
+  code = tsdbCommitInfoInit(tsdb);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  // scan time-series data
+  iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
+  for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) {
+    STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn);
+
+    // scan time-series data
+    STsdbRowKey from = {
+        .key.ts = INT64_MIN,
+        .key.numOfPKs = 0,
+        .version = INT64_MIN,
+    };
+    for (;;) {
+      int64_t     minKey, maxKey;
+      STbDataIter tbDataIter = {0};
+      TSDBROW    *row;
+      int32_t     fid;
+
+      tsdbTbDataIterOpen(pTbData, &from, 0, &tbDataIter);
+      if ((row = tsdbTbDataIterGet(&tbDataIter)) == NULL) {
+        break;
+      }
+
+      fid = tsdbKeyFid(TSDBROW_TS(row), tsdb->keepCfg.days, tsdb->keepCfg.precision);
+      tsdbFidKeyRange(fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
+
+      SFileSetCommitInfo *info;
+      SFileSetCommitInfo  tinfo = {
+           .fid = fid,
+      };
+      vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info);
+      if (info == NULL) {
+        code = tsdbCommitInfoAdd(tsdb, fid);
+        TSDB_CHECK_CODE(code, lino, _exit);
+      }
+
+      from.key.ts = maxKey + 1;
+    }
+  }
+
+  taosThreadMutexLock(&tsdb->mutex);
+
+  // scan tomb data
+  if (tsdb->imem->nDel > 0) {
+    TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
+      if (tsdbTFileSetIsEmpty(fset)) {
+        continue;
+      }
+
+      SFileSetCommitInfo *info;
+      SFileSetCommitInfo  tinfo = {
+           .fid = fset->fid,
+      };
+
+      // check if the file set already on the commit list
+      vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info);
+      if (info != NULL) {
+        continue;
+      }
+
+      int64_t minKey, maxKey;
+      bool    hasDataToCommit = false;
+      tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey);
+      iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1);
+      for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) {
+        STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn);
+        for (SDelData *pDelData = pTbData->pHead; pDelData; pDelData = pDelData->pNext) {
+          if (pDelData->sKey > maxKey || pDelData->eKey < minKey) {
+            continue;
+          } else {
+            hasDataToCommit = true;
+            if ((code = tsdbCommitInfoAdd(tsdb, fset->fid))) {
+              taosThreadMutexUnlock(&tsdb->mutex);
+              TSDB_CHECK_CODE(code, lino, _exit);
+            }
+            break;
+          }
+        }
+
+        if (hasDataToCommit) {
+          break;
+        }
+      }
+    }
+  }
+
+  // begin tasks on file set
+  for (int i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
+    SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
+    tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset);
+    if (fset) {
+      tsdbTFileSetInitCopy(tsdb, fset, &info->fset);
+    }
+  }
+
+  taosThreadMutexUnlock(&tsdb->mutex);
+
+_exit:
+  if (code) {
+    tsdbCommitInfoDestroy(tsdb);
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code));
   }
   return code;
 }
@@ -458,11 +656,7 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co
   int32_t code = 0;
   int32_t lino = 0;
 
-  memset(committer, 0, sizeof(committer[0]));
-
   committer->tsdb = tsdb;
-  code = tsdbFSCreateCopySnapshot(tsdb->pFS, &committer->fsetArr);
-  TSDB_CHECK_CODE(code, lino, _exit);
   committer->minutes = tsdb->keepCfg.days;
   committer->precision = tsdb->keepCfg.precision;
   committer->minRow = info->info.config.tsdbCfg.minRows;
@@ -471,21 +665,11 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co
   committer->sttTrigger = info->info.config.sttTrigger;
   committer->szPage = info->info.config.tsdbPageSize;
   committer->compactVersion = INT64_MAX;
-  committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS);
+  committer->cid = tsdbFSAllocEid(tsdb->pFS);
-  committer->ctx->now = taosGetTimestampSec();
+  committer->now = taosGetTimestampSec();
 
-  committer->ctx->nextKey = tsdb->imem->minKey;
+  code = tsdbCommitInfoBuild(tsdb);
-  if (tsdb->imem->nDel > 0) {
+  TSDB_CHECK_CODE(code, lino, _exit);
-    SRBTreeIter iter[1] = {tRBTreeIterCreate(tsdb->imem->tbDataTree, 1)};
-
-    for (SRBTreeNode *node = tRBTreeIterNext(iter); node; node = tRBTreeIterNext(iter)) {
-      STbData *tbData = TCONTAINER_OF(node, STbData, rbtn);
-
-      for (SDelData *delData = tbData->pHead; delData; delData = delData->pNext) {
-        committer->ctx->nextKey = TMIN(committer->ctx->nextKey, delData->sKey);
-      }
-    }
-  }
 
 _exit:
   if (code) {
@@ -516,14 +700,13 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) {
   TARRAY2_DESTROY(committer->sttReaderArray, NULL);
   TARRAY2_DESTROY(committer->fopArray, NULL);
   TARRAY2_DESTROY(committer->sttReaderArray, NULL);
-  tsdbFSDestroyCopySnapshot(&committer->fsetArr);
 
 _exit:
   if (code) {
     tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, lino,
-              tstrerror(code), committer->ctx->cid);
+              tstrerror(code), committer->cid);
   } else {
-    tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->cid);
+    tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->cid);
   }
   return code;
 }
@@ -553,17 +736,18 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
     taosThreadMutexUnlock(&tsdb->mutex);
     tsdbUnrefMemTable(imem, NULL, true);
   } else {
-    SCommitter2 committer[1];
+    SCommitter2 committer = {0};
 
-    code = tsdbOpenCommitter(tsdb, info, committer);
+    code = tsdbOpenCommitter(tsdb, info, &committer);
     TSDB_CHECK_CODE(code, lino, _exit);
 
-    while (committer->ctx->nextKey != TSKEY_MAX) {
+    for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
-      code = tsdbCommitFileSet(committer);
+      committer.ctx->info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
+      code = tsdbCommitFileSet(&committer);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
 
-    code = tsdbCloseCommitter(committer, code);
+    code = tsdbCloseCommitter(&committer, code);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
@@ -580,22 +764,33 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
   int32_t code = 0;
   int32_t lino = 0;
 
-  if (tsdb->imem == NULL) goto _exit;
+  if (tsdb->imem) {
+    SMemTable *pMemTable = tsdb->imem;
+
+    taosThreadMutexLock(&tsdb->mutex);
+
+    if ((code = tsdbFSEditCommit(tsdb->pFS))) {
+      taosThreadMutexUnlock(&tsdb->mutex);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    tsdb->imem = NULL;
+
+    for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) {
+      SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i);
+      if (info->fset) {
+        tsdbFinishTaskOnFileSet(tsdb, info->fid);
+      }
+    }
 
-  SMemTable *pMemTable = tsdb->imem;
-  taosThreadMutexLock(&tsdb->mutex);
-  code = tsdbFSEditCommit(tsdb->pFS);
-  if (code) {
     taosThreadMutexUnlock(&tsdb->mutex);
-    TSDB_CHECK_CODE(code, lino, _exit);
+
+    tsdbCommitInfoDestroy(tsdb);
+    tsdbUnrefMemTable(pMemTable, NULL, true);
   }
-  tsdb->imem = NULL;
-  taosThreadMutexUnlock(&tsdb->mutex);
-  tsdbUnrefMemTable(pMemTable, NULL, true);
 
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code));
   } else {
     tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
   }
@@ -611,6 +806,16 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) {
   code = tsdbFSEditAbort(pTsdb->pFS);
   TSDB_CHECK_CODE(code, lino, _exit);
 
+  taosThreadMutexLock(&pTsdb->mutex);
+  for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {
+    SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i);
+    if (info->fset) {
+      tsdbFinishTaskOnFileSet(pTsdb, info->fid);
+    }
+  }
+  taosThreadMutexUnlock(&pTsdb->mutex);
+  tsdbCommitInfoDestroy(pTsdb);
+
 _exit:
   if (code) {
     tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
@@ -618,4 +823,4 @@ _exit:
     tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__);
   }
   return code;
-}
+}

source/dnode/vnode/src/tsdb/tsdbFS2.c

@@ -22,9 +22,6 @@
 
 extern void remove_file(const char *fname);
 
-#define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT
-#define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1)
-
 typedef struct STFileHashEntry {
   struct STFileHashEntry *next;
   char                    fname[TSDB_FILENAME_LEN];
@@ -290,10 +287,8 @@ static int32_t commit_edit(STFileSystem *fs) {
   current_fname(fs->tsdb, current, TSDB_FCURRENT);
   if (fs->etype == TSDB_FEDIT_COMMIT) {
     current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C);
-  } else if (fs->etype == TSDB_FEDIT_MERGE) {
-    current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
   } else {
-    ASSERT(0);
+    current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
   }
 
   int32_t code;
@@ -324,11 +319,8 @@ static int32_t abort_edit(STFileSystem *fs) {
 
   if (fs->etype == TSDB_FEDIT_COMMIT) {
     current_fname(fs->tsdb, fname, TSDB_FCURRENT_C);
-  } else if (fs->etype == TSDB_FEDIT_MERGE) {
-    current_fname(fs->tsdb, fname, TSDB_FCURRENT_M);
   } else {
-    tsdbError("vgId:%d %s failed since invalid etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype);
+    current_fname(fs->tsdb, fname, TSDB_FCURRENT_M);
-    ASSERT(0);
   }
 
   int32_t code;
@@ -767,9 +759,12 @@ extern int32_t tsdbStopAllCompTask(STsdb *tsdb);
 
 int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
   STFileSystem *fs = pTsdb->pFS;
-  TARRAY2(int64_t) channelArr = {0};
+  SArray       *channelArray = taosArrayInit(0, sizeof(SVAChannelID));
+  if (channelArray == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
 
-  taosThreadMutexLock(&fs->tsdb->mutex);
+  taosThreadMutexLock(&pTsdb->mutex);
 
   // disable
   pTsdb->bgTaskDisabled = true;
@@ -777,20 +772,23 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
   // collect channel
   STFileSet *fset;
   TARRAY2_FOREACH(fs->fSetArr, fset) {
-    if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) {
+    if (fset->channelOpened) {
-      TARRAY2_APPEND(&channelArr, fset->bgTaskChannel);
+      taosArrayPush(channelArray, &fset->channel);
-      fset->bgTaskChannel = 0;
+      fset->channel = (SVAChannelID){0};
+      fset->mergeScheduled = false;
+      tsdbFSSetBlockCommit(fset, false);
+      fset->channelOpened = false;
     }
-    fset->mergeScheduled = false;
-    tsdbFSSetBlockCommit(fset, false);
   }
 
-  taosThreadMutexUnlock(&fs->tsdb->mutex);
+  taosThreadMutexUnlock(&pTsdb->mutex);
 
   // destroy all channels
-  int64_t channel;
+  for (int32_t i = 0; i < taosArrayGetSize(channelArray); i++) {
-  TARRAY2_FOREACH(&channelArr, channel) { vnodeAChannelDestroy(vnodeAsyncHandle[1], channel, true); }
+    SVAChannelID *channel = taosArrayGet(channelArray, i);
-  TARRAY2_DESTROY(&channelArr, NULL);
+    vnodeAChannelDestroy(channel, true);
+  }
+  taosArrayDestroy(channelArray);
 
 #ifdef TD_ENTERPRISE
   tsdbStopAllCompTask(pTsdb);
@@ -832,15 +830,10 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e
   int32_t lino;
   char    current_t[TSDB_FILENAME_LEN];
 
-  switch (etype) {
+  if (etype == TSDB_FEDIT_COMMIT) {
-    case TSDB_FEDIT_COMMIT:
+    current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C);
-      current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C);
+  } else {
-      break;
+    current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
-    case TSDB_FEDIT_MERGE:
-      current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M);
-      break;
-    default:
-      ASSERT(0);
   }
 
   tsem_wait(&fs->canEdit);
@@ -932,8 +925,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
         arg->tsdb = fs->tsdb;
         arg->fid = fset->fid;
 
-        code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg,
+        code = vnodeAsync(&fset->channel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg, NULL);
-                           NULL);
         TSDB_CHECK_CODE(code, lino, _exit);
         fset->mergeScheduled = true;
       }
@@ -946,33 +938,6 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
     }
   }
 
-  // clear empty level and fset
-  int32_t i = 0;
-  while (i < TARRAY2_SIZE(fs->fSetArr)) {
-    STFileSet *fset = TARRAY2_GET(fs->fSetArr, i);
-
-    int32_t j = 0;
-    while (j < TARRAY2_SIZE(fset->lvlArr)) {
-      SSttLvl *lvl = TARRAY2_GET(fset->lvlArr, j);
-
-      if (TARRAY2_SIZE(lvl->fobjArr) == 0) {
-        TARRAY2_REMOVE(fset->lvlArr, j, tsdbSttLvlClear);
-      } else {
-        j++;
-      }
-    }
-
-    if (tsdbTFileSetIsEmpty(fset)) {
-      if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) {
-        vnodeAChannelDestroy(vnodeAsyncHandle[1], fset->bgTaskChannel, false);
-        fset->bgTaskChannel = 0;
-      }
-      TARRAY2_REMOVE(fs->fSetArr, i, tsdbTFileSetClear);
-    } else {
-      i++;
-    }
-  }
-
 _exit:
   if (code) {
     tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(fs->tsdb->pVnode), __func__, lino, tstrerror(code));
@@ -1211,3 +1176,47 @@ _out:
 }
 
 int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { return tsdbTFileSetRangeArrayDestroy(fsrArr); }
+
+int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset) {
+  int16_t sttTrigger = tsdb->pVnode->config.sttTrigger;
+
+  tsdbFSGetFSet(tsdb->pFS, fid, fset);
+  if (sttTrigger == 1 && (*fset)) {
+    for (;;) {
+      if ((*fset)->taskRunning) {
+        (*fset)->numWaitTask++;
+
+        taosThreadCondWait(&(*fset)->beginTask, &tsdb->mutex);
+
+        tsdbFSGetFSet(tsdb->pFS, fid, fset);
+        ASSERT(fset != NULL);
+
+        (*fset)->numWaitTask--;
+        ASSERT((*fset)->numWaitTask >= 0);
+      } else {
+        (*fset)->taskRunning = true;
+        break;
+      }
+    }
+    tsdbInfo("vgId:%d begin task on file set:%d", TD_VID(tsdb->pVnode), fid);
+  }
+
+  return 0;
+}
+
+int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid) {
+  int16_t sttTrigger = tsdb->pVnode->config.sttTrigger;
+  if (sttTrigger == 1) {
+    STFileSet *fset = NULL;
+    tsdbFSGetFSet(tsdb->pFS, fid, &fset);
+    if (fset != NULL && fset->taskRunning) {
+      fset->taskRunning = false;
+      if (fset->numWaitTask > 0) {
+        taosThreadCondSignal(&fset->beginTask);
+      }
+      tsdbInfo("vgId:%d finish task on file set:%d", TD_VID(tsdb->pVnode), fid);
+    }
+  }
+
+  return 0;
+}

source/dnode/vnode/src/tsdb/tsdbFS2.h

@@ -24,7 +24,9 @@ extern "C" {
 
 typedef enum {
   TSDB_FEDIT_COMMIT = 1,  //
-  TSDB_FEDIT_MERGE
+  TSDB_FEDIT_MERGE,
+  TSDB_FEDIT_COMPACT,
+  TSDB_FEDIT_RETENTION,
 } EFEditT;
 
 typedef enum {
@@ -59,6 +61,8 @@ int32_t tsdbFSEditAbort(STFileSystem *fs);
 // other
 int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset);
 int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid);
+int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset);
+int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid);
 // utils
 int32_t save_fs(const TFileSetArray *arr, const char *fname);
 int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype);
@@ -72,10 +76,6 @@ struct STFileSystem {
   EFEditT       etype;
   TFileSetArray fSetArr[1];
   TFileSetArray fSetArrTmp[1];
-
-  // background task queue
-  bool    stop;
-  int64_t taskid;
 };
 
 #ifdef __cplusplus

source/dnode/vnode/src/tsdb/tsdbFSet2.c

@@ -456,13 +456,14 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) {
   TARRAY2_INIT(fset[0]->lvlArr);
 
   // background task queue
-  fset[0]->bgTaskChannel = 0;
+  taosThreadCondInit(&(*fset)->beginTask, NULL);
-  fset[0]->mergeScheduled = false;
+  (*fset)->taskRunning = false;
+  (*fset)->numWaitTask = 0;
 
   // block commit variables
   taosThreadCondInit(&fset[0]->canCommit, NULL);
-  fset[0]->numWaitCommit = 0;
+  (*fset)->numWaitCommit = 0;
-  fset[0]->blockCommit = false;
+  (*fset)->blockCommit = false;
 
   return 0;
 }
@@ -598,21 +599,19 @@ int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr) {
   return 0;
 }
 
-int32_t tsdbTFileSetClear(STFileSet **fset) {
+void tsdbTFileSetClear(STFileSet **fset) {
-  if (!fset[0]) return 0;
+  if (fset && *fset) {
+    for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
+      if ((*fset)->farr[ftype] == NULL) continue;
+      tsdbTFileObjUnref((*fset)->farr[ftype]);
+    }
 
-  for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
+    TARRAY2_DESTROY((*fset)->lvlArr, tsdbSttLvlClear);
-    if (fset[0]->farr[ftype] == NULL) continue;
+
-    tsdbTFileObjUnref(fset[0]->farr[ftype]);
+    taosThreadCondDestroy(&(*fset)->beginTask);
+    taosThreadCondDestroy(&(*fset)->canCommit);
+    taosMemoryFreeClear(*fset);
   }
-
-  TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlClear);
-
-  taosThreadCondDestroy(&fset[0]->canCommit);
-  taosMemoryFree(fset[0]);
-  fset[0] = NULL;
-
-  return 0;
 }
 
 int32_t tsdbTFileSetRemove(STFileSet *fset) {
@@ -665,6 +664,12 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) {
 }
 
 int32_t tsdbTFileSetOpenChannel(STFileSet *fset) {
-  if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) return 0;
+  int32_t code;
-  return vnodeAChannelInit(vnodeAsyncHandle[1], &fset->bgTaskChannel);
+  if (!fset->channelOpened) {
+    if ((code = vnodeAChannelInit(2, &fset->channel))) {
+      return code;
+    }
+    fset->channelOpened = true;
+  }
+  return 0;
 }

source/dnode/vnode/src/tsdb/tsdbFSet2.h

@@ -42,7 +42,7 @@ typedef enum {
 int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset);
 int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
 int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
-int32_t tsdbTFileSetClear(STFileSet **fset);
+void    tsdbTFileSetClear(STFileSet **fset);
 int32_t tsdbTFileSetRemove(STFileSet *fset);
 
 int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset,
@@ -90,9 +90,15 @@ struct STFileSet {
   STFileObj   *farr[TSDB_FTYPE_MAX];  // file array
   TSttLvlArray lvlArr[1];             // level array
 
-  // background task channel
+  // background task
-  int64_t bgTaskChannel;
+  bool         channelOpened;
-  bool    mergeScheduled;
+  SVAChannelID channel;
+  bool         mergeScheduled;
+
+  // sttTrigger = 1
+  TdThreadCond beginTask;
+  bool         taskRunning;
+  int32_t      numWaitTask;
 
   // block commit variables
   TdThreadCond canCommit;

source/dnode/vnode/src/tsdb/tsdbMerge.c

@@ -593,5 +593,6 @@ _exit:
     exit(EXIT_FAILURE);
   }
   tsdbTFileSetClear(&merger->fset);
+  taosMemoryFree(arg);
   return code;
 }

source/dnode/vnode/src/tsdb/tsdbRetention.c

@@ -24,8 +24,8 @@ typedef struct {
   int64_t now;
   int64_t cid;
 
-  TFileSetArray *fsetArr;
+  STFileSet   *fset;
-  TFileOpArray   fopArr[1];
+  TFileOpArray fopArr;
 } SRTNer;
 
 static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
@@ -35,7 +35,7 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
       .of = fobj->f[0],
   };
 
-  return TARRAY2_APPEND(rtner->fopArr, op);
+  return TARRAY2_APPEND(&rtner->fopArr, op);
 }
 
 static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) {
@@ -71,7 +71,7 @@ static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFi
 
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
     if (fdFrom) taosCloseFile(&fdFrom);
     if (fdTo) taosCloseFile(&fdTo);
   }
@@ -108,7 +108,7 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile
 
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
     taosCloseFile(&fdFrom);
     taosCloseFile(&fdTo);
   }
@@ -128,7 +128,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
       .of = fobj->f[0],
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // create new
@@ -152,7 +152,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
           },
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // do copy the file
@@ -167,7 +167,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
 
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
   }
   return code;
 }
@@ -176,80 +176,51 @@ typedef struct {
   STsdb  *tsdb;
   int64_t now;
   int32_t fid;
+  bool    s3Migrate;
 } SRtnArg;
 
-static int32_t tsdbDoRetentionBegin(SRtnArg *arg, SRTNer *rtner) {
-  int32_t code = 0;
-  int32_t lino = 0;
-
-  STsdb *tsdb = arg->tsdb;
-
-  rtner->tsdb = tsdb;
-  rtner->szPage = tsdb->pVnode->config.tsdbPageSize;
-  rtner->now = arg->now;
-  rtner->cid = tsdbFSAllocEid(tsdb->pFS);
-
-  code = tsdbFSCreateCopySnapshot(tsdb->pFS, &rtner->fsetArr);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-_exit:
-  if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
-  } else {
-    tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__);
-  }
-  return code;
-}
-
 static int32_t tsdbDoRetentionEnd(SRTNer *rtner) {
   int32_t code = 0;
   int32_t lino = 0;
 
-  if (TARRAY2_SIZE(rtner->fopArr) == 0) goto _exit;
+  if (TARRAY2_SIZE(&rtner->fopArr) > 0) {
-
+    code = tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, TSDB_FEDIT_RETENTION);
-  code = tsdbFSEditBegin(rtner->tsdb->pFS, rtner->fopArr, TSDB_FEDIT_MERGE);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  taosThreadMutexLock(&rtner->tsdb->mutex);
-
-  code = tsdbFSEditCommit(rtner->tsdb->pFS);
-  if (code) {
-    taosThreadMutexUnlock(&rtner->tsdb->mutex);
     TSDB_CHECK_CODE(code, lino, _exit);
+
+    taosThreadMutexLock(&rtner->tsdb->mutex);
+
+    code = tsdbFSEditCommit(rtner->tsdb->pFS);
+    if (code) {
+      taosThreadMutexUnlock(&rtner->tsdb->mutex);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+
+    taosThreadMutexUnlock(&rtner->tsdb->mutex);
+
+    TARRAY2_DESTROY(&rtner->fopArr, NULL);
   }
 
-  taosThreadMutexUnlock(&rtner->tsdb->mutex);
-
-  TARRAY2_DESTROY(rtner->fopArr, NULL);
-
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
   } else {
     tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__);
   }
-  tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
   return code;
 }
 
-static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
+static int32_t tsdbDoRetention(SRTNer *rtner) {
   int32_t    code = 0;
   int32_t    lino = 0;
   STFileObj *fobj = NULL;
+  STFileSet *fset = rtner->fset;
   int32_t    expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now);
 
   if (expLevel < 0) {  // remove the fileset
     for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) {
       if (fobj == NULL) continue;
-      /*
-      int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs);
-      if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) {
-        code = tsdbRemoveFileObjectS3(rtner, fobj);
-        TSDB_CHECK_CODE(code, lino, _exit);
-        } else {*/
       code = tsdbDoRemoveFileObject(rtner, fobj);
       TSDB_CHECK_CODE(code, lino, _exit);
-      //}
     }
 
     SSttLvl *lvl;
@@ -275,10 +246,6 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
       if (fobj == NULL) continue;
 
       if (fobj->f->did.level == did.level) {
-        /*
-        code = tsdbCheckMigrateS3(rtner, fobj, ftype, &did);
-        TSDB_CHECK_CODE(code, lino, _exit);
-        */
         continue;
       }
 
@@ -306,91 +273,110 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) {
 
 _exit:
   if (code) {
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino);
   }
   return code;
 }
 
-static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); }
+static void tsdbRetentionCancel(void *arg) { taosMemoryFree(arg); }
 
-static int32_t tsdbDoRetentionAsync(void *arg) {
+static int32_t tsdbDoS3Migrate(SRTNer *rtner);
+
+static int32_t tsdbRetention(void *arg) {
   int32_t code = 0;
   int32_t lino = 0;
-  SRTNer  rtner[1] = {0};
 
-  code = tsdbDoRetentionBegin(arg, rtner);
+  SRtnArg   *rtnArg = (SRtnArg *)arg;
-  TSDB_CHECK_CODE(code, lino, _exit);
+  STsdb     *pTsdb = rtnArg->tsdb;
+  SVnode    *pVnode = pTsdb->pVnode;
+  STFileSet *fset = NULL;
+  SRTNer     rtner = {
+          .tsdb = pTsdb,
+          .szPage = pVnode->config.tsdbPageSize,
+          .now = rtnArg->now,
+          .cid = tsdbFSAllocEid(pTsdb->pFS),
+  };
 
-  STFileSet *fset;
+  // begin task
-  TARRAY2_FOREACH(rtner->fsetArr, fset) {
+  taosThreadMutexLock(&pTsdb->mutex);
-    if (fset->fid != ((SRtnArg *)arg)->fid) continue;
+  tsdbBeginTaskOnFileSet(pTsdb, rtnArg->fid, &fset);
+  if (fset && (code = tsdbTFileSetInitCopy(pTsdb, fset, &rtner.fset))) {
+    taosThreadMutexUnlock(&pTsdb->mutex);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  taosThreadMutexUnlock(&pTsdb->mutex);
 
-    code = tsdbDoRetentionOnFileSet(rtner, fset);
+  // do retention
+  if (rtner.fset) {
+    if (rtnArg->s3Migrate) {
+      code = tsdbDoS3Migrate(&rtner);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    } else {
+      code = tsdbDoRetention(&rtner);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+
+    code = tsdbDoRetentionEnd(&rtner);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
-  code = tsdbDoRetentionEnd(rtner);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
 _exit:
-  if (code) {
+  if (rtner.fset) {
-    if (TARRAY2_DATA(rtner->fopArr)) {
+    taosThreadMutexLock(&pTsdb->mutex);
-      TARRAY2_DESTROY(rtner->fopArr, NULL);
+    tsdbFinishTaskOnFileSet(pTsdb, rtnArg->fid);
-    }
+    taosThreadMutexUnlock(&pTsdb->mutex);
-    TFileSetArray **fsetArr = &rtner->fsetArr;
+  }
-    if (fsetArr[0]) {
-      tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
-    }
 
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
+  // clear resources
+  tsdbTFileSetClear(&rtner.fset);
+  TARRAY2_DESTROY(&rtner.fopArr, NULL);
+  taosMemoryFree(arg);
+  if (code) {
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(((SRtnArg *)arg)->tsdb->pVnode), __func__, code, lino);
   }
   return code;
 }
 
-int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
+static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, int64_t now, bool s3Migrate) {
   int32_t code = 0;
-
+  int32_t lino = 0;
-  taosThreadMutexLock(&tsdb->mutex);
-
-  if (tsdb->bgTaskDisabled) {
-    taosThreadMutexUnlock(&tsdb->mutex);
-    return 0;
-  }
 
   STFileSet *fset;
-  TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
-    code = tsdbTFileSetOpenChannel(fset);
-    if (code) {
-      taosThreadMutexUnlock(&tsdb->mutex);
-      return code;
-    }
 
-    SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
+  if (!tsdb->bgTaskDisabled) {
-    if (arg == NULL) {
+    TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
-      taosThreadMutexUnlock(&tsdb->mutex);
+      code = tsdbTFileSetOpenChannel(fset);
-      return TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
-    }
 
-    arg->tsdb = tsdb;
+      SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
-    arg->now = now;
+      if (arg == NULL) {
-    arg->fid = fset->fid;
+        TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
+      }
 
-    if (sync) {
+      arg->tsdb = tsdb;
-      code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync,
+      arg->now = now;
-                         tsdbFreeRtnArg, arg, NULL);
+      arg->fid = fset->fid;
-    } else {
+      arg->s3Migrate = s3Migrate;
-      code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync,
+
-                         tsdbFreeRtnArg, arg, NULL);
+      if ((code = vnodeAsync(&fset->channel, EVA_PRIORITY_LOW, tsdbRetention, tsdbRetentionCancel, arg, NULL))) {
-    }
+        taosMemoryFree(arg);
-    if (code) {
+        TSDB_CHECK_CODE(code, lino, _exit);
-      tsdbFreeRtnArg(arg);
+      }
-      taosThreadMutexUnlock(&tsdb->mutex);
-      return code;
     }
   }
 
-  taosThreadMutexUnlock(&tsdb->mutex);
+_exit:
+  if (code) {
+    tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(tsdb->pVnode), __func__, code, lino);
+  }
+  return code;
+}
 
+int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now) {
+  int32_t code = 0;
+  taosThreadMutexLock(&tsdb->mutex);
+  code = tsdbAsyncRetentionImpl(tsdb, now, false);
+  taosThreadMutexUnlock(&tsdb->mutex);
   return code;
 }
 
@@ -462,7 +448,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
       .of = fobj->f[0],
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // create new
@@ -486,7 +472,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int
           },
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   char fname[TSDB_FILENAME_LEN];
@@ -566,7 +552,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
       .of = fobj->f[0],
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // create new
@@ -590,7 +576,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64
           },
   };
 
-  code = TARRAY2_APPEND(rtner->fopArr, op);
+  code = TARRAY2_APPEND(&rtner->fopArr, op);
   TSDB_CHECK_CODE(code, lino, _exit);
 
   char fname[TSDB_FILENAME_LEN];
@@ -653,10 +639,11 @@ _exit:
   return code;
 }
 
-static int32_t tsdbDoS3MigrateOnFileSet(SRTNer *rtner, STFileSet *fset) {
+static int32_t tsdbDoS3Migrate(SRTNer *rtner) {
   int32_t code = 0;
   int32_t lino = 0;
 
+  STFileSet *fset = rtner->fset;
   STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA];
   if (!fobj) return code;
 
@@ -720,41 +707,7 @@ _exit:
   return code;
 }
 
-static int32_t tsdbDoS3MigrateAsync(void *arg) {
+int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) {
-  int32_t code = 0;
-  int32_t lino = 0;
-  SRTNer  rtner[1] = {0};
-
-  code = tsdbDoRetentionBegin(arg, rtner);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-  STFileSet *fset;
-  TARRAY2_FOREACH(rtner->fsetArr, fset) {
-    if (fset->fid != ((SRtnArg *)arg)->fid) continue;
-
-    code = tsdbDoS3MigrateOnFileSet(rtner, fset);
-    TSDB_CHECK_CODE(code, lino, _exit);
-  }
-
-  code = tsdbDoRetentionEnd(rtner);
-  TSDB_CHECK_CODE(code, lino, _exit);
-
-_exit:
-  if (code) {
-    if (TARRAY2_DATA(rtner->fopArr)) {
-      TARRAY2_DESTROY(rtner->fopArr, NULL);
-    }
-    TFileSetArray **fsetArr = &rtner->fsetArr;
-    if (fsetArr[0]) {
-      tsdbFSDestroyCopySnapshot(&rtner->fsetArr);
-    }
-
-    TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code);
-  }
-  return code;
-}
-
-int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
   int32_t code = 0;
 
   extern int8_t tsS3EnabledCfg;
@@ -772,45 +725,7 @@ int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
   }
 
   taosThreadMutexLock(&tsdb->mutex);
-
+  code = tsdbAsyncRetentionImpl(tsdb, now, true);
-  if (tsdb->bgTaskDisabled) {
-    taosThreadMutexUnlock(&tsdb->mutex);
-    return 0;
-  }
-
-  STFileSet *fset;
-  TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
-    code = tsdbTFileSetOpenChannel(fset);
-    if (code) {
-      taosThreadMutexUnlock(&tsdb->mutex);
-      return code;
-    }
-
-    SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
-    if (arg == NULL) {
-      taosThreadMutexUnlock(&tsdb->mutex);
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-
-    arg->tsdb = tsdb;
-    arg->now = now;
-    arg->fid = fset->fid;
-
-    if (sync) {
-      code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
-                         tsdbFreeRtnArg, arg, NULL);
-    } else {
-      code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync,
-                         tsdbFreeRtnArg, arg, NULL);
-    }
-    if (code) {
-      tsdbFreeRtnArg(arg);
-      taosThreadMutexUnlock(&tsdb->mutex);
-      return code;
-    }
-  }
-
   taosThreadMutexUnlock(&tsdb->mutex);
-
   return code;
 }

source/dnode/vnode/src/vnd/vnodeAsync.c

@@ -16,6 +16,7 @@
 #include "vnd.h"
 #include "vnodeHash.h"
 
+typedef struct SVAsync    SVAsync;
 typedef struct SVATask    SVATask;
 typedef struct SVAChannel SVAChannel;
 
@@ -54,7 +55,7 @@ struct SVATask {
   int32_t     priorScore;
   SVAChannel *channel;
   int32_t (*execute)(void *);
-  void (*complete)(void *);
+  void (*cancel)(void *);
   void        *arg;
   EVATaskState state;
 
@@ -67,6 +68,11 @@ struct SVATask {
   struct SVATask *next;
 };
 
+typedef struct {
+  void (*cancel)(void *);
+  void *arg;
+} SVATaskCancelInfo;
+
 #define VATASK_PIORITY(task_) ((task_)->priority - ((task_)->priorScore / 4))
 
 // async channel
@@ -112,6 +118,10 @@ struct SVAsync {
   SVHashTable *taskTable;
 };
 
+SVAsync *vnodeAsyncs[3];
+#define MIN_ASYNC_ID 1
+#define MAX_ASYNC_ID (sizeof(vnodeAsyncs) / sizeof(vnodeAsyncs[0]) - 1)
+
 static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
   int32_t ret;
 
@@ -160,11 +170,6 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
   }
   async->numTasks--;
 
-  // call complete callback
-  if (task->complete) {
-    task->complete(task->arg);
-  }
-
   if (task->numWait == 0) {
     taosThreadCondDestroy(&task->waitCond);
     taosMemoryFree(task);
@@ -176,7 +181,7 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
   return 0;
 }
 
-static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
+static int32_t vnodeAsyncCancelAllTasks(SVAsync *async, SArray *cancelArray) {
   while (async->queue[0].next != &async->queue[0] || async->queue[1].next != &async->queue[1] ||
          async->queue[2].next != &async->queue[2]) {
     for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) {
@@ -184,6 +189,12 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
         SVATask *task = async->queue[i].next;
         task->prev->next = task->next;
         task->next->prev = task->prev;
+        if (task->cancel) {
+          taosArrayPush(cancelArray, &(SVATaskCancelInfo){
+                                         .cancel = task->cancel,
+                                         .arg = task->arg,
+                                     });
+        }
         vnodeAsyncTaskDone(async, task);
       }
     }
@@ -194,6 +205,7 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
 static void *vnodeAsyncLoop(void *arg) {
   SVWorker *worker = (SVWorker *)arg;
   SVAsync  *async = worker->async;
+  SArray   *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo));
 
   setThreadName(async->label);
 
@@ -209,12 +221,12 @@ static void *vnodeAsyncLoop(void *arg) {
     for (;;) {
       if (async->stop || worker->workerId >= async->numWorkers) {
         if (async->stop) {  // cancel all tasks
-          vnodeAsyncCancelAllTasks(async);
+          vnodeAsyncCancelAllTasks(async, cancelArray);
         }
         worker->state = EVA_WORKER_STATE_STOP;
         async->numLaunchWorkers--;
         taosThreadMutexUnlock(&async->mutex);
-        return NULL;
+        goto _exit;
       }
 
       for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) {
@@ -259,6 +271,12 @@ static void *vnodeAsyncLoop(void *arg) {
     worker->runningTask->execute(worker->runningTask->arg);
   }
 
+_exit:
+  for (int32_t i = 0; i < taosArrayGetSize(cancelArray); i++) {
+    SVATaskCancelInfo *cancel = (SVATaskCancelInfo *)taosArrayGet(cancelArray, i);
+    cancel->cancel(cancel->arg);
+  }
+  taosArrayDestroy(cancelArray);
   return NULL;
 }
 
@@ -294,7 +312,7 @@ static int32_t vnodeAsyncChannelCompare(const void *obj1, const void *obj2) {
   return 0;
 }
 
-int32_t vnodeAsyncInit(SVAsync **async, char *label) {
+static int32_t vnodeAsyncInit(SVAsync **async, const char *label) {
   int32_t ret;
 
   if (async == NULL) {
@@ -360,7 +378,7 @@ int32_t vnodeAsyncInit(SVAsync **async, char *label) {
   return 0;
 }
 
-int32_t vnodeAsyncDestroy(SVAsync **async) {
+static int32_t vnodeAsyncDestroy(SVAsync **async) {
   if ((*async) == NULL) {
     return TSDB_CODE_INVALID_PARA;
   }
@@ -433,20 +451,39 @@ static int32_t vnodeAsyncLaunchWorker(SVAsync *async) {
   return 0;
 }
 
-#ifdef BUILD_NO_CALL
+int32_t vnodeAsyncOpen(int32_t numOfThreads) {
-int32_t vnodeAsync(SVAsync *async, EVAPriority priority, int32_t (*execute)(void *), void (*complete)(void *),
+  int32_t code = 0;
-                   void *arg, int64_t *taskId) {
+  int32_t lino = 0;
-  return vnodeAsyncC(async, 0, priority, execute, complete, arg, taskId);
-}
-#endif
 
-int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void *),
+  // vnode-commit
-                    void (*complete)(void *), void *arg, int64_t *taskId) {
+  code = vnodeAsyncInit(&vnodeAsyncs[1], "vnode-commit");
-  if (async == NULL || execute == NULL || channelId < 0) {
+  TSDB_CHECK_CODE(code, lino, _exit);
+  vnodeAsyncSetWorkers(1, numOfThreads);
+
+  // vnode-merge
+  code = vnodeAsyncInit(&vnodeAsyncs[2], "vnode-merge");
+  TSDB_CHECK_CODE(code, lino, _exit);
+  vnodeAsyncSetWorkers(2, numOfThreads);
+
+_exit:
+  return 0;
+}
+
+int32_t vnodeAsyncClose() {
+  vnodeAsyncDestroy(&vnodeAsyncs[1]);
+  vnodeAsyncDestroy(&vnodeAsyncs[2]);
+  return 0;
+}
+
+int32_t vnodeAsync(SVAChannelID *channelID, EVAPriority priority, int32_t (*execute)(void *), void (*cancel)(void *),
+                   void *arg, SVATaskID *taskID) {
+  if (channelID == NULL || channelID->async < MIN_ASYNC_ID || channelID->async > MAX_ASYNC_ID || execute == NULL ||
+      channelID->id < 0) {
     return TSDB_CODE_INVALID_PARA;
   }
 
-  int64_t id;
+  int64_t  id;
+  SVAsync *async = vnodeAsyncs[channelID->async];
 
   // create task object
   SVATask *task = (SVATask *)taosMemoryCalloc(1, sizeof(SVATask));
@@ -457,7 +494,7 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int
   task->priority = priority;
   task->priorScore = 0;
   task->execute = execute;
-  task->complete = complete;
+  task->cancel = cancel;
   task->arg = arg;
   task->state = EVA_TASK_STATE_WAITTING;
   task->numWait = 0;
@@ -466,10 +503,12 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int
   // schedule task
   taosThreadMutexLock(&async->mutex);
 
-  if (channelId == 0) {
+  if (channelID->id == 0) {
     task->channel = NULL;
   } else {
-    SVAChannel channel = {.channelId = channelId};
+    SVAChannel channel = {
+        .channelId = channelID->id,
+    };
     vHashGet(async->channelTable, &channel, (void **)&task->channel);
     if (task->channel == NULL) {
       taosThreadMutexUnlock(&async->mutex);
@@ -540,20 +579,24 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int
 
   taosThreadMutexUnlock(&async->mutex);
 
-  if (taskId != NULL) {
+  if (taskID != NULL) {
-    *taskId = id;
+    taskID->async = channelID->async;
+    taskID->id = id;
   }
 
   return 0;
 }
 
-int32_t vnodeAWait(SVAsync *async, int64_t taskId) {
+int32_t vnodeAWait(SVATaskID *taskID) {
-  if (async == NULL || taskId <= 0) {
+  if (taskID == NULL || taskID->async < MIN_ASYNC_ID || taskID->async > MAX_ASYNC_ID || taskID->id <= 0) {
     return TSDB_CODE_INVALID_PARA;
   }
 
+  SVAsync *async = vnodeAsyncs[taskID->async];
   SVATask *task = NULL;
-  SVATask  task2 = {.taskId = taskId};
+  SVATask  task2 = {
+       .taskId = taskID->id,
+  };
 
   taosThreadMutexLock(&async->mutex);
 
@@ -574,21 +617,27 @@ int32_t vnodeAWait(SVAsync *async, int64_t taskId) {
   return 0;
 }
 
-int32_t vnodeACancel(SVAsync *async, int64_t taskId) {
+int32_t vnodeACancel(SVATaskID *taskID) {
-  if (async == NULL) {
+  if (taskID == NULL || taskID->async < MIN_ASYNC_ID || taskID->async > MAX_ASYNC_ID || taskID->id <= 0) {
     return TSDB_CODE_INVALID_PARA;
   }
 
   int32_t  ret = 0;
+  SVAsync *async = vnodeAsyncs[taskID->async];
   SVATask *task = NULL;
-  SVATask  task2 = {.taskId = taskId};
+  SVATask  task2 = {
+       .taskId = taskID->id,
+  };
+  void (*cancel)(void *) = NULL;
+  void *arg = NULL;
 
   taosThreadMutexLock(&async->mutex);
 
   vHashGet(async->taskTable, &task2, (void **)&task);
   if (task) {
     if (task->state == EVA_TASK_STATE_WAITTING) {
-      // remove from queue
+      cancel = task->cancel;
+      arg = task->arg;
       task->next->prev = task->prev;
       task->prev->next = task->next;
       vnodeAsyncTaskDone(async, task);
@@ -599,14 +648,18 @@ int32_t vnodeACancel(SVAsync *async, int64_t taskId) {
 
   taosThreadMutexUnlock(&async->mutex);
 
+  if (cancel) {
+    cancel(arg);
+  }
+
   return ret;
 }
 
-int32_t vnodeAsyncSetWorkers(SVAsync *async, int32_t numWorkers) {
+int32_t vnodeAsyncSetWorkers(int64_t asyncID, int32_t numWorkers) {
-  if (async == NULL || numWorkers <= 0 || numWorkers > VNODE_ASYNC_MAX_WORKERS) {
+  if (asyncID < MIN_ASYNC_ID || asyncID > MAX_ASYNC_ID || numWorkers <= 0 || numWorkers > VNODE_ASYNC_MAX_WORKERS) {
     return TSDB_CODE_INVALID_PARA;
   }
-
+  SVAsync *async = vnodeAsyncs[asyncID];
   taosThreadMutexLock(&async->mutex);
   async->numWorkers = numWorkers;
   if (async->numIdleWorkers > 0) {
@@ -617,11 +670,13 @@ int32_t vnodeAsyncSetWorkers(SVAsync *async, int32_t numWorkers) {
   return 0;
 }
 
-int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) {
+int32_t vnodeAChannelInit(int64_t asyncID, SVAChannelID *channelID) {
-  if (async == NULL || channelId == NULL) {
+  if (channelID == NULL || asyncID < MIN_ASYNC_ID || asyncID > MAX_ASYNC_ID) {
     return TSDB_CODE_INVALID_PARA;
   }
 
+  SVAsync *async = vnodeAsyncs[asyncID];
+
   // create channel object
   SVAChannel *channel = (SVAChannel *)taosMemoryMalloc(sizeof(SVAChannel));
   if (channel == NULL) {
@@ -637,7 +692,7 @@ int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) {
   // register channel
   taosThreadMutexLock(&async->mutex);
 
-  channel->channelId = *channelId = ++async->nextChannelId;
+  channel->channelId = channelID->id = ++async->nextChannelId;
 
   // add to hash table
   int32_t ret = vHashPut(async->channelTable, channel);
@@ -657,16 +712,24 @@ int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) {
 
   taosThreadMutexUnlock(&async->mutex);
 
+  channelID->async = asyncID;
   return 0;
 }
 
-int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning) {
+int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) {
-  if (async == NULL || channelId <= 0) {
+  if (channelID == NULL || channelID->async < MIN_ASYNC_ID || channelID->async > MAX_ASYNC_ID || channelID->id <= 0) {
     return TSDB_CODE_INVALID_PARA;
   }
 
+  SVAsync    *async = vnodeAsyncs[channelID->async];
   SVAChannel *channel = NULL;
-  SVAChannel  channel2 = {.channelId = channelId};
+  SVAChannel  channel2 = {
+       .channelId = channelID->id,
+  };
+  SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo));
+  if (cancelArray == NULL) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
 
   taosThreadMutexLock(&async->mutex);
 
@@ -684,6 +747,12 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning
         SVATask *task = channel->queue[i].next;
         task->prev->next = task->next;
         task->next->prev = task->prev;
+        if (task->cancel) {
+          taosArrayPush(cancelArray, &(SVATaskCancelInfo){
+                                         .cancel = task->cancel,
+                                         .arg = task->arg,
+                                     });
+        }
         vnodeAsyncTaskDone(async, task);
       }
     }
@@ -693,6 +762,12 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning
       if (channel->scheduled) {
         channel->scheduled->prev->next = channel->scheduled->next;
         channel->scheduled->next->prev = channel->scheduled->prev;
+        if (channel->scheduled->cancel) {
+          taosArrayPush(cancelArray, &(SVATaskCancelInfo){
+                                         .cancel = channel->scheduled->cancel,
+                                         .arg = channel->scheduled->arg,
+                                     });
+        }
         vnodeAsyncTaskDone(async, channel->scheduled);
       }
       taosMemoryFree(channel);
@@ -713,12 +788,16 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning
         channel->state = EVA_CHANNEL_STATE_CLOSE;
       }
     }
-  } else {
-    taosThreadMutexUnlock(&async->mutex);
-    return TSDB_CODE_INVALID_PARA;
   }
 
   taosThreadMutexUnlock(&async->mutex);
+  for (int32_t i = 0; i < taosArrayGetSize(cancelArray); i++) {
+    SVATaskCancelInfo *cancel = (SVATaskCancelInfo *)taosArrayGet(cancelArray, i);
+    cancel->cancel(cancel->arg);
+  }
+  taosArrayDestroy(cancelArray);
 
+  channelID->async = 0;
+  channelID->id = 0;
   return 0;
 }

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -289,7 +289,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
   int64_t lastCommitted = pInfo->info.state.committed;
 
   // wait last commit task
-  vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask);
+  vnodeAWait(&pVnode->commitTask);
 
   if (syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg) != 0) goto _exit;
 
@@ -361,15 +361,14 @@ static void vnodeReturnBufPool(SVnode *pVnode) {
 
   taosThreadMutexUnlock(&pVnode->mutex);
 }
-static int32_t vnodeCommitTask(void *arg) {
+static int32_t vnodeCommit(void *arg) {
   int32_t code = 0;
 
   SCommitInfo *pInfo = (SCommitInfo *)arg;
   SVnode      *pVnode = pInfo->pVnode;
 
   // commit
-  code = vnodeCommitImpl(pInfo);
+  if ((code = vnodeCommitImpl(pInfo))) {
-  if (code) {
     vFatal("vgId:%d, failed to commit vnode since %s", TD_VID(pVnode), terrstr());
     taosMsleep(100);
     exit(EXIT_FAILURE);
@@ -379,37 +378,34 @@ static int32_t vnodeCommitTask(void *arg) {
   vnodeReturnBufPool(pVnode);
 
 _exit:
+  taosMemoryFree(arg);
   return code;
 }
 
-static void vnodeCompleteCommit(void *arg) { taosMemoryFree(arg); }
+static void vnodeCommitCancel(void *arg) { taosMemoryFree(arg); }
 
 int vnodeAsyncCommit(SVnode *pVnode) {
   int32_t code = 0;
+  int32_t lino = 0;
 
   SCommitInfo *pInfo = (SCommitInfo *)taosMemoryCalloc(1, sizeof(*pInfo));
   if (NULL == pInfo) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit);
-    goto _exit;
   }
 
   // prepare to commit
   code = vnodePrepareCommit(pVnode, pInfo);
-  if (TSDB_CODE_SUCCESS != code) {
+  TSDB_CHECK_CODE(code, lino, _exit);
-    goto _exit;
-  }
 
   // schedule the task
-  code = vnodeAsyncC(vnodeAsyncHandle[0], pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommitTask,
+  code =
-                     vnodeCompleteCommit, pInfo, &pVnode->commitTask);
+      vnodeAsync(&pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommit, vnodeCommitCancel, pInfo, &pVnode->commitTask);
+  TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
   if (code) {
-    if (NULL != pInfo) {
+    taosMemoryFree(pInfo);
-      taosMemoryFree(pInfo);
+    vError("vgId:%d %s failed at line %d since %s" PRId64, TD_VID(pVnode), __func__, lino, tstrerror(code));
-    }
-    vError("vgId:%d, %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code),
-           pVnode->state.commitID);
   } else {
     vInfo("vgId:%d, vnode async commit done, commitId:%" PRId64 " term:%" PRId64 " applied:%" PRId64, TD_VID(pVnode),
           pVnode->state.commitID, pVnode->state.applyTerm, pVnode->state.applied);
@@ -419,7 +415,7 @@ _exit:
 
 int vnodeSyncCommit(SVnode *pVnode) {
   vnodeAsyncCommit(pVnode);
-  vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask);
+  vnodeAWait(&pVnode->commitTask);
   return 0;
 }
 

source/dnode/vnode/src/vnd/vnodeHash.h

@@ -22,23 +22,6 @@
 extern "C" {
 #endif
 
-typedef struct SVHashTable SVHashTable;
-
-struct SVHashTable {
-  uint32_t (*hash)(const void*);
-  int32_t (*compare)(const void*, const void*);
-  int32_t              numEntries;
-  uint32_t             numBuckets;
-  struct SVHashEntry** buckets;
-};
-
-#define vHashNumEntries(ht) ((ht)->numEntries)
-int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*));
-int32_t vHashDestroy(SVHashTable** ht);
-int32_t vHashPut(SVHashTable* ht, void* obj);
-int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj);
-int32_t vHashDrop(SVHashTable* ht, const void* obj);
-
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/vnd/vnodeModule.c

@@ -18,8 +18,6 @@
 
 static volatile int32_t VINIT = 0;
 
-SVAsync* vnodeAsyncHandle[2];
-
 int vnodeInit(int nthreads) {
   int32_t init;
 
@@ -28,13 +26,9 @@ int vnodeInit(int nthreads) {
     return 0;
   }
 
-  // vnode-commit
+  if (vnodeAsyncOpen(nthreads) != 0) {
-  vnodeAsyncInit(&vnodeAsyncHandle[0], "vnode-commit");
+    return -1;
-  vnodeAsyncSetWorkers(vnodeAsyncHandle[0], nthreads);
+  }
-
-  // vnode-merge
-  vnodeAsyncInit(&vnodeAsyncHandle[1], "vnode-merge");
-  vnodeAsyncSetWorkers(vnodeAsyncHandle[1], nthreads);
 
   if (walInit() < 0) {
     return -1;
@@ -48,8 +42,7 @@ void vnodeCleanup() {
   if (init == 0) return;
 
   // set stop
-  vnodeAsyncDestroy(&vnodeAsyncHandle[0]);
+  vnodeAsyncClose();
-  vnodeAsyncDestroy(&vnodeAsyncHandle[1]);
 
   walCleanUp();
   smaCleanUp();

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -399,7 +399,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
   taosThreadMutexInit(&pVnode->mutex, NULL);
   taosThreadCondInit(&pVnode->poolNotEmpty, NULL);
 
-  if (vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel) != 0) {
+  if (vnodeAChannelInit(1, &pVnode->commitChannel) != 0) {
     vError("vgId:%d, failed to init commit channel", TD_VID(pVnode));
     goto _err;
   }
@@ -527,8 +527,8 @@ void vnodePostClose(SVnode *pVnode) { vnodeSyncPostClose(pVnode); }
 
 void vnodeClose(SVnode *pVnode) {
   if (pVnode) {
-    vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask);
+    vnodeAWait(&pVnode->commitTask);
-    vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true);
+    vnodeAChannelDestroy(&pVnode->commitChannel, true);
     vnodeSyncClose(pVnode);
     vnodeQueryClose(pVnode);
     tqClose(pVnode->pTq);

source/dnode/vnode/src/vnd/vnodeRetention.c

@@ -15,20 +15,15 @@
 
 #include "vnd.h"
 
-int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) {
+extern int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now);
-  int32_t code = TSDB_CODE_SUCCESS;
+extern int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now);
 
-  code = tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
+int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now) {
-
+  // async retention
-  if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now);
+  return tsdbAsyncRetention(pVnode->pTsdb, now);
-
-  return code;
 }
 
-int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now) {
+int32_t vnodeAsyncS3Migrate(SVnode *pVnode, int64_t now) {
-  int32_t code = TSDB_CODE_SUCCESS;
+  // async migration
-
+  return tsdbAsyncS3Migrate(pVnode->pTsdb, now);
-  code = tsdbS3Migrate(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1);
-
-  return code;
 }

source/dnode/vnode/src/vnd/vnodeSnapshot.c

@@ -622,13 +622,13 @@ extern int32_t tsdbEnableBgTask(STsdb *pTsdb);
 static int32_t vnodeCancelAndDisableAllBgTask(SVnode *pVnode) {
   tsdbDisableAndCancelAllBgTask(pVnode->pTsdb);
   vnodeSyncCommit(pVnode);
-  vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true);
+  vnodeAChannelDestroy(&pVnode->commitChannel, true);
   return 0;
 }
 
 static int32_t vnodeEnableBgTask(SVnode *pVnode) {
   tsdbEnableBgTask(pVnode->pTsdb);
-  vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel);
+  vnodeAChannelInit(1, &pVnode->commitChannel);
   return 0;
 }
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -50,7 +50,7 @@ static int32_t vnodeProcessArbCheckSyncReq(SVnode *pVnode, void *pReq, int32_t l
 
 static int32_t vnodePreCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
 static int32_t vnodeCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
-static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp);
+static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 
 extern int32_t vnodeProcessKillCompactReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 extern int32_t vnodeQueryCompactProgress(SVnode *pVnode, SRpcMsg *pMsg);
@@ -866,7 +866,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) {
   pMetaRsp->precision = pVnode->config.tsdbCfg.precision;
 }
 
-extern int32_t vnodeDoRetention(SVnode *pVnode, int64_t now);
+extern int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now);
 
 static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   int32_t     code = 0;
@@ -880,13 +880,13 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int3
 
   vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp);
 
-  code = vnodeDoRetention(pVnode, trimReq.timestamp);
+  code = vnodeAsyncRetention(pVnode, trimReq.timestamp);
 
 _exit:
   return code;
 }
 
-extern int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now);
+extern int32_t vnodeAsyncS3Migrate(SVnode *pVnode, int64_t now);
 
 static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   int32_t          code = 0;
@@ -900,7 +900,7 @@ static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq,
 
   vInfo("vgId:%d, s3migrate vnode request will be processed, time:%d", pVnode->config.vgId, s3migrateReq.timestamp);
 
-  code = vnodeDoS3Migrate(pVnode, s3migrateReq.timestamp);
+  code = vnodeAsyncS3Migrate(pVnode, s3migrateReq.timestamp);
 
 _exit:
   return code;
@@ -931,13 +931,13 @@ end:
   return ret;
 }
 
-static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp) {
+static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   int32_t                 code = -1;
   SMetaReader             mr = {0};
   SVDropTtlTableReq       ttlReq = {0};
   SVFetchTtlExpiredTbsRsp rsp = {0};
   SEncoder                encoder = {0};
-  SArray*                 pNames = NULL;
+  SArray                 *pNames = NULL;
   pRsp->msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP;
   pRsp->code = TSDB_CODE_SUCCESS;
   pRsp->pCont = NULL;
@@ -950,8 +950,8 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void*
 
   ASSERT(ttlReq.nUids == taosArrayGetSize(ttlReq.pTbUids));
 
-  tb_uid_t       suid;
+  tb_uid_t    suid;
-  char           ctbName[TSDB_TABLE_NAME_LEN];
+  char        ctbName[TSDB_TABLE_NAME_LEN];
   SVDropTbReq expiredTb = {.igNotExists = true};
   metaReaderDoInit(&mr, pVnode->pMeta, 0);
   rsp.vgId = TD_VID(pVnode);
@@ -965,12 +965,12 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void*
   }
   char buf[TSDB_TABLE_NAME_LEN];
   for (int32_t i = 0; i < ttlReq.nUids; ++i) {
-    tb_uid_t* uid = taosArrayGet(ttlReq.pTbUids, i);
+    tb_uid_t *uid = taosArrayGet(ttlReq.pTbUids, i);
     expiredTb.suid = *uid;
     terrno = metaReaderGetTableEntryByUid(&mr, *uid);
     if (terrno < 0) goto _end;
     strncpy(buf, mr.me.name, TSDB_TABLE_NAME_LEN);
-    void* p = taosArrayPush(pNames, buf);
+    void *p = taosArrayPush(pNames, buf);
     expiredTb.name = p;
     if (mr.me.type == TSDB_CHILD_TABLE) {
       expiredTb.suid = mr.me.ctbEntry.suid;
@@ -1144,7 +1144,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
       if (i < tbNames->size - 1) {
         taosStringBuilderAppendChar(&sb, ',');
       }
-      //taosMemoryFreeClear(*key);
+      // taosMemoryFreeClear(*key);
     }
 
     size_t len = 0;
@@ -2241,14 +2241,14 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq,
   return TSDB_CODE_SUCCESS;
 }
 
-extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
+extern int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
 
 static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
   if (!pVnode->restored) {
     vInfo("vgId:%d, ignore compact req during restoring. ver:%" PRId64, TD_VID(pVnode), ver);
     return 0;
   }
-  return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp);
+  return vnodeAsyncCompact(pVnode, ver, pReq, len, pRsp);
 }
 
 static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
@@ -2355,8 +2355,6 @@ _OVER:
 }
 
 #ifndef TD_ENTERPRISE
-int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
+int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; }
-  return 0;
-}
 int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync) { return 0; }
 #endif

source/libs/executor/src/timesliceoperator.c

@@ -511,7 +511,6 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB
   }
 
   pInfo->isPrevRowSet = false;
-
   return TSDB_CODE_SUCCESS;
 }
 
@@ -825,8 +824,12 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato
   SSDataBlock* pResBlock = pSliceInfo->pRes;
   SInterval*   pInterval = &pSliceInfo->interval;
 
-  while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT &&
+  if (pSliceInfo->fillType == TSDB_FILL_NEXT || pSliceInfo->fillType == TSDB_FILL_LINEAR ||
-         pSliceInfo->fillType != TSDB_FILL_LINEAR) {
+      pSliceInfo->pPrevGroupKey == NULL) {
+    return;
+  }
+
+  while (pSliceInfo->current <= pSliceInfo->win.ekey) {
     genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, index, false);
     pSliceInfo->current =
         taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
@@ -1068,6 +1071,8 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode
 
   blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
 
+//  int32_t code = initKeeperInfo(pSliceInfo, pBlock, &pOperator->exprSupp);
+
   code = appendDownstream(pOperator, &downstream, 1);
   return pOperator;
 

source/libs/monitorfw/inc/taos_linked_list_i.h

@@ -47,7 +47,7 @@ int taos_linked_list_push(taos_linked_list_t *self, void *item);
 /**
  * @brief API PRIVATE Pop the first item off of the list
  */
-void *taos_linked_list_pop(taos_linked_list_t *self);
+//void *taos_linked_list_pop(taos_linked_list_t *self);
 
 /**
  * @brief API PRIVATE Returns the item at the head of the list or NULL if not present

source/libs/monitorfw/src/taos_collector.c

@@ -33,6 +33,8 @@ taos_map_t *taos_collector_default_collect(taos_collector_t *self) { return self
 taos_collector_t *taos_collector_new(const char *name) {
   int r = 0;
   taos_collector_t *self = (taos_collector_t *)taos_malloc(sizeof(taos_collector_t));
+  if (self == NULL) return NULL;
+  memset(self, 0, sizeof(taos_collector_t));
   self->name = taos_strdup(name);
   self->metrics = taos_map_new();
   if (self->metrics == NULL) {
@@ -66,9 +68,11 @@ int taos_collector_destroy(taos_collector_t *self) {
   if (r) ret = r;
   self->metrics = NULL;
 
-  r = taos_string_builder_destroy(self->string_builder);
+  if(self->string_builder != NULL){
-  if (r) ret = r;
+    r = taos_string_builder_destroy(self->string_builder);
-  self->string_builder = NULL;
+    if (r) ret = r;
+    self->string_builder = NULL;
+  }
 
   taos_free((char *)self->name);
   self->name = NULL;

source/libs/monitorfw/src/taos_collector_registry.c

@@ -59,6 +59,7 @@ taos_collector_registry_t *taos_collector_registry_new(const char *name) {
   r = pthread_rwlock_init(self->lock, NULL);
   if (r) {
     TAOS_LOG("failed to initialize rwlock");
+    taos_free(self);
     return NULL;
   }
   return self;
@@ -301,6 +302,9 @@ const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self,
 
 _OVER:
   tjsonDelete(pJson);
+  if(tmp_builder != NULL){
+    taos_string_builder_destroy(tmp_builder);
+  }
 
   return NULL;
 }

source/libs/monitorfw/src/taos_linked_list.c

@@ -117,6 +117,7 @@ int taos_linked_list_push(taos_linked_list_t *self, void *item) {
   return 0;
 }
 
+/*
 void *taos_linked_list_pop(taos_linked_list_t *self) {
   TAOS_ASSERT(self != NULL);
   if (self == NULL) return NULL;
@@ -141,6 +142,7 @@ void *taos_linked_list_pop(taos_linked_list_t *self) {
   }
   return item;
 }
+*/
 
 int taos_linked_list_remove(taos_linked_list_t *self, void *item) {
   TAOS_ASSERT(self != NULL);

source/libs/monitorfw/src/taos_map.c

@@ -90,7 +90,7 @@ taos_map_t *taos_map_new() {
     return NULL;
   }
 
-  self->addrs = taos_malloc(sizeof(taos_linked_list_t) * self->max_size);
+  self->addrs = taos_malloc(sizeof(taos_linked_list_t*) * self->max_size);
   self->free_value_fn = destroy_map_node_value_no_op;
 
   for (int i = 0; i < self->max_size; i++) {
@@ -273,7 +273,7 @@ int taos_map_ensure_space(taos_map_t *self) {
   if (r) return r;
 
   // Create a new array of addrs
-  taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t) * new_max);
+  taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t*) * new_max);
 
   // Initialize the new array
   for (int i = 0; i < new_max; i++) {

source/libs/monitorfw/src/taos_metric.c

@@ -33,6 +33,8 @@ taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name,
                                size_t label_key_count, const char **label_keys) {
   int r = 0;
   taos_metric_t *self = (taos_metric_t *)taos_malloc(sizeof(taos_metric_t));
+  if (self == NULL) return NULL;
+  memset(self, 0, sizeof(taos_metric_t));
   self->type = metric_type;
   int len = strlen(name) + 1;
   self->name = taos_malloc(len);
@@ -79,6 +81,7 @@ taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name,
   r = pthread_rwlock_init(self->rwlock, NULL);
   if (r) {
     TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR);
+    taos_free(self);
     return NULL;
   }
   return self;
@@ -91,9 +94,11 @@ int taos_metric_destroy(taos_metric_t *self) {
   int r = 0;
   int ret = 0;
 
-  r = taos_map_destroy(self->samples);
+  if(self->samples != NULL){
-  self->samples = NULL;
+    r = taos_map_destroy(self->samples);
-  if (r) ret = r;
+    self->samples = NULL;
+    if (r) ret = r;
+  }
 
   r = taos_metric_formatter_destroy(self->formatter);
   self->formatter = NULL;
@@ -152,8 +157,7 @@ taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const
   return NULL;
 
   // Get l_value
-  r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys,
+  r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys, label_values);
-                                         label_values);
   if (r) {
     TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK();
   }
@@ -170,10 +174,12 @@ taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const
     sample = taos_metric_sample_new(self->type, l_value, 0.0);
     r = taos_map_set(self->samples, l_value, sample);
     if (r) {
+      taos_free((void *)l_value);
       TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK();
     }
   }
-  pthread_rwlock_unlock(self->rwlock);
+  r = pthread_rwlock_unlock(self->rwlock);
+  if (r) TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); 
   taos_free((void *)l_value);
   return sample;
 }

source/libs/nodes/src/nodesCloneFuncs.c

@@ -845,6 +845,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) {
   COPY_SCALAR_FIELD(precision);
   COPY_SCALAR_FIELD(isEmptyResult);
   COPY_SCALAR_FIELD(timeLineResMode);
+  COPY_SCALAR_FIELD(timeLineFromOrderBy);
   COPY_SCALAR_FIELD(timeLineCurMode);
   COPY_SCALAR_FIELD(hasAggFuncs);
   COPY_SCALAR_FIELD(hasRepeatScanFuncs);
@@ -862,6 +863,8 @@ static int32_t setOperatorCopy(const SSetOperator* pSrc, SSetOperator* pDst) {
   COPY_CHAR_ARRAY_FIELD(stmtName);
   COPY_SCALAR_FIELD(precision);
   COPY_SCALAR_FIELD(timeLineResMode);
+  COPY_SCALAR_FIELD(timeLineFromOrderBy);
+  
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/parser/src/parTranslater.c

@@ -959,7 +959,8 @@ static bool isTimeLineQuery(SNode* pStmt) {
     return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineCurMode) ||
            (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineCurMode);
   } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) {
-    return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode;
+    return (TIME_LINE_MULTI == ((SSetOperator*)pStmt)->timeLineResMode) ||
+           (TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode);
   } else {
     return false;
   }
@@ -1000,18 +1001,64 @@ static bool isBlockTimeLineAlignedQuery(SNode* pStmt) {
   return false;
 }
 
+SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesNum) {
+  SNodeList* pPartitionList = NULL;
+  SNode* pNode = NULL;
+  if (pOrderList->length <= nodesNum) {
+    return NULL;
+  }
+
+  pNode = nodesListGetNode(pOrderList, nodesNum);
+  SOrderByExprNode* pOrder = (SOrderByExprNode*)pNode;
+  if (!isPrimaryKeyImpl(pOrder->pExpr)) {
+    return NULL;
+  }
+
+  for (int32_t i = 0; i < nodesNum; ++i) {
+    pNode = nodesListGetNode(pOrderList, i);
+    pOrder = (SOrderByExprNode*)pNode;
+    nodesListMakeStrictAppend(&pPartitionList, nodesCloneNode(pOrder->pExpr));
+  }
+
+  return pPartitionList;
+}
+
+
 static bool isTimeLineAlignedQuery(SNode* pStmt) {
   SSelectStmt* pSelect = (SSelectStmt*)pStmt;
   if (!isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
     return false;
   }
-  if (QUERY_NODE_SELECT_STMT != nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
+  if (QUERY_NODE_SELECT_STMT == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
-    return false;
+    SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
+    if (pSelect->pPartitionByList) {
+      if (!pSub->timeLineFromOrderBy && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) {
+        return true;
+      }
+      if (pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
+        SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
+        bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
+        nodesDestroyList(pPartitionList);
+
+        if (match) {
+          return true;
+        }
+      }
+    }
   }
-  SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
+  if (QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
-  if (pSelect->pPartitionByList && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) {
+    SSetOperator* pSub = (SSetOperator*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
-    return true;
+    if (pSelect->pPartitionByList && pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) {
+      SNodeList* pPartitionList = buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length);
+      bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList);
+      nodesDestroyList(pPartitionList);
+
+      if (match) {
+        return true;
+      }
+    }
   }
+
   return false;
 }
 
@@ -6025,9 +6072,19 @@ static void resetResultTimeline(SSelectStmt* pSelect) {
   if ((QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder)) ||
       (QUERY_NODE_TEMP_TABLE != nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder))) {
     pSelect->timeLineResMode = TIME_LINE_GLOBAL;
-  } else {
+    return;
-    pSelect->timeLineResMode = TIME_LINE_NONE;
+  } else if (pSelect->pOrderByList->length > 1) {
+    for (int32_t i = 1; i < pSelect->pOrderByList->length; ++i) {
+      pOrder = ((SOrderByExprNode*)nodesListGetNode(pSelect->pOrderByList, i))->pExpr;
+      if (isPrimaryKeyImpl(pOrder)) {
+        pSelect->timeLineResMode = TIME_LINE_MULTI;
+        pSelect->timeLineFromOrderBy = true;
+        return;
+      }
+    }
   }
+  
+  pSelect->timeLineResMode = TIME_LINE_NONE;
 }
 
 static int32_t replaceOrderByAliasForSelect(STranslateContext* pCxt, SSelectStmt* pSelect) {
@@ -6180,16 +6237,13 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS
     }
     snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName);
     SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft);
-    if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight)) {
+    bool isLeftPrimTs = isPrimaryKeyImpl(pLeft);
-      SColumnNode* pLCol = (SColumnNode*)pLeft;
+    bool isRightPrimTs = isPrimaryKeyImpl(pRight);
-      SColumnNode* pRCol = (SColumnNode*)pRight;
+
+    if (isLeftPrimTs && isRightPrimTs) {
       SColumnNode* pFCol = (SColumnNode*)pProj;
-      if (pLCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID && pRCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+      pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
-        pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
+      pFCol->isPrimTs = true;
-        if (pLCol->isPrimTs && pRCol->isPrimTs) {
-          pFCol->isPrimTs = true;
-        }
-      }
     }
     if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) {
       return TSDB_CODE_OUT_OF_MEMORY;
@@ -6225,9 +6279,19 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS
     SNode* pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, 0))->pExpr;
     if (isPrimaryKeyImpl(pOrder)) {
       pSetOperator->timeLineResMode = TIME_LINE_GLOBAL;
-    } else {
+      return code;
-      pSetOperator->timeLineResMode = TIME_LINE_NONE;
+    } else if (pSetOperator->pOrderByList->length > 1) {
+      for (int32_t i = 1; i < pSetOperator->pOrderByList->length; ++i) {
+        pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, i))->pExpr;
+        if (isPrimaryKeyImpl(pOrder)) {
+          pSetOperator->timeLineResMode = TIME_LINE_MULTI;
+          pSetOperator->timeLineFromOrderBy = true;
+          return code;
+        }
+      
}
     }
+    
+    pSetOperator->timeLineResMode = TIME_LINE_NONE;
   }
   return code;
 }

source/libs/planner/src/planLogicCreater.c

@@ -1587,6 +1587,7 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator
     TSWAP(pProject->node.pLimit, pSetOperator->pLimit);
   }
   pProject->ignoreGroupId = true;
+  pProject->isSetOpProj = true;
 
   int32_t code = TSDB_CODE_SUCCESS;
 

source/libs/planner/src/planOptimizer.c

@@ -3284,18 +3284,34 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan*
   SNodeList*  pNewChildTargets = nodesMakeList();
 
   if (NULL == pProjectNode->node.pParent) {
-    SNode* pProjection = NULL;
+    SNode *pProjection = NULL, *pChildTarget = NULL;
-    FOREACH(pProjection, pProjectNode->pProjections) {
+    bool needOrderMatch = QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChild) && ((SProjectLogicNode*)pChild)->isSetOpProj;
-      SNode* pChildTarget = NULL;
+    bool orderMatch = true;
-      FOREACH(pChildTarget, pChild->pTargets) {
+    if (needOrderMatch) {
-        if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) {
+      // For sql: select ... from (select ... union all select ...);
-          nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget));
+      // When eliminating the outer proj (the outer select), we have to make sure that the outer proj projections and
+      // union all project targets have same columns in the same order. See detail in TD-30188
+      FORBOTH(pProjection, pProjectNode->pProjections, pChildTarget, pChild->pTargets) {
+        if (!pProjection) break;
+        if (0 != strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) {
+          orderMatch = false;
           break;
         }
+        nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget));
+      }
+    } else {
+      FOREACH(pProjection, pProjectNode->pProjections) {
+        FOREACH(pChildTarget, pChild->pTargets) {
+          if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) {
+            nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget));
+            break;
+          }
+        }
       }
     }
 
-    if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets)) {
+    if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets) &&
+        (!needOrderMatch || (needOrderMatch && orderMatch))) {
       nodesDestroyList(pChild->pTargets);
       pChild->pTargets = pNewChildTargets;
     } else {

source/libs/qworker/src/qwMsg.c

@@ -370,12 +370,12 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran
       if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) {
         QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask);
         tFreeSSubQueryMsg(&msg);
-        QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED);
+        QW_ERR_RET(TSDB_CODE_GRANT_VIEW_EXPIRED);
       }
       if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) {
         QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask);
         tFreeSSubQueryMsg(&msg);
-        QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED);
+        QW_ERR_RET(TSDB_CODE_GRANT_AUDIT_EXPIRED);
       }
     }
   }

source/libs/scheduler/test/schedulerTests.cpp

@@ -798,6 +798,7 @@ TEST(queryTest, normalCase) {
 
   schedulerFreeJob(&job, 0);
 
+  taosThreadJoin(thread1, NULL);
 }
 
 TEST(queryTest, readyFirstCase) {
@@ -907,6 +908,8 @@ TEST(queryTest, readyFirstCase) {
   schedulerDestroy();
 
   schedulerFreeJob(&job, 0);
+
+  taosThreadJoin(thread1, NULL);
 }
 
 TEST(queryTest, flowCtrlCase) {
@@ -1001,6 +1004,8 @@ TEST(queryTest, flowCtrlCase) {
   schedulerDestroy();
 
   schedulerFreeJob(&job, 0);
+
+  taosThreadJoin(thread1, NULL);  
 }
 
 TEST(insertTest, normalCase) {
@@ -1061,6 +1066,8 @@ TEST(insertTest, normalCase) {
   schedulerFreeJob(&insertJobRefId, 0);
 
   schedulerDestroy();
+
+  taosThreadJoin(thread1, NULL);  
 }
 
 TEST(multiThread, forceFree) {

source/libs/stream/src/streamBackendRocksdb.c

@@ -1155,10 +1155,13 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) {
     taskDbAddRef(pTaskDb);
 
     int64_t chkpId = pTaskDb->chkpId;
+    taskDbRefChkp(pTaskDb, chkpId);
     code = taskDbDoCheckpoint(pTaskDb, chkpId);
-    taskDbRemoveRef(pTaskDb);
+    if (code != 0) {
+      taskDbUnRefChkp(pTaskDb, chkpId);
+    }
 
-    taskDbRefChkp(pTaskDb, pTaskDb->chkpId);
+    taskDbRemoveRef(pTaskDb);
 
     SStreamTask*    pTask = pTaskDb->pTask;
     SStreamTaskSnap snap = {.streamId = pTask->id.streamId,
@@ -1182,14 +1185,15 @@ int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) {
   for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) {
     SStreamTaskSnap* pSnap = taosArrayGet(pSnapInfo, i);
     sprintf(buf, "0x%" PRIx64 "-0x%x", pSnap->streamId, (int32_t)pSnap->taskId);
-    STaskDbWrapper* pTaskDb = taosHashGet(pMeta->pTaskDbUnique, buf, strlen(buf));
+    STaskDbWrapper** pTaskDb = taosHashGet(pMeta->pTaskDbUnique, buf, strlen(buf));
-    if (pTaskDb == NULL) {
+    if (pTaskDb == NULL || *pTaskDb == NULL) {
       stWarn("stream backend:%p failed to find task db, streamId:% " PRId64 "", pMeta, pSnap->streamId);
+      memset(buf, 0, sizeof(buf));
       continue;
     }
     memset(buf, 0, sizeof(buf));
 
-    taskDbUnRefChkp(pTaskDb, pSnap->chkpId);
+    taskDbUnRefChkp(*pTaskDb, pSnap->chkpId);
   }
   taosThreadMutexUnlock(&pMeta->backendMutex);
   return 0;
@@ -1989,7 +1993,8 @@ void taskDbRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) {
 
 void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) {
   taosThreadRwlockWrlock(&pTaskDb->chkpDirLock);
-  for (int i = 0; i < taosArrayGetSize(pTaskDb->chkpInUse); i++) {
+  int32_t size = taosArrayGetSize(pTaskDb->chkpInUse);
+  for (int i = 0; i < size; i++) {
     int64_t* p = taosArrayGet(pTaskDb->chkpInUse, i);
     if (*p == chkp) {
       taosArrayRemove(pTaskDb->chkpInUse, i);

source/libs/stream/src/streamExec.c

@@ -184,16 +184,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i
   return code;
 }
 
-static int32_t handleResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) {
+static int32_t handleSanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) {
   int32_t code = TSDB_CODE_SUCCESS;
   if (taosArrayGetSize(pRes) > 0) {
     SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(NULL, pTask, size, pRes);
     code = doOutputResultBlockImpl(pTask, pStreamBlocks);
-    if (code != TSDB_CODE_SUCCESS) {
+    if (code != TSDB_CODE_SUCCESS) {  // should not have error code
-      stDebug("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code));
+      stError("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code));
     }
   } else {
-    taosArrayDestroy(pRes);
+    taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes);
   }
   return code;
 }
@@ -268,6 +268,17 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
       return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0);
     }
 
+    // output queue is full, idle for 5 sec.
+    if (streamQueueIsFull(pTask->outputq.queue)) {
+      stWarn("s-task:%s outputQ is full, idle for 1sec and retry", id);
+      return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, STREAM_SCAN_HISTORY_TIMESLICE);
+    }
+
+    if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
+      stWarn("s-task:%s downstream task inputQ blocked, idle for 5sec and retry", id);
+      return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, FILL_HISTORY_TASK_EXEC_INTERVAL);
+    }
+
     SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
     if (pRes == NULL) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
@@ -284,19 +295,13 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) {
     }
 
     // dispatch the generated results
-    /*int32_t code = */handleResultBlocks(pTask, pRes, size);
+    /*int32_t code = */handleSanhistoryResultBlocks(pTask, pRes, size);
-
-    int64_t el = taosGetTimestampMs() - st;
-
-    // downstream task input queue is full, try in 5sec
-    if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED && (pTask->info.fillHistory == 1)) {
-      return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, FILL_HISTORY_TASK_EXEC_INTERVAL);
-    }
 
     if (finished) {
       return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0);
     }
 
+    int64_t el = taosGetTimestampMs() - st;
     if (el >= STREAM_SCAN_HISTORY_TIMESLICE && (pTask->info.fillHistory == 1)) {
       stDebug("s-task:%s fill-history:%d time slice exhausted, elapsed time:%.2fs, retry in 100ms", id,
               pTask->info.fillHistory, el / 1000.0);
@@ -558,7 +563,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) {
 
     if (streamQueueIsFull(pTask->outputq.queue)) {
       stWarn("s-task:%s outputQ is full, idle for 500ms and retry", id);
-      streamTaskSetIdleInfo(pTask, 500);
+      streamTaskSetIdleInfo(pTask, 1000);
       return 0;
     }
 

source/libs/stream/src/streamMeta.c

@@ -562,13 +562,16 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
   tEncoderClear(&encoder);
 
   int64_t id[2] = {pTask->id.streamId, pTask->id.taskId};
-  if (tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn) < 0) {
+
-    stError("s-task:%s save to disk failed, code:%s", pTask->id.idStr, tstrerror(terrno));
+  code = tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn);
-    return -1;
+  if (code != TSDB_CODE_SUCCESS) {
+    stError("s-task:%s task meta save to disk failed, code:%s", pTask->id.idStr, tstrerror(terrno));
+  } else {
+    stDebug("s-task:%s task meta save to disk", pTask->id.idStr);
   }
 
   taosMemoryFree(buf);
-  return 0;
+  return code;
 }
 
 int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pTaskId) {

source/libs/stream/src/streamQueue.c

@@ -381,7 +381,7 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc
     }
   }
 
-  return TSDB_CODE_SUCCESS;
+  return code;
 }
 
 int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate,

source/libs/stream/src/streamSnapshot.c

@@ -144,7 +144,7 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) {
 
 int32_t streamCreateTaskDbSnapInfo(void* arg, char* path, SArray* pSnap) { return taskDbBuildSnap(arg, pSnap); }
 
-int32_t streamDestroyTasdDbSnapInfo(void* arg, SArray* snap) { return taskDbDestroySnap(arg, snap); }
+int32_t streamDestroyTaskDbSnapInfo(void* arg, SArray* snap) { return taskDbDestroySnap(arg, snap); }
 
 void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) {
   if (qDebugFlag & DEBUG_DEBUG) {
@@ -333,7 +333,7 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) {
     }
     taosArrayDestroy(handle->pDbSnapSet);
   }
-  streamDestroyTasdDbSnapInfo(handle->pMeta, handle->pSnapInfoSet);
+  streamDestroyTaskDbSnapInfo(handle->pMeta, handle->pSnapInfoSet);
   if (handle->pSnapInfoSet) {
     for (int32_t i = 0; i < taosArrayGetSize(handle->pSnapInfoSet); i++) {
       SStreamTaskSnap* pSnap = taosArrayGet(handle->pSnapInfoSet, i);

source/libs/stream/src/tstreamFileState.c

@@ -759,7 +759,7 @@ void* getRowStateBuff(SStreamFileState* pFileState) { return pFileState->rowStat
 void* getStateFileStore(SStreamFileState* pFileState) { return pFileState->pFileStore; }
 
 bool isDeteled(SStreamFileState* pFileState, TSKEY ts) {
-  return pFileState->deleteMark > 0 && ts < (pFileState->maxTs - pFileState->deleteMark);
+  return pFileState->deleteMark != INT64_MAX && pFileState->maxTs > 0 && ts < (pFileState->maxTs - pFileState->deleteMark);
 }
 
 bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap) { return ts <= (pFileState->flushMark + gap); }

source/libs/tfs/src/tfs.c

@@ -177,11 +177,10 @@ int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) {
       continue;
     }
 
-    return 0;
+    return (terrno = 0);
   }
 
-  terrno = TSDB_CODE_FS_NO_VALID_DISK;
+  return (terrno = TSDB_CODE_FS_NO_VALID_DISK);
-  return -1;
 }
 
 const char *tfsGetPrimaryPath(STfs *pTfs) { return TFS_PRIMARY_DISK(pTfs)->path; }

source/util/src/tcompression.c

@@ -2741,7 +2741,7 @@ int32_t tsDecompressTimestamp2(void *pIn, int32_t nIn, int32_t nEle, void *pOut,
 int32_t tsCompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
                          int32_t nBuf) {
   DEFINE_VAR(cmprAlg)
-  if (lvl != 0 && lossyFloat) {
+  if (l2 == L2_TSZ && lvl != 0 && lossyFloat) {
     return tsCompressFloatLossyImp(pIn, nEle, pOut);
   }
   FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_FLOAT, 1);
@@ -2760,7 +2760,7 @@ int32_t tsDecompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int
 int32_t tsCompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
                           int32_t nBuf) {
   DEFINE_VAR(cmprAlg)
-  if (lvl != 0 && lossyDouble) {
+  if (l2 == L2_TSZ && lvl != 0 && lossyDouble) {
     // lossy mode
     return tsCompressDoubleLossyImp(pIn, nEle, pOut);
   }

source/util/src/terror.c

@@ -259,7 +259,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING,           "Database in creating
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "Encryption is not allowed to be changed after database is created")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY,    "Inconsistent encryption key")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ENCRYPT_KEY,      "The cluster has not been set properly for database encryption")
-TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED, "The database encryption function grant expired")
 
 // mnode-node
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST,      "Mnode already exists")
@@ -499,9 +498,20 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_LACK_OF_BASIC,          "Lack of basic functio
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJ_NOT_EXIST,          "Grant object not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_LAST_ACTIVE_NOT_FOUND,  "The historial active code does not match")
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MACHINES_MISMATCH,      "Cluster machines mismatch with active code")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE,   "Expire time of optional grant item is too large")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE,   "Expiration time of optional grant item is too large")
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUPLICATED_ACTIVE,      "The active code can't be activated repeatedly")
-TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED,           "Number of view has reached the licensed upper limit")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED,           "Number of views has reached the licensed upper limit")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_BASIC_EXPIRED,          "License expired for basic functions")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_EXPIRED,         "License expired for stream function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SUBSCRIPTION_EXPIRED,   "License expired for subscription function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_EXPIRED,           "License expired for view function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_AUDIT_EXPIRED,          "License expired for audit function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CSV_EXPIRED,            "License expired for CSV function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED,  "License expired for multi-tier storage function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED, "License expired for object storage function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED,"License expired for dual-replica HA function")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED,  "License expired for database encryption function")
+
 
 // sync
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT,                  "Sync timeout")

tests/army/community/insert/test_column_tag_boundary.py

@@ -22,6 +22,7 @@ class TDTestCase(TBase):
         self.child_table_num = 1
         self.insert_round_num = 700
         self.row_num_per_round = 15
+        self.start_ts = 1704082431000
 
     def prepare_data(self):
         # database
@@ -39,7 +40,7 @@ class TDTestCase(TBase):
             for j in range(self.insert_round_num):
                 sql = "insert into ct_binary%s values" % (i+1)
                 for k in range(self.row_num_per_round):
-                    sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), 'a' * self.max_column_length)
+                    sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), 'a' * self.max_column_length)
                 tdSql.execute(sql)
                 tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_binary{i+1} {j+1} times successfully")
         tdSql.execute("flush database db;")
@@ -63,7 +64,7 @@ class TDTestCase(TBase):
             for j in range(self.insert_round_num):
                 sql = "insert into ct_varchar%s values" % (i+1)
                 for k in range(self.row_num_per_round):
-                    sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), 'b' * self.max_column_length)
+                    sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), 'b' * self.max_column_length)
                 tdSql.execute(sql)
                 tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_varchar{i+1} {j+1} times successfully")
         tdSql.execute("flush database db;")
@@ -98,7 +99,7 @@ class TDTestCase(TBase):
             for j in range(self.insert_round_num):
                 sql = "insert into ct_nchar%s values" % (i+1)
                 for k in range(self.row_num_per_round):
-                    sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), column)
+                    sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), column)
                 tdSql.execute(sql)
                 tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_nchar{i+1} {j+1} times successfully")
         tdSql.execute("flush database db;")
@@ -124,7 +125,7 @@ class TDTestCase(TBase):
             for j in range(self.insert_round_num):
                 sql = "insert into ct_varbinary%s values" % (i+1)
                 for k in range(row_num_per_round):
-                    sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), '\\x' + column)
+                    sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), '\\x' + column)
                 tdSql.execute(sql)
                 tdLog.info(f"Insert {row_num_per_round} rows data into ct_varbinary{i+1} {j+1} times successfully")
         tdSql.execute("flush database db;")
@@ -153,7 +154,7 @@ class TDTestCase(TBase):
             for j in range(self.insert_round_num):
                 sql = "insert into ct_json_tag%s values" % (i+1)
                 for k in range(row_num_per_round):
-                    sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), '\\x' + column)
+                    sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), '\\x' + column)
                 tdSql.execute(sql)
                 tdLog.info(f"Insert {row_num_per_round} rows data into ct_json_tag{i+1} {j+1} times successfully")
         tdSql.execute("flush database db;")

tests/army/community/query/accuracy/test_query_accuracy.py

@@ -0,0 +1,77 @@
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+from frame.eos import *
+import random
+import string
+
+
+class TDTestCase(TBase):
+    """Add test case to verify the complicated query accuracy
+    """
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def prepare_data(self):
+        # database for case TS-4806
+        tdSql.execute("create database db_ts4806;")
+        tdSql.execute("use db_ts4806;")
+        # super table
+        tdSql.execute("create table st (ts timestamp, adl float, bdl float, cdl float, ady float, bdy float, cdy float) \
+            tags(pt_radio float, ct_ratio float, rated_cap float, ta_id varchar(128), id varchar(128), area_code \
+            varchar(128), zdy_flag int, elec_cust_name bigint,bureau_code bigint, fl_name varchar(32), classify_id \
+                varchar(128));")
+        # child table
+        tdSql.execute("create table ct_1 using st tags(1.2, 1.3, 3.4, '271000276', '30000001', '10001', 1, 10001, 2000001, 'beijing', '13169');")
+        tdSql.execute("create table ct_2 using st tags(2.1, 1.2, 3.3, '271000277', '30000002', '10002', 1, 10002, 2000002, 'shanghai', '13141');")
+        tdSql.execute("create table ct_3 using st tags(3.1, 4.2, 5.3, '271000278', '30000003', '10003', 0, 10003, 2000003, 'guangzhou', '13151');")
+        # insert data for ts4806
+        start_ts = 1705783972000
+        data = [
+            (1.1, 2.2, 3.3, 1.1, 2.2, 3.3),
+            (1.2, 2.3, 3.4, 1.2, 2.3, 3.4),
+            (1.3, 2.4, 3.5, 1.3, 2.4, 3.5),
+            (1.4, 2.5, 3.6, 1.4, 2.5, 3.6),
+            (1.5, 2.6, 3.7, 1.5, 2.6, 3.7),
+            (1.6, 2.7, 3.8, 1.6, 2.7, 3.8),
+            (1.7, 2.8, 3.9, 1.7, 2.8, 3.9),
+            (1.8, 2.9, 4.0, 1.8, 2.9, 4.0),
+            (1.9, 4.2, 4.1, 1.9, 3.0, 4.1),
+            (1.2, 3.1, 4.2, 2.0, 3.1, 4.2)
+        ]
+        index = [1, 2, 5, 0, 7, 3, 8, 4, 6, 9]
+        for ct in ['ct_1', 'ct_2']:
+            for i in range(10):
+                sql = f"insert into {ct} values"
+                for j in range(1000):
+                    sql += f"({start_ts + i * 1000 * 1000 + j * 1000}, {','.join([str(item) for item in data[index[i]]])}),"
+                sql += ";"
+                tdSql.execute(sql)
+
+    def test_ts4806(self):
+        tdSql.execute("use db_ts4806;")
+        tdSql.query("select _wstart, cj.id, count(*) from st cj where cj.ts >= '2024-01-21 04:52:52.000' and cj.ts <= ' 2024-01-21 07:39:31.000' \
+            and cj.zdy_flag = 1 and cj.id in ('30000001', '30000002') partition by cj.id event_window start with \
+            (CASE WHEN cj.adl >= cj.bdl AND cj.adl >= cj.cdl THEN cj.adl WHEN cj.bdl >= cj.adl AND cj.bdl >= cj.cdl \
+            THEN cj.bdl ELSE cj.cdl END) * cj.ct_ratio * 0.4 * 1.732 / cj.rated_cap > 1 end with (CASE WHEN cj.adl >= \
+            cj.bdl AND cj.adl >= cj.cdl THEN cj.adl WHEN cj.bdl >= cj.adl AND cj.bdl >= cj.cdl THEN cj.bdl ELSE cj.cdl \
+            END) * cj.ct_ratio * 0.4 * 1.732 / cj.rated_cap <= 1 HAVING count(*) >= 4 order by _wstart, cj.id;")
+        tdSql.checkRows(5)
+        tdSql.checkData(4, 1, '30000002')
+        tdSql.checkData(4, 2, 1001)
+
+    def run(self):
+        self.prepare_data()
+        self.test_ts4806()
+
+    def stop(self):
+        tdSql.execute("drop database db_ts4806;")
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/army/enterprise/alter/alterConfig.py

@@ -0,0 +1,55 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import time
+
+import taos
+import frame
+import frame.etool
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+
+
+class TDTestCase(TBase):
+    def alterSupportVnodes(self):
+        tdLog.info(f"test function of altering supportVnodes")
+
+        tdSql.execute("alter dnode 1 'supportVnodes' '128'")
+        time.sleep(1)
+        tdSql.query('show dnodes')
+        tdSql.checkData(0, 3, "128")
+        
+        tdSql.execute("alter dnode 1 'supportVnodes' '64'")
+        time.sleep(1)
+        tdSql.query('show dnodes')
+        tdSql.checkData(0, 3, "64")
+
+    # run
+    def run(self):
+        tdLog.debug(f"start to excute {__file__}")
+
+        # TS-4721
+        self.alterSupportVnodes()
+
+
+        tdLog.success(f"{__file__} successfully executed")
+
+        
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/army/enterprise/db-encrypt/basic.py

@@ -0,0 +1,63 @@
+import taos
+import sys
+import os
+import subprocess
+import glob
+import shutil
+import time
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.srvCtl import *
+from frame.caseBase import *
+from frame import *
+from frame.autogen import *
+# from frame.server.dnodes import *
+# from frame.server.cluster import *
+
+
+class TDTestCase(TBase):
+    
+    def init(self, conn, logSql, replicaVar=1):
+        super(TDTestCase, self).init(conn, logSql, replicaVar=1, checkColName="c1")
+        self.valgrind = 0
+        self.db = "test"
+        self.stb = "meters"
+        self.childtable_count = 10
+        tdSql.init(conn.cursor(), logSql)  
+    
+    def create_encrypt_db(self):        
+        
+        tdSql.execute("create encrypt_key '1234567890'")
+        autoGen = AutoGen()
+        autoGen.create_db(self.db, 2, 1, "ENCRYPT_ALGORITHM 'sm4'")
+        tdSql.execute(f"use {self.db}")
+        autoGen.create_stable(self.stb, 2, 3, 8, 8)
+        autoGen.create_child(self.stb, "d", self.childtable_count)
+        autoGen.insert_data(1000)
+        
+        tdSql.query(f"select * from {self.db}.{self.stb}")
+        tdSql.checkRows(1000 * self.childtable_count)
+        
+        self.timestamp_step = 1000
+        self.insert_rows = 1000
+        
+        self.checkInsertCorrect()           
+    
+    def create_encrypt_db_error(self):
+        tdSql.error("create encrypt_key '123'")
+        tdSql.error("create encrypt_key '12345678abcdefghi'")
+        tdSql.error("create database test ENCRYPT_ALGORITHM 'sm4'")
+
+    def run(self):
+        self.create_encrypt_db_error()
+        self.create_encrypt_db()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/parallel_test/cases.task

@@ -11,6 +11,7 @@
 # army-test
 #
 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2
+,,y,army,./pytest.sh python3 ./test.py -f enterprise/db-encrypt/basic.py
 ,,n,army,python3 ./test.py -f enterprise/s3/s3Basic.py -N 3
 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2
 ,,y,army,./pytest.sh python3 ./test.py -f community/query/function/test_func_elapsed.py
@@ -20,10 +21,12 @@
 ,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2
 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3
 ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3
+,,y,army,./pytest.sh python3 ./test.py -f community/query/accuracy/test_query_accuracy.py
 ,,y,army,./pytest.sh python3 ./test.py -f community/insert/insert_basic.py -N 3
 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3
 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
 ,,n,army,python3 ./test.py -f community/query/show.py -N 3
+,,n,army,python3 ./test.py -f enterprise/alter/alterConfig.py -N 3
 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1
 ,,y,army,./pytest.sh python3 ./test.py -f community/storage/compressBasic.py -N 3
 
@@ -994,6 +997,7 @@
 ,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py
 
 #tsim test
+,,y,script,./test.sh -f tsim/query/timeline.sim
 ,,y,script,./test.sh -f tsim/join/join.sim
 ,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim
 ,,y,script,./test.sh -f tsim/parser/where.sim

tests/parallel_test/run.sh

@@ -315,19 +315,22 @@ function run_thread() {
             fi
             if [ -n "$corefile" ]; then
                 echo -e "\e[34m corefiles: $corefile \e[0m"
-                local build_dir=$log_dir/build_${hosts[index]}
-                local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build"
-                # if [ $ent -ne 0 ]; then
-                #     remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build"
-                # fi
-                mkdir "$build_dir" 2>/dev/null
-                if [ $? -eq 0 ]; then
-                    # scp build binary
-                    cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/"
-                    echo "$cmd"
-                    $cmd >/dev/null
-                fi
             fi
+            # scp build binary and unit test log
+            local build_dir=$log_dir/build_${hosts[index]}
+            local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build"
+            local remote_unit_test_log_dir="${workdirs[index]}/${DEBUGPATH}/Testing/Temporary/"
+
+            mkdir "$build_dir" 2>/dev/null
+            if [ $? -eq 0 ]; then
+                cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/"
+                echo "$cmd"
+                $cmd >/dev/null
+                cmd="$scpcmd:${remote_unit_test_log_dir}/* ${build_dir}/"
+                echo "$cmd"
+                $cmd >/dev/null
+            fi
+
             # get remote sim dir
             local remote_sim_dir="${workdirs[index]}/tmp/thread_volume/$thread_no"
             local tarcmd="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}"

tests/pytest/util/common.py

@@ -201,6 +201,9 @@ class TDCom:
         self.cast_tag_stb_filter_des_select_elm = "ts, t1, t2, t3, t4, cast(t1 as TINYINT UNSIGNED), t6, t7, t8, t9, t10, cast(t2 as varchar(256)), t12, cast(t3 as bool)"
         self.tag_count = len(self.tag_filter_des_select_elm.split(","))
         self.state_window_range = list()
+
+        self.custom_col_val = 0
+        self.part_val_list = [1, 2]
     # def init(self, conn, logSql):
     #     # tdSql.init(conn.cursor(), logSql)
 
@@ -1259,7 +1262,7 @@ class TDCom:
                 default_ctbname_index_start_num += 1
                 tdSql.execute(create_stable_sql)
 
-    def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None):
+    def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None, additional_ts=None, custom_col_index=None, col_value_type=None, force_pk_val=None):
         """_summary_
 
         Args:
@@ -1269,6 +1272,8 @@ class TDCom:
         """
         self.column_value_list = list()
         self.ts_value = self.genTs()[0]
+        if additional_ts is not None:
+            self.additional_ts = self.genTs(additional_ts=additional_ts)[2]
         if ts_value is not None:
             self.ts_value = ts_value
 
@@ -1292,7 +1297,22 @@ class TDCom:
             for i in range(int(len(self.column_value_list)/2)):
                 index_num = random.randint(0, len(self.column_value_list)-1)
                 self.column_value_list[index_num] = None
-        self.column_value_list = [self.ts_value] + self.column_value_list
+
+        if custom_col_index is not None:
+            if col_value_type == "Random":
+                pass
+            elif col_value_type == "Incremental":
+                self.column_value_list[custom_col_index] = self.custom_col_val
+                self.custom_col_val += 1
+            elif col_value_type == "Part_equal":
+                self.column_value_list[custom_col_index] = random.choice(self.part_val_list)
+
+        self.column_value_list = [self.ts_value] + [self.additional_ts] + self.column_value_list if additional_ts is not None else [self.ts_value] + self.column_value_list
+        if col_value_type == "Incremental" and custom_col_index==1:
+            self.column_value_list[custom_col_index] = self.custom_col_val if force_pk_val is None else force_pk_val
+        if col_value_type == "Part_equal" and custom_col_index==1:
+            self.column_value_list[custom_col_index] = random.randint(0, self.custom_col_val) if force_pk_val is None else force_pk_val
+
 
     def screate_table(self, dbname=None, tbname="tb", use_name="table", column_elm_list=None,
                     count=1, default_tbname_prefix="tb", default_tbname_index_start_num=1,
@@ -1333,7 +1353,7 @@ class TDCom:
                 default_tbname_index_start_num += 1
                 tdSql.execute(create_table_sql)
 
-    def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False):
+    def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False, custom_col_index=None, col_value_type="random"):
         """insert rows
 
         Args:
@@ -1353,7 +1373,7 @@ class TDCom:
             if tbname is not None:
                 self.tbname = tbname
 
-        self.sgen_column_value_list(column_ele_list, need_null, ts_value)
+        self.sgen_column_value_list(column_ele_list, need_null, ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
         # column_value_str = ", ".join(str(v) for v in self.column_value_list)
         column_value_str = ""
         for column_value in self.column_value_list:
@@ -1370,7 +1390,7 @@ class TDCom:
         else:
             for num in range(count):
                 ts_value = self.genTs()[0]
-                self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s')
+                self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s', custom_col_index=custom_col_index, col_value_type=col_value_type)
                 column_value_str = ""
                 for column_value in self.column_value_list:
                     if column_value is None:
@@ -1777,7 +1797,7 @@ class TDCom:
             self.sdelete_rows(tbname=self.ctb_name, start_ts=self.time_cast(self.record_history_ts, "-"))
             self.sdelete_rows(tbname=self.tb_name, start_ts=self.time_cast(self.record_history_ts, "-"))
 
-    def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None):
+    def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None, custom_col_index=None, col_value_type="random"):
         """prepare stream data
 
         Args:
@@ -1840,8 +1860,8 @@ class TDCom:
         if fill_history_value == 1:
             for i in range(self.range_count):
                 ts_value = str(self.date_time)+f'-{self.default_interval*(i+1)}s'
-                self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value)
+                self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
-                self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value)
+                self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
                 if i == 1:
                     self.record_history_ts = ts_value
 
@@ -1862,6 +1882,18 @@ class TDCom:
                 time.sleep(1)
         return tbname
 
+    def get_group_id_from_stb(self, stbname):
+        tdSql.query(f'select distinct group_id from {stbname}')
+        cnt = 0
+        while len(tdSql.queryResult) == 0:
+            tdSql.query(f'select distinct group_id from {stbname}')
+            if cnt < self.default_interval:
+                cnt += 1
+                time.sleep(1)
+            else:
+                return False
+        return tdSql.queryResult[0][0]
+
     def update_json_file_replica(self, json_file_path, new_replica_value, output_file_path=None):
         """
         Read a JSON file, update the 'replica' value, and write the result back to a file.

tests/script/tsim/query/timeline.sim

@@ -0,0 +1,51 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql create database test;
+sql use test;
+
+sql CREATE STABLE `demo` (`_ts` TIMESTAMP, `faev` DOUBLE) TAGS (`deviceid` VARCHAR(256));
+sql CREATE TABLE demo_1 USING demo (deviceid) TAGS ('1');
+sql CREATE TABLE demo_2 USING demo (deviceid) TAGS ('2');
+sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 1.0);
+sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 01:00:00.001', 2.0);
+sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 02:00:00.002', 3.0);
+sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-05 03:00:00.003', 4.0);
+sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 5.0);
+sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 01:00:00.001', 6.0);
+sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 02:00:00.002', 7.0);
+sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-29 03:00:00.003', 8.0);
+
+sql_error select diff(faev) from ((select ts, faev from demo union all select ts, faev from demo));
+sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts);
+sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts) partition by faev;
+sql select diff(faev) from (select _ts, faev from demo union all select _ts + 1s, faev from demo order by faev, _ts) partition by faev;
+sql_error select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by deviceid, _ts) partition by faev;
+sql select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev;
+
+sql_error select diff(faev) from (select _ts, faev from demo);
+sql_error select diff(faev) from (select _ts, faev from demo order by faev, _ts);
+sql select diff(faev) from (select _ts, faev from demo order by faev, _ts) partition by faev;
+sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts) partition by deviceid;
+sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by deviceid, _ts) partition by faev;
+sql select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY deviceid, ts) PARTITION by deviceid;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY ts, deviceid) PARTITION by deviceid;
+
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid;
+sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY deviceid, ts) PARTITION by deviceid;
+
+sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY ts, deviceid) PARTITION by deviceid;
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/system-test/0-others/information_schema.py

@@ -222,7 +222,7 @@ class TDTestCase:
 
         tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
         tdLog.info(len(tdSql.queryResult))
-        tdSql.checkEqual(True, len(tdSql.queryResult) in range(254, 255))
+        tdSql.checkEqual(True, len(tdSql.queryResult) in range(255, 256))
 
         tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
         tdSql.checkEqual(54, len(tdSql.queryResult))
@@ -336,7 +336,7 @@ class TDTestCase:
             tdSql.checkEqual(True, result[i][1] in key_status_list[1])
             index += 1
         tdSql.checkEqual(True, index > 0)
-                    
+
         tdSql.query(f'show encryptions')
         result = tdSql.queryResult
         index = 0
@@ -344,7 +344,7 @@ class TDTestCase:
             tdSql.checkEqual(True, result[i][1] in key_status_list[1])
             index += 1
         tdSql.checkEqual(True, index > 0)
-        
+
         # loaded/sm4
         tdSql.execute('drop database if exists db2')
         tdSql.execute('create encrypt_key \'12345678\'')
@@ -357,7 +357,7 @@ class TDTestCase:
             tdSql.checkEqual(True, result[i][1] in key_status_list[3])
             index += 1
         tdSql.checkEqual(True, index > 0)
-                    
+
         tdSql.query(f'show encryptions')
         result = tdSql.queryResult
         index = 0

tests/system-test/2-query/interp.py

@@ -15,6 +15,30 @@ class TDTestCase:
         #tdSql.init(conn.cursor())
         tdSql.init(conn.cursor(), logSql)  # output sql.txt file
 
+    def interp_on_empty_table(self):
+        dbname = "db"
+        tbname = "t"        
+
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+
+        tdSql.execute(f'''create table if not exists {dbname}.{tbname} (ts timestamp, k int)''')
+
+        tdLog.printNoPrefix("==========step2:interp query on empty table")
+
+        tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(prev)")
+        tdSql.checkRows(0)
+        
+        tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(next)")
+        tdSql.checkRows(0)
+        
+        tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(linear)")
+        tdSql.checkRows(0)
+        
+        tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(value, 2)")        
+        tdSql.checkRows(0)
+    
     def run(self):
         dbname = "db"
         tbname = "tb"
@@ -5658,6 +5682,7 @@ class TDTestCase:
         tdSql.checkData(0,  0, '2023-08-06 23:59:00')
         tdSql.checkData(0,  1, None)
 
+        self.interp_on_empty_table()
 
 
     def stop(self):

tests/system-test/2-query/union1.py

@@ -240,9 +240,22 @@ class TDTestCase:
         tdSql.error( " '' union all select c1 from ct1 " )
         # tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ")
 
+    def test_select_from_union_all(self):
+        tdSql.query('select c8, ts from ((select ts, c8,c1 from stb1 order by c1) union all select ts, c8, c1 from stb1 limit 15)')
+        tdSql.checkRows(15)
+        tdSql.query('select c8, ts from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
+        tdSql.checkRows(15)
+        tdSql.query('select ts, c1 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
+        tdSql.checkRows(15)
+        tdSql.query('select ts, c1, c8 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
+        tdSql.checkRows(15)
+        tdSql.query('select ts, c8, c1, 123 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)')
+        tdSql.checkRows(15)
+
     def all_test(self):
         self.__test_error()
         self.union_check()
+        self.test_select_from_union_all()
 
 
     def __create_tb(self):

tests/system-test/8-stream/at_once_interval.py

@@ -15,9 +15,12 @@ class TDTestCase:
 
     def at_once_interval(self, interval, partition="tbname", delete=False, fill_value=None, fill_history_value=None, case_when=None):
         tdLog.info(f"*** testing stream at_once+interval: interval: {interval}, partition: {partition}, fill_history: {fill_history_value}, fill: {fill_value}, delete: {delete}, case_when: {case_when} ***")
+        col_value_type = "Incremental" if partition=="c1" else "random"
+        custom_col_index = 1 if partition=="c1" else None
+        self.tdCom.custom_col_val = 0
         self.delete = delete
         self.tdCom.case_name = sys._getframe().f_code.co_name
-        self.tdCom.prepare_data(interval=interval, fill_history_value=fill_history_value)
+        self.tdCom.prepare_data(interval=interval, fill_history_value=fill_history_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
         self.stb_name = self.tdCom.stb_name.replace(f"{self.tdCom.dbname}.", "")
         self.ctb_name = self.tdCom.ctb_name.replace(f"{self.tdCom.dbname}.", "")
         self.tb_name = self.tdCom.tb_name.replace(f"{self.tdCom.dbname}.", "")
@@ -76,15 +79,15 @@ class TDTestCase:
         for i in range(self.tdCom.range_count):
             ts_value = str(self.tdCom.date_time+self.tdCom.dataDict["interval"])+f'+{i*10}s'
             ts_cast_delete_value = self.tdCom.time_cast(ts_value)
-            self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value)
+            self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
             if i%2 == 0:
-                self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value)
+                self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
             if self.delete and i%2 != 0:
                 self.tdCom.sdelete_rows(tbname=self.tdCom.ctb_name, start_ts=ts_cast_delete_value)
             self.tdCom.date_time += 1
-            self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value)
+            self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
             if i%2 == 0:
-                self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value)
+                self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
             if self.delete and i%2 != 0:
                 self.tdCom.sdelete_rows(tbname=self.tdCom.tb_name, start_ts=ts_cast_delete_value)
             self.tdCom.date_time += 1
@@ -102,6 +105,7 @@ class TDTestCase:
 
         if self.tdCom.subtable:
             for tname in [self.stb_name, self.ctb_name]:
+                group_id = self.tdCom.get_group_id_from_stb(f'{tname}_output')
                 tdSql.query(f'select * from {self.ctb_name}')
                 ptn_counter = 0
                 for c1_value in tdSql.queryResult:
@@ -116,11 +120,11 @@ class TDTestCase:
                         tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{abs_c1_value}{self.tdCom.subtable_suffix}')
                         tdSql.query(f'select count(*) from `{tbname}`')
                     elif partition == "tbname" and ptn_counter == 0:
-                        tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}')
+                        tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}_{tname}_output_{group_id}')
                         tdSql.query(f'select count(*) from `{tbname}`')
                         ptn_counter += 1
                     tdSql.checkEqual(tdSql.queryResult[0][0] > 0, True)
-
+            group_id = self.tdCom.get_group_id_from_stb(f'{self.tb_name}_output')
             tdSql.query(f'select * from {self.tb_name}')
             ptn_counter = 0
             for c1_value in tdSql.queryResult:
@@ -135,7 +139,7 @@ class TDTestCase:
                     tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{abs_c1_value}{self.tdCom.subtable_suffix}')
                     tdSql.query(f'select count(*) from `{tbname}`')
                 elif partition == "tbname" and ptn_counter == 0:
-                    tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}')
+                    tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}_{self.tb_name}_output_{group_id}')
                     tdSql.query(f'select count(*) from `{tbname}`')
                     ptn_counter += 1
 

tests/system-test/8-stream/at_once_session.py

@@ -15,9 +15,12 @@ class TDTestCase:
 
     def at_once_session(self, session, ignore_expired=None, ignore_update=None, partition="tbname", delete=False, fill_history_value=None, case_when=None, subtable=True):
         tdLog.info(f"*** testing stream at_once+interval: session: {session}, ignore_expired: {ignore_expired}, ignore_update: {ignore_update}, partition: {partition}, delete: {delete}, fill_history: {fill_history_value}, case_when: {case_when}, subtable: {subtable} ***")
+        col_value_type = "Incremental" if partition=="c1" else "random"
+        custom_col_index = 1 if partition=="c1" else None
+        self.tdCom.custom_col_val = 0
         self.delete = delete
         self.tdCom.case_name = sys._getframe().f_code.co_name
-        self.tdCom.prepare_data(session=session, fill_history_value=fill_history_value)
+        self.tdCom.prepare_data(session=session, fill_history_value=fill_history_value, custom_col_index=custom_col_index, col_value_type=col_value_type)
         self.stb_name = self.tdCom.stb_name.replace(f"{self.tdCom.dbname}.", "")
         self.ctb_name = self.tdCom.ctb_name.replace(f"{self.tdCom.dbname}.", "")
         self.tb_name = self.tdCom.tb_name.replace(f"{self.tdCom.dbname}.", "")
@@ -79,11 +82,11 @@ class TDTestCase:
             if i == 0:
                 record_window_close_ts = window_close_ts
             for ts_value in [self.tdCom.date_time, window_close_ts]:
-                self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True)
+                self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type)
-                self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True)
+                self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type)
                 if self.tdCom.update and i%2 == 0:
-                    self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True)
+                    self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type)
-                    self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True)
+                    self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type)
                 if self.delete and i%2 != 0:
                     dt = f'cast({self.tdCom.date_time-1} as timestamp)'
                     self.tdCom.sdelete_rows(tbname=self.ctb_name, start_ts=dt)
@@ -166,6 +169,7 @@ class TDTestCase:
                 self.tdCom.sdelete_rows(tbname=self.tb_name, start_ts=self.tdCom.time_cast(self.tdCom.record_history_ts, "-"))
 
         if self.tdCom.subtable:
+            group_id = self.tdCom.get_group_id_from_stb(f'{self.ctb_name}_output')
             tdSql.query(f'select * from {self.ctb_name}')
             ptn_counter = 0
             for c1_value in tdSql.queryResult:
@@ -182,11 +186,11 @@ class TDTestCase:
                             tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{partition_elm_alias}{self.tdCom.subtable_suffix}')
                             tdSql.query(f'select count(*) from `{tbname}`')
                     elif partition == "tbname" and ptn_counter == 0:
-                        tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}')
+                        tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}_{self.ctb_name}_output_{group_id}')
                         tdSql.query(f'select count(*) from `{tbname}`')
                         ptn_counter += 1
                     tdSql.checkEqual(tdSql.queryResult[0][0] > 0, True) if subtable is not None else tdSql.checkEqual(tdSql.queryResult[0][0] >= 0, True)
-
+            group_id = self.tdCom.get_group_id_from_stb(f'{self.tb_name}_output')
             tdSql.query(f'select * from {self.tb_name}')
             ptn_counter = 0
             for c1_value in tdSql.queryResult:
@@ -203,7 +207,7 @@ class TDTestCase:
                             tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{partition_elm_alias}{self.tdCom.subtable_suffix}')
                             tdSql.query(f'select count(*) from `{tbname}`')
                     elif partition == "tbname" and ptn_counter == 0:
-                        tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}')
+                        tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}_{self.tb_name}_output_{group_id}')
                         tdSql.query(f'select count(*) from `{tbname}`')
                         ptn_counter += 1
 

tests/system-test/8-stream/stream_multi_agg.py

@@ -39,7 +39,7 @@ class TDTestCase:
 
         os.system("nohup taosBenchmark -y -B 1 -t 40 -S 1000 -n 10 -i 1000 -v 5  > /dev/null 2>&1 &")
         time.sleep(10)
-        tdSql.query("use test")
+        tdSql.execute("use test", queryTimes=100)
         tdSql.query("create stream if not exists s1 trigger at_once  ignore expired 0 ignore update 0  fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)")
         tdLog.debug("========create stream and insert data ok========")
         time.sleep(15)
@@ -66,7 +66,7 @@ class TDTestCase:
 
         os.system("taosBenchmark -d db -t 20 -v 6 -n 1000 -y  > /dev/null 2>&1")
         # create stream
-        tdSql.execute("use db")
+        tdSql.execute("use db", queryTimes=100)
         tdSql.execute("create stream stream1 fill_history 1 into sta as select count(*) as cnt from meters interval(10a);",show=True)
         time.sleep(5)
 

tests/system-test/pytest.sh

@@ -89,8 +89,8 @@ else
   export LD_PRELOAD="$(realpath "$(gcc -print-file-name=libasan.so)") $(realpath "$(gcc -print-file-name=libstdc++.so)")"
   echo "Preload AsanSo:" $?
 
-  $* -a 2>$AsanFile
+  $* -a 2> $AsanFile 
-
+  cat $AsanFile
   unset LD_PRELOAD
   for ((i = 1; i <= 20; i++)); do
     AsanFileLen=$(cat $AsanFile | wc -l)

tools/CMakeLists.txt

@@ -140,8 +140,8 @@ ELSE ()
             BUILD_COMMAND
             COMMAND set CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client
             COMMAND set CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib
-            COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
+            COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
-            COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
+            COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
 
             INSTALL_COMMAND
             COMMAND cmake -E echo "Comparessing taosadapter.exe"
@@ -167,8 +167,8 @@ ELSE ()
                 PATCH_COMMAND
                 COMMAND     git clean -f -d
                 BUILD_COMMAND
-                COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib  go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
+                COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib  go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
-                COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib  go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
+                COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib  go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
                 INSTALL_COMMAND
                 COMMAND cmake -E echo "Copy taosadapter"
                 COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin
@@ -192,19 +192,19 @@ ELSE ()
             PATCH_COMMAND
             COMMAND     git clean -f -d
             BUILD_COMMAND
-            COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib  go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
+            COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib  go build -a -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
-            COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib  go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'"
+#            COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib  go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'"
             INSTALL_COMMAND
-            COMMAND cmake -E echo "Comparessing taosadapter.exe"
+#            COMMAND cmake -E echo "Comparessing taosadapter.exe"
-            COMMAND upx taosadapter || :
+#            COMMAND upx taosadapter || :
             COMMAND cmake -E echo "Copy taosadapter"
             COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin
             COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/
             COMMAND cmake -E echo "Copy taosadapter.toml"
             COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/
             COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/
-            COMMAND cmake -E echo "Copy taosadapter-debug"
+#            COMMAND cmake -E echo "Copy taosadapter-debug"
-            COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin
+#            COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin
             )
     ENDIF ()
 ENDIF ()

utils/test/c/write_raw_block_test.c

@@ -78,7 +78,7 @@ int buildStable(TAOS* pConn) {
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "insert into ntba values(now,'hello')");
+  pRes = taos_query(pConn, "insert into ntba values(now + 1s,'hello')");
   if (taos_errno(pRes) != 0) {
     printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes));
     return -1;