Merge branch '3.0' into case/army

.gitignore

@@ -11,6 +11,7 @@ CMakeSettings.json
 cmake-build-debug/
 cmake-build-release/
 cscope.out
+cscope.files
 .DS_Store
 debug/
 release/

docs/en/14-reference/02-rest-api/02-rest-api.mdx

@@ -262,6 +262,63 @@ The following types may be returned:
 - "INT UNSIGNED"
 - "BIGINT UNSIGNED"
 - "JSON"
+- "VARBINARY"
+- "GEOMETRY"
+
+`VARBINARY` and `GEOMETRY` types return data as Hex string, example:
+
+Prepare data
+
+```bash
+create database demo
+use demo
+create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
+insert into t values(now,'\x7f8290','point(100 100)')
+```
+
+Execute query
+
+```bash
+curl --location 'http://<fqdn>:<port>/rest/sql' \
+--header 'Content-Type: text/plain' \
+--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
+--data 'select * from demo.t'
+```
+
+Return results
+
+```json
+{
+     "code": 0,
+     "column_meta": [
+         [
+             "ts",
+             "TIMESTAMP",
+             8
+         ],
+         [
+             "c1",
+             "VARBINARY",
+             20
+         ],
+         [
+             "c2",
+             "GEOMETRY",
+             100
+         ]
+     ],
+     "data": [
+         [
+             "2023-11-01T06:28:15.210Z",
+             "7f8290",
+             "010100000000000000000059400000000000005940"
+         ]
+     ],
+     "rows": 1
+}
+```
+
+- `010100000000000000000059400000000000005940` is [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/) format for `point(100 100)`
 
 #### Errors
 

docs/zh/08-connector/02-rest-api.mdx

@@ -257,6 +257,63 @@ curl -L -u username:password -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name][?tz=timez
 - "INT UNSIGNED"
 - "BIGINT UNSIGNED"
 - "JSON"
+- "VARBINARY"
+- "GEOMETRY"
+
+`VARBINARY` 和 `GEOMETRY` 类型返回数据为 Hex 字符串，样例：
+
+准备数据
+
+```bash
+create database demo
+use demo
+create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
+insert into t values(now,'\x7f8290','point(100 100)')
+```
+
+执行查询
+
+```bash
+curl --location 'http://<fqdn>:<port>/rest/sql' \
+--header 'Content-Type: text/plain' \
+--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
+--data 'select * from demo.t'
+```
+
+返回结果
+
+```json
+{
+    "code": 0,
+    "column_meta": [
+        [
+            "ts",
+            "TIMESTAMP",
+            8
+        ],
+        [
+            "c1",
+            "VARBINARY",
+            20
+        ],
+        [
+            "c2",
+            "GEOMETRY",
+            100
+        ]
+    ],
+    "data": [
+        [
+            "2023-11-01T06:28:15.210Z",
+            "7f8290",
+            "010100000000000000000059400000000000005940"
+        ]
+    ],
+    "rows": 1
+}
+```
+
+- `010100000000000000000059400000000000005940` 为 `point(100 100)` 的 [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/)  格式
 
 #### 错误
 

include/common/tcommon.h

@@ -253,6 +253,7 @@ typedef struct SQueryTableDataCond {
   STimeWindow  twindows;
   int64_t      startVersion;
   int64_t      endVersion;
+  bool         notLoadData;    // response the actual data, not only the rows in the attribute of info.row of ssdatablock
 } SQueryTableDataCond;
 
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);

include/common/tmsgdef.h

@@ -26,10 +26,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) "null",
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) MSG, MSG "-rsp",
-  #define TD_CLOSE_MSG_TYPE(TYPE)
+  #define TD_CLOSE_MSG_SEG(TYPE)
 
   char *tMsgInfo[] = {
 
@@ -37,20 +37,20 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) 
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
-  #define TD_CLOSE_MSG_TYPE(TYPE) TYPE,
+  #define TD_CLOSE_MSG_SEG(TYPE) TYPE,
   int32_t tMsgRangeDict[] = {
 
 #elif defined(TD_MSG_NUMBER_)
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM,
-  #define TD_CLOSE_MSG_TYPE(TYPE)
+  #define TD_CLOSE_MSG_SEG(TYPE)
 
   enum {
 
@@ -58,10 +58,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
-  #define TD_CLOSE_MSG_TYPE(type)
+  #define TD_CLOSE_MSG_SEG(type)
 
   int32_t tMsgDict[] = {
 
@@ -70,10 +70,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE##_SEG_CODE,
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
-  #define TD_CLOSE_MSG_TYPE(TYPE)
+  #define TD_CLOSE_MSG_SEG(TYPE)
 
   enum {
 
@@ -82,10 +82,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8),
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP,
-  #define TD_CLOSE_MSG_TYPE(TYPE) TYPE,
+  #define TD_CLOSE_MSG_SEG(TYPE) TYPE,
 
   enum { // WARN: new msg should be appended to segment tail
 #endif
@@ -109,7 +109,7 @@
   TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_DND_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_DND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_MND_MSG)  // 1<<8
   TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL)
@@ -218,7 +218,7 @@
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_MND_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_VND_MSG)  // 2<<8
   TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
@@ -268,7 +268,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_VND_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_SCH_MSG)  // 3<<8
   TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL)
@@ -283,7 +283,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_SCH_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_SCH_MSG) 
 
 
   TD_NEW_MSG_SEG(TDMT_STREAM_MSG)   //4 << 8
@@ -301,11 +301,11 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_HTASK_DROP, "stream-htask-drop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_STREAM_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_MON_MSG)  //5 << 8
   TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_MON_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_MON_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_SYNC_MSG) //6 << 8
   TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL)
@@ -337,7 +337,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT_REPLY, "sync-prep-snapshot-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_SYNC_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_SYNC_MSG) 
 
 
   TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8
@@ -348,7 +348,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_VND_STREAM_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG)  //8 << 8
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp)
@@ -362,10 +362,10 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committedinfo", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_TMQ_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_TMQ_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_MAX_MSG)  // msg end mark
-  
+  TD_CLOSE_MSG_SEG(TDMT_END_MAX_MSG) 
 
   
 

include/libs/executor/storageapi.h

@@ -166,8 +166,7 @@ typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyI
 
 typedef struct TsdReader {
   int32_t      (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
-                           SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly,
+                           SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables);
-                           SHashObj** pIgnoreTables);
   void         (*tsdReaderClose)();
   void         (*tsdSetReaderTaskId)(void *pReader, const char *pId);
   int32_t      (*tsdSetQueryTableList)();

include/libs/sync/sync.h

@@ -289,6 +289,9 @@ const char* syncStr(ESyncState state);
 
 int32_t    syncNodeGetConfig(int64_t rid, SSyncCfg *cfg);
 
+// util
+int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tdef.h

@@ -288,6 +288,7 @@ typedef enum ELogicConditionType {
 #define TSDB_CONN_ACTIVE_KEY_LEN 255
 
 #define TSDB_DEFAULT_PKT_SIZE 65480  // same as RPC_MAX_UDP_SIZE
+#define TSDB_SNAP_DATA_PAYLOAD_SIZE (1 * 1024 * 1024)
 
 #define TSDB_PAYLOAD_SIZE         TSDB_DEFAULT_PKT_SIZE
 #define TSDB_DEFAULT_PAYLOAD_SIZE 5120  // default payload size, greater than PATH_MAX value

source/dnode/vnode/inc/vnode.h

@@ -154,8 +154,7 @@ typedef struct STsdbReader STsdbReader;
 #define CACHESCAN_RETRIEVE_LAST        0x8
 
 int32_t      tsdbReaderOpen2(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
-                             SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly,
+                             SSDataBlock *pResBlock, void **ppReader, const char *idstr, SHashObj **pIgnoreTables);
-                             SHashObj **pIgnoreTables);
 int32_t      tsdbSetTableList2(STsdbReader *pReader, const void *pTableList, int32_t num);
 void         tsdbReaderSetId2(STsdbReader *pReader, const char *idstr);
 void         tsdbReaderClose2(STsdbReader *pReader);

source/dnode/vnode/src/inc/tsdb.h

@@ -676,46 +676,57 @@ struct SDelFWriter {
 };
 
 #include "tarray2.h"
-// #include "tsdbFS2.h"
-//  struct STFileSet;
 typedef struct STFileSet STFileSet;
 typedef TARRAY2(STFileSet *) TFileSetArray;
 
-typedef struct STSnapRange STSnapRange;
+// fset range
-typedef TARRAY2(STSnapRange *) TSnapRangeArray;  // disjoint snap ranges
+typedef struct STFileSetRange STFileSetRange;
+typedef TARRAY2(STFileSetRange *) TFileSetRangeArray;  // disjoint ranges
 
-// util
+int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr);
-int32_t   tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR);
+int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr);
-int32_t   tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR);
-void      tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap);
-SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges);
-
-// snap partition list
-typedef TARRAY2(SVersionRange) SVerRangeList;
-typedef struct STsdbSnapPartition STsdbSnapPartition;
-typedef TARRAY2(STsdbSnapPartition *) STsdbSnapPartList;
-// util
-STsdbSnapPartList *tsdbSnapPartListCreate();
-void               tsdbSnapPartListDestroy(STsdbSnapPartList **ppList);
-int32_t            tSerializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList);
-int32_t            tDeserializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList);
-int32_t            tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TSnapRangeArray **ppRanges);
 
+// fset partition
 enum {
-  TSDB_SNAP_RANGE_TYP_HEAD = 0,
+  TSDB_FSET_RANGE_TYP_HEAD = 0,
-  TSDB_SNAP_RANGE_TYP_DATA,
+  TSDB_FSET_RANGE_TYP_DATA,
-  TSDB_SNAP_RANGE_TYP_SMA,
+  TSDB_FSET_RANGE_TYP_SMA,
-  TSDB_SNAP_RANGE_TYP_TOMB,
+  TSDB_FSET_RANGE_TYP_TOMB,
-  TSDB_SNAP_RANGE_TYP_STT,
+  TSDB_FSET_RANGE_TYP_STT,
-  TSDB_SNAP_RANGE_TYP_MAX,
+  TSDB_FSET_RANGE_TYP_MAX,
 };
 
-struct STsdbSnapPartition {
+typedef TARRAY2(SVersionRange) SVerRangeList;
+
+struct STsdbFSetPartition {
   int64_t       fid;
   int8_t        stat;
-  SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX];
+  SVerRangeList verRanges[TSDB_FSET_RANGE_TYP_MAX];
 };
 
+typedef struct STsdbFSetPartition STsdbFSetPartition;
+typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList;
+
+STsdbFSetPartList *tsdbFSetPartListCreate();
+void               tsdbFSetPartListDestroy(STsdbFSetPartList **ppList);
+int32_t            tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList);
+int32_t            tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList);
+int32_t            tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges);
+
+// snap rep format
+typedef enum ETsdbRepFmt {
+  TSDB_SNAP_REP_FMT_DEFAULT = 0,
+  TSDB_SNAP_REP_FMT_RAW,
+  TSDB_SNAP_REP_FMT_HYBRID,
+} ETsdbRepFmt;
+
+typedef struct STsdbRepOpts {
+  ETsdbRepFmt format;
+} STsdbRepOpts;
+
+int32_t tSerializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo);
+int32_t tDeserializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo);
+
 // snap read
 struct STsdbReadSnap {
   SMemTable     *pMem;
@@ -773,11 +784,16 @@ typedef struct SBlockDataInfo {
   int32_t    sttBlockIndex;
 } SBlockDataInfo;
 
+// todo: move away
+typedef struct {
+  SArray *pUid;
+  SArray *pFirstKey;
+  SArray *pLastKey;
+  SArray *pCount;
+} SSttTableRowsInfo;
+
 typedef struct SSttBlockLoadInfo {
   SBlockDataInfo  blockData[2];  // buffered block data
-  int32_t        statisBlockIndex;  // buffered statistics block index
-  void          *statisBlock;       // buffered statistics block data
-  void          *pSttStatisBlkArray;
   SArray         *aSttBlk;
   int32_t         currentLoadBlockIndex;
   STSchema       *pSchema;
@@ -786,7 +802,7 @@ typedef struct SSttBlockLoadInfo {
   bool            checkRemainingRow;  // todo: no assign value?
   bool            isLast;
   bool            sttBlockLoaded;
-
+  SSttTableRowsInfo info;
   SSttBlockLoadCostInfo cost;
 } SSttBlockLoadInfo;
 
@@ -875,10 +891,15 @@ typedef struct {
   _load_tomb_fn loadTombFn;
   void         *pReader;
   void         *idstr;
+  bool          rspRows;         // response the rows in stt-file, if possible
 } SMergeTreeConf;
 
-int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf);
+typedef struct SSttDataInfoForTable {
+  SArray* pTimeWindowList;
+  int64_t numOfRows;
+} SSttDataInfoForTable;
 
+int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoForTable* pTableInfo);
 void    tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
 bool    tMergeTreeNext(SMergeTree *pMTree);
 void    tMergeTreePinSttBlock(SMergeTree *pMTree);
@@ -887,15 +908,14 @@ bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
 void    tMergeTreeClose(SMergeTree *pMTree);
 
 SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols);
-void               getSttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, SSttBlockLoadCostInfo *pLoadCost);
 void              *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
 void              *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost);
 
 // tsdbCache ==============================================================================================
 typedef enum {
-  READ_MODE_COUNT_ONLY = 0x1,
+  READER_EXEC_DATA = 0x1,
-  READ_MODE_ALL,
+  READER_EXEC_ROWS = 0x2,
-} EReadMode;
+} EExecMode;
 
 typedef struct {
   TSKEY   ts;
@@ -1042,7 +1062,7 @@ typedef enum {
 
 // utils
 ETsdbFsState tsdbSnapGetFsState(SVnode *pVnode);
-int32_t      tsdbSnapGetDetails(SVnode *pVnode, SSnapshot *pSnap);
+int32_t      tsdbSnapPrepDescription(SVnode *pVnode, SSnapshot *pSnap);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -65,6 +65,8 @@ typedef struct SMetaSnapReader    SMetaSnapReader;
 typedef struct SMetaSnapWriter    SMetaSnapWriter;
 typedef struct STsdbSnapReader    STsdbSnapReader;
 typedef struct STsdbSnapWriter    STsdbSnapWriter;
+typedef struct STsdbSnapRAWReader STsdbSnapRAWReader;
+typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter;
 typedef struct STqSnapReader      STqSnapReader;
 typedef struct STqSnapWriter      STqSnapWriter;
 typedef struct STqOffsetReader    STqOffsetReader;
@@ -236,10 +238,8 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg);
 
-int32_t tqRestartStreamTasks(STQ* pTq);
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
 int32_t tqScanWal(STQ* pTq);
-int32_t tqStartStreamTasks(STQ* pTq);
 
 int     tqCommit(STQ*);
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
@@ -313,6 +313,15 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang
 int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr);
 int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter);
 int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback);
+// STsdbSnapRAWReader ========================================
+int32_t tsdbSnapRAWReaderOpen(STsdb* pTsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** ppReader);
+int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** ppReader);
+int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* pReader, uint8_t** ppData);
+// STsdbSnapRAWWriter ========================================
+int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** ppWriter);
+int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* pWriter, SSnapDataHdr* pHdr);
+int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter);
+int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback);
 // STqSnapshotReader ==
 int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader);
 int32_t tqSnapReaderClose(STqSnapReader** ppReader);
@@ -531,6 +540,7 @@ enum {
   SNAP_DATA_STREAM_STATE = 11,
   SNAP_DATA_STREAM_STATE_BACKEND = 12,
   SNAP_DATA_TQ_CHECKINFO = 13,
+  SNAP_DATA_RAW = 14,
 };
 
 struct SSnapDataHdr {

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -1877,7 +1877,7 @@ static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb
       .idstr = pr->idstr,
   };
 
-  code = tMergeTreeOpen2(&iter->mergeTree, &conf);
+  code = tMergeTreeOpen2(&iter->mergeTree, &conf, NULL);
   if (code != TSDB_CODE_SUCCESS) {
     return -1;
   }

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -435,7 +435,7 @@ _exit:
     tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode),
               __func__, committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel);
   }
-  return 0;
+  return code;
 }
 
 static int32_t tsdbCommitFileSetEnd(SCommitter2 *committer) {

source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c

@@ -0,0 +1,222 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbDataFileRAW.h"
+
+// SDataFileRAWReader =============================================
+int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config,
+                                  SDataFileRAWReader **reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  reader[0] = taosMemoryCalloc(1, sizeof(SDataFileRAWReader));
+  if (reader[0] == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  reader[0]->config[0] = config[0];
+
+  if (fname) {
+    if (fname) {
+      code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  } else {
+    char fname1[TSDB_FILENAME_LEN];
+    tsdbTFileName(config->tsdb, &config->file, fname1);
+    code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) {
+  if (reader[0] == NULL) return 0;
+
+  if (reader[0]->fd) {
+    tsdbCloseFile(&reader[0]->fd);
+  }
+
+  taosMemoryFree(reader[0]);
+  reader[0] = NULL;
+  return 0;
+}
+
+int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *pBlock) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  pBlock->file.type = reader->config->file.type;
+  pBlock->file.fid = reader->config->file.fid;
+  pBlock->file.cid = reader->config->file.cid;
+  pBlock->file.size = reader->config->file.size;
+  pBlock->file.minVer = reader->config->file.minVer;
+  pBlock->file.maxVer = reader->config->file.maxVer;
+  pBlock->file.stt->level = reader->config->file.stt->level;
+
+  code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+// SDataFileRAWWriter =============================================
+int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **ppWriter) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  SDataFileRAWWriter *writer = taosMemoryCalloc(1, sizeof(SDataFileRAWWriter));
+  if (!writer) return TSDB_CODE_OUT_OF_MEMORY;
+
+  writer->config[0] = config[0];
+
+  code = tsdbDataFileRAWWriterDoOpen(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    taosMemoryFree(writer);
+    writer = NULL;
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  ppWriter[0] = writer;
+  return code;
+}
+
+static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) {
+  ASSERT(0);
+  return 0;
+}
+
+static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; }
+
+static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) {
+  int32_t  code = 0;
+  int32_t  lino = 0;
+  ASSERT(writer->ctx->offset == writer->file.size);
+  ASSERT(writer->config->fid == writer->file.fid);
+
+  STFileOp op = (STFileOp){
+      .optype = TSDB_FOP_CREATE,
+      .fid = writer->config->fid,
+      .nf = writer->file,
+  };
+  code = TARRAY2_APPEND(opArr, op);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  if (writer->fd) {
+    code = tsdbFsyncFile(writer->fd);
+    TSDB_CHECK_CODE(code, lino, _exit);
+    tsdbCloseFile(&writer->fd);
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  char    fname[TSDB_FILENAME_LEN];
+  int32_t flag = TD_FILE_READ | TD_FILE_WRITE;
+
+  if (writer->ctx->offset == 0) {
+    flag |= (TD_FILE_CREATE | TD_FILE_TRUNC);
+  }
+
+  tsdbTFileName(writer->config->tsdb, &writer->file, fname);
+  code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  writer->file = writer->config->file;
+  writer->ctx->offset = 0;
+
+  code = tsdbDataFileRAWWriterOpenDataFD(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->opened = true;
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) {
+  if (writer[0] == NULL) return 0;
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  if (writer[0]->ctx->opened) {
+    if (abort) {
+      code = tsdbDataFileRAWWriterCloseAbort(writer[0]);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    } else {
+      code = tsdbDataFileRAWWriterCloseCommit(writer[0], opArr);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    tsdbDataFileRAWWriterDoClose(writer[0]);
+  }
+  taosMemoryFree(writer[0]);
+  writer[0] = NULL;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->offset += pDataBlock->dataLength;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}

source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tarray2.h"
+#include "tsdbDef.h"
+#include "tsdbFSet2.h"
+#include "tsdbFile2.h"
+#include "tsdbUtil2.h"
+
+#ifndef _TSDB_DATA_FILE_RAW_H
+#define _TSDB_DATA_FILE_RAW_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// STsdbDataRAWBlockHeader =======================================
+typedef struct STsdbDataRAWBlockHeader {
+  struct {
+    int32_t type;
+    int64_t fid;
+    int64_t cid;
+    int64_t size;
+    int64_t minVer;
+    int64_t maxVer;
+    union {
+      struct {
+        int32_t level;
+      } stt[1];
+    };
+  } file;
+
+  int64_t offset;
+  int64_t dataLength;
+  uint8_t data[0];
+} STsdbDataRAWBlockHeader;
+
+// SDataFileRAWReader =============================================
+typedef struct SDataFileRAWReaderConfig {
+  STsdb  *tsdb;
+  int32_t szPage;
+
+  STFile file;
+} SDataFileRAWReaderConfig;
+
+typedef struct SDataFileRAWReader {
+  SDataFileRAWReaderConfig config[1];
+
+  struct {
+    bool    opened;
+    int64_t offset;
+  } ctx[1];
+
+  STsdbFD *fd;
+} SDataFileRAWReader;
+
+typedef TARRAY2(SDataFileRAWReader *) SDataFileRAWReaderArray;
+
+int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config,
+                                  SDataFileRAWReader **reader);
+int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader);
+
+int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *bHdr);
+
+// SDataFileRAWWriter =============================================
+typedef struct SDataFileRAWWriterConfig {
+  STsdb  *tsdb;
+  int32_t szPage;
+
+  SDiskID did;
+  int64_t fid;
+  int64_t cid;
+  int32_t level;
+
+  STFile file;
+} SDataFileRAWWriterConfig;
+
+typedef struct SDataFileRAWWriter {
+  SDataFileRAWWriterConfig config[1];
+
+  struct {
+    bool    opened;
+    int64_t offset;
+  } ctx[1];
+
+  STFile   file;
+  STsdbFD *fd;
+} SDataFileRAWWriter;
+
+typedef struct SDataFileRAWWriter SDataFileRAWWriter;
+
+int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **writer);
+int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr);
+
+int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer);
+int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr);
+int32_t tsdbDataFileRAWFlush(SDataFileRAWWriter *writer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TSDB_DATA_FILE_RAW_H*/

source/dnode/vnode/src/tsdb/tsdbFS2.c

@@ -1072,7 +1072,25 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) {
   return 0;
 }
 
-int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRanges, TFileSetArray **fsetArr,
+static SHashObj *tsdbFSetRangeArrayToHash(TFileSetRangeArray *pRanges) {
+  int32_t   capacity = TARRAY2_SIZE(pRanges) * 2;
+  SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
+  if (pHash == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) {
+    STFileSetRange *u = TARRAY2_GET(pRanges, i);
+    int32_t         fid = u->fid;
+    int32_t         code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u));
+    ASSERT(code == 0);
+    tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
+  }
+  return pHash;
+}
+
+int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRanges, TFileSetArray **fsetArr,
                                        TFileOpArray *fopArr) {
   int32_t    code = 0;
   STFileSet *fset;
@@ -1084,7 +1102,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange
   TARRAY2_INIT(fsetArr[0]);
 
   if (pRanges) {
-    pHash = tsdbGetSnapRangeHash(pRanges);
+    pHash = tsdbFSetRangeArrayToHash(pRanges);
     if (pHash == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _out;
@@ -1096,7 +1114,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange
     int64_t ever = VERSION_MAX;
     if (pHash) {
       int32_t      fid = fset->fid;
-      STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid));
+      STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid));
       if (u) {
         ever = u->sver - 1;
       }
@@ -1123,29 +1141,13 @@ _out:
   return code;
 }
 
-SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) {
+int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr) { return tsdbFSDestroyCopySnapshot(fsetArr); }
-  int32_t   capacity = TARRAY2_SIZE(pRanges) * 2;
-  SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
-  if (pHash == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
 
-  for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) {
+int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges,
-    STSnapRange *u = TARRAY2_GET(pRanges, i);
+                                      TFileSetRangeArray **fsrArr) {
-    int32_t      fid = u->fid;
-    int32_t      code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u));
-    ASSERT(code == 0);
-    tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
-  }
-  return pHash;
-}
-
-int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges,
-                                      TSnapRangeArray **fsrArr) {
   int32_t      code = 0;
   STFileSet   *fset;
-  STSnapRange *fsr1 = NULL;
+  STFileSetRange *fsr1 = NULL;
   SHashObj    *pHash = NULL;
 
   fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0]));
@@ -1156,7 +1158,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
 
   tsdbInfo("pRanges size:%d", (pRanges == NULL ? 0 : TARRAY2_SIZE(pRanges)));
   if (pRanges) {
-    pHash = tsdbGetSnapRangeHash(pRanges);
+    pHash = tsdbFSetRangeArrayToHash(pRanges);
     if (pHash == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _out;
@@ -1170,7 +1172,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
 
     if (pHash) {
       int32_t      fid = fset->fid;
-      STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid));
+      STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid));
       if (u) {
         sver1 = u->sver;
         tsdbDebug("range hash get fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
@@ -1184,7 +1186,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
 
     tsdbDebug("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1);
 
-    code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1);
+    code = tsdbTFileSetRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1);
     if (code) break;
 
     code = TARRAY2_APPEND(fsrArr[0], fsr1);
@@ -1195,8 +1197,8 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
   taosThreadMutexUnlock(&fs->tsdb->mutex);
 
   if (code) {
-    tsdbTSnapRangeClear(&fsr1);
+    tsdbTFileSetRangeClear(&fsr1);
-    TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear);
+    TARRAY2_DESTROY(fsrArr[0], tsdbTFileSetRangeClear);
     fsrArr[0] = NULL;
   }
 
@@ -1207,3 +1209,5 @@ _out:
   }
   return code;
 }
+
+int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { return tsdbTFileSetRangeArrayDestroy(fsrArr); }

source/dnode/vnode/src/tsdb/tsdbFS2.h

@@ -44,12 +44,12 @@ int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr);
 int32_t tsdbFSCreateRefSnapshotWithoutLock(STFileSystem *fs, TFileSetArray **fsetArr);
 int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr);
 
-int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclude, TFileSetArray **fsetArr,
+int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pExclude, TFileSetArray **fsetArr,
                                        TFileOpArray *fopArr);
-int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr, TFileOpArray *fopArr);
+int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr);
-int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges,
+int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges,
-                                      TSnapRangeArray **fsrArr);
+                                      TFileSetRangeArray **fsrArr);
-int32_t tsdbFSDestroyRefRangedSnapshot(TSnapRangeArray **fsrArr);
+int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr);
 //  txn
 int64_t tsdbFSAllocEid(STFileSystem *fs);
 int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype);

source/dnode/vnode/src/tsdb/tsdbFSet2.c

@@ -533,7 +533,8 @@ int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_
   return 0;
 }
 
-int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr) {
+int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever,
+                                 STFileSetRange **fsr) {
   fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0]));
   if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
   fsr[0]->fid = fset1->fid;
@@ -575,7 +576,7 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs
   return 0;
 }
 
-int32_t tsdbTSnapRangeClear(STSnapRange **fsr) {
+int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr) {
   if (!fsr[0]) return 0;
 
   tsdbTFileSetClear(&fsr[0]->fset);
@@ -584,6 +585,15 @@ int32_t tsdbTSnapRangeClear(STSnapRange **fsr) {
   return 0;
 }
 
+int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray** ppArr) {
+  if (ppArr && ppArr[0]) {
+    TARRAY2_DESTROY(ppArr[0], tsdbTFileSetRangeClear);
+    taosMemoryFree(ppArr[0]);
+    ppArr[0] = NULL;
+  }
+  return 0;
+}
+
 int32_t tsdbTFileSetClear(STFileSet **fset) {
   if (!fset[0]) return 0;
 

source/dnode/vnode/src/tsdb/tsdbFSet2.h

@@ -49,8 +49,8 @@ int32_t tsdbTFileSetRemove(STFileSet *fset);
 int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset,
                                     TFileOpArray *fopArr);
 
-int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr);
+int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever,
-int32_t tsdbTSnapRangeClear(STSnapRange **fsr);
+                                 STFileSetRange **fsr);
 
 // to/from json
 int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json);
@@ -101,7 +101,7 @@ struct STFileSet {
   bool         blockCommit;
 };
 
-struct STSnapRange {
+struct STFileSetRange {
   int32_t    fid;
   int64_t    sver;
   int64_t    ever;

source/dnode/vnode/src/tsdb/tsdbFSetRAW.c

@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbFSetRAW.h"
+
+// SFSetRAWWriter ==================================================
+typedef struct SFSetRAWWriter {
+  SFSetRAWWriterConfig config[1];
+
+  struct {
+    TFileOpArray fopArr[1];
+    STFile       file;
+    int64_t      offset;
+  } ctx[1];
+
+  // writer
+  SDataFileRAWWriter *dataWriter;
+} SFSetRAWWriter;
+
+int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter));
+  if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+
+  writer[0]->config[0] = config[0];
+
+  TARRAY2_INIT(writer[0]->ctx->fopArr);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbFSetRAWWriterFinish(SFSetRAWWriter *writer, TFileOpArray *fopArr) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STsdb *tsdb = writer->config->tsdb;
+
+  STFileOp op;
+  TARRAY2_FOREACH(writer->ctx->fopArr, op) {
+    code = TARRAY2_APPEND(fopArr, op);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  TARRAY2_CLEAR(writer->ctx->fopArr, NULL);
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  SDataFileRAWWriterConfig config = {
+      .tsdb = writer->config->tsdb,
+      .szPage = writer->config->szPage,
+      .fid = bHdr->file.fid,
+      .did = writer->config->did,
+      .cid = writer->config->cid,
+      .level = writer->config->level,
+
+      .file =
+          {
+              .type = bHdr->file.type,
+              .fid = bHdr->file.fid,
+              .did = writer->config->did,
+              .cid = writer->config->cid,
+              .size = bHdr->file.size,
+              .minVer = bHdr->file.minVer,
+              .maxVer = bHdr->file.maxVer,
+              .stt = {{
+                  .level = bHdr->file.stt->level,
+              }},
+          },
+  };
+
+  writer->ctx->offset = 0;
+  writer->ctx->file = config.file;
+
+  code = tsdbDataFileRAWWriterOpen(&config, &writer->dataWriter);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbFSetRAWWriteFileDataEnd(SFSetRAWWriter *writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbDataFileRAWWriterClose(&writer->dataWriter, false, writer->ctx->fopArr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr) {
+  if (writer[0] == NULL) return 0;
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STsdb *tsdb = writer[0]->config->tsdb;
+
+  // end
+  code = tsdbFSetRAWWriteFileDataEnd(writer[0]);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  code = tsdbDataFileRAWWriterClose(&writer[0]->dataWriter, abort, writer[0]->ctx->fopArr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  code = tsdbFSetRAWWriterFinish(writer[0], fopArr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+  // free
+  TARRAY2_DESTROY(writer[0]->ctx->fopArr, NULL);
+  taosMemoryFree(writer[0]);
+  writer[0] = NULL;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  ASSERT(writer->ctx->offset >= 0 && writer->ctx->offset <= writer->ctx->file.size);
+
+  if (writer->ctx->offset == writer->ctx->file.size) {
+    code = tsdbFSetRAWWriteFileDataEnd(writer);
+    TSDB_CHECK_CODE(code, lino, _exit);
+
+    code = tsdbFSetRAWWriteFileDataBegin(writer, bHdr);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->offset += bHdr->dataLength;
+  ASSERT(writer->ctx->offset == writer->dataWriter->ctx->offset);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}

source/dnode/vnode/src/tsdb/tsdbFSetRAW.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbDataFileRAW.h"
+
+#ifndef _TSDB_FSET_RAW_H
+#define _TSDB_FSET_RAW_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SFSetRAWWriterConfig {
+  STsdb  *tsdb;
+  int32_t szPage;
+
+  SDiskID did;
+  int64_t fid;
+  int64_t cid;
+  int32_t level;
+} SFSetRAWWriterConfig;
+
+typedef struct SFSetRAWWriter SFSetRAWWriter;
+
+int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer);
+int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr);
+int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TSDB_FSET_RAW_H*/

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -15,6 +15,7 @@
 
 #include "tsdb.h"
 #include "tsdbFSet2.h"
+#include "tsdbUtil2.h"
 #include "tsdbMerge.h"
 #include "tsdbReadUtil.h"
 #include "tsdbSttFileRW.h"
@@ -52,15 +53,6 @@ SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList,
   return pLoadInfo;
 }
 
-void getSttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, SSttBlockLoadCostInfo* pLoadCost) {
-  for (int32_t i = 0; i < 1; ++i) {
-    pLoadCost->blockElapsedTime += pLoadInfo[i].cost.blockElapsedTime;
-    pLoadCost->loadBlocks += pLoadInfo[i].cost.loadBlocks;
-    pLoadCost->loadStatisBlocks += pLoadInfo[i].cost.loadStatisBlocks;
-    pLoadCost->statisElapsedTime += pLoadInfo[i].cost.statisElapsedTime;
-  }
-}
-
 void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
   if (pLoadInfo == NULL) {
     return NULL;
@@ -78,9 +70,11 @@ void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
   pInfo->sttBlockIndex = -1;
   pInfo->pin = false;
 
-  if (pLoadInfo->statisBlock != NULL) {
+  if (pLoadInfo->info.pCount != NULL) {
-    tStatisBlockDestroy(pLoadInfo->statisBlock);
+    taosArrayDestroy(pLoadInfo->info.pUid);
-    taosMemoryFreeClear(pLoadInfo->statisBlock);
+    taosArrayDestroy(pLoadInfo->info.pFirstKey);
+    taosArrayDestroy(pLoadInfo->info.pLastKey);
+    taosArrayDestroy(pLoadInfo->info.pCount);
   }
 
   taosArrayDestroy(pLoadInfo->aSttBlk);
@@ -172,7 +166,7 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) {
   pInfo->cost.blockElapsedTime += el;
   pInfo->cost.loadBlocks += 1;
 
-  tsdbDebug("read last block, total load:%" PRId64 ", trigger by uid:%" PRIu64 ", stt-fileVer:%" PRId64
+  tsdbDebug("read stt block, total load:%" PRId64 ", trigger by uid:%" PRIu64 ", stt-fileVer:%" PRId64
             ", last block index:%d, entry:%d, rows:%d, uidRange:%" PRId64 "-%" PRId64 " tsRange:%" PRId64 "-%" PRId64
             " %p, elapsed time:%.2f ms, %s",
             pInfo->cost.loadBlocks, pIter->uid, pIter->cid, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock->nRow,
@@ -323,95 +317,77 @@ static int32_t extractSttBlockInfo(SLDataIter *pIter, const TSttBlkArray *pArray
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t suidComparFn(const void *target, const void *p2) {
+static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBlockLoadInfo *pBlockLoadInfo,
-  const uint64_t *targetUid = target;
+                                          TStatisBlkArray *pStatisBlkArray, uint64_t suid, const char *id) {
-  const uint64_t *uid2 = p2;
+  int32_t numOfBlocks = TARRAY2_SIZE(pStatisBlkArray);
-  if (*uid2 == (*targetUid)) {
+  if (numOfBlocks <= 0) {
     return 0;
-  } else {
-    return (*targetUid) < (*uid2) ? -1 : 1;
   }
-}
 
-static bool existsFromSttBlkStatis(SSttBlockLoadInfo *pBlockLoadInfo, uint64_t suid, uint64_t uid,
+  int32_t startIndex = 0;
-                                   SSttFileReader *pReader) {
+  while((startIndex < numOfBlocks) && (pStatisBlkArray->data[startIndex].maxTbid.suid < suid)) {
-  const TStatisBlkArray *pStatisBlkArray = pBlockLoadInfo->pSttStatisBlkArray;
+    ++startIndex;
-  if (TARRAY2_SIZE(pStatisBlkArray) <= 0) {
-    return true;
   }
 
+  if (startIndex >= numOfBlocks || pStatisBlkArray->data[startIndex].minTbid.suid > suid) {
+    return 0;
+  }
+
+  int32_t endIndex = startIndex;
+  while(endIndex < numOfBlocks && pStatisBlkArray->data[endIndex].minTbid.suid <= suid) {
+    ++endIndex;
+  }
+
+  int32_t num = endIndex - startIndex;
+  pBlockLoadInfo->cost.loadStatisBlocks += num;
+
+  STbStatisBlock block;
+  tStatisBlockInit(&block);
+
+  int64_t st = taosGetTimestampUs();
+
+  for(int32_t k = startIndex; k < endIndex; ++k) {
+    tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[k], &block);
+
     int32_t i = 0;
-  for (i = 0; i < TARRAY2_SIZE(pStatisBlkArray); ++i) {
+    int32_t rows = TARRAY2_SIZE(block.suid);
-    SStatisBlk *p = &pStatisBlkArray->data[i];
+    while (i < rows && block.suid->data[i] != suid) {
-    if (p->minTbid.suid <= suid && p->maxTbid.suid >= suid) {
+      ++i;
-      break;
-    }
     }
 
-  if (i >= TARRAY2_SIZE(pStatisBlkArray)) {
+    // existed
-    return false;
+    if (i < rows) {
+      if (pBlockLoadInfo->info.pUid == NULL) {
+        pBlockLoadInfo->info.pUid = taosArrayInit(rows, sizeof(int64_t));
+        pBlockLoadInfo->info.pFirstKey = taosArrayInit(rows, sizeof(int64_t));
+        pBlockLoadInfo->info.pLastKey = taosArrayInit(rows, sizeof(int64_t));
+        pBlockLoadInfo->info.pCount = taosArrayInit(rows, sizeof(int64_t));
       }
 
-  while (i < TARRAY2_SIZE(pStatisBlkArray)) {
+      if (pStatisBlkArray->data[k].maxTbid.suid == suid) {
-    SStatisBlk *p = &pStatisBlkArray->data[i];
+        taosArrayAddBatch(pBlockLoadInfo->info.pUid, &block.uid->data[i], rows - i);
-    if (p->minTbid.suid > suid) {
+        taosArrayAddBatch(pBlockLoadInfo->info.pFirstKey, &block.firstKey->data[i], rows - i);
-      return false;
+        taosArrayAddBatch(pBlockLoadInfo->info.pLastKey, &block.lastKey->data[i], rows - i);
-    }
+        taosArrayAddBatch(pBlockLoadInfo->info.pCount, &block.count->data[i], rows - i);
-
-//    if (pBlockLoadInfo->statisBlock == NULL) {
-//      pBlockLoadInfo->statisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock));
-//
-//      int64_t st = taosGetTimestampMs();
-//      tsdbSttFileReadStatisBlock(pReader, p, pBlockLoadInfo->statisBlock);
-//      pBlockLoadInfo->statisBlockIndex = i;
-//
-//      double el = (taosGetTimestampMs() - st) / 1000.0;
-//      pBlockLoadInfo->cost.loadStatisBlocks += 1;
-//      pBlockLoadInfo->cost.statisElapsedTime += el;
-//    } else if (pBlockLoadInfo->statisBlockIndex != i) {
-//      tStatisBlockDestroy(pBlockLoadInfo->statisBlock);
-//
-//      int64_t st = taosGetTimestampMs();
-//      tsdbSttFileReadStatisBlock(pReader, p, pBlockLoadInfo->statisBlock);
-//      pBlockLoadInfo->statisBlockIndex = i;
-//
-//      double el = (taosGetTimestampMs() - st) / 1000.0;
-//      pBlockLoadInfo->cost.loadStatisBlocks += 1;
-//      pBlockLoadInfo->cost.statisElapsedTime += el;
-//    }
-
-    STbStatisBlock* pBlock = pBlockLoadInfo->statisBlock;
-    int32_t index = tarray2SearchIdx(pBlock->suid, &suid, sizeof(int64_t), suidComparFn, TD_EQ);
-    if (index == -1) {
-      return false;
-    }
-
-    int32_t j = index;
-    if (pBlock->uid->data[j] == uid) {
-      return true;
-    } else if (pBlock->uid->data[j] > uid) {
-      while (j >= 0 && pBlock->suid->data[j] == suid) {
-        if (pBlock->uid->data[j] == uid) {
-          return true;
       } else {
-          j -= 1;
+        while (i < rows && block.suid->data[i] == suid) {
-        }
+          taosArrayPush(pBlockLoadInfo->info.pUid, &block.uid->data[i]);
-      }
+          taosArrayPush(pBlockLoadInfo->info.pFirstKey, &block.firstKey->data[i]);
-    } else {
+          taosArrayPush(pBlockLoadInfo->info.pLastKey, &block.lastKey->data[i]);
-      j = index + 1;
+          taosArrayPush(pBlockLoadInfo->info.pCount, &block.count->data[i]);
-      while (j < pBlock->suid->size && pBlock->suid->data[j] == suid) {
-        if (pBlock->uid->data[j] == uid) {
-          return true;
-        } else {
-          j += 1;
-        }
-      }
-    }
-
           i += 1;
         }
+      }
+    }
+  }
 
-  return false;
+  tStatisBlockDestroy(&block);
+
+  double el = (taosGetTimestampUs() - st) / 1000.0;
+  pBlockLoadInfo->cost.statisElapsedTime += el;
+
+  tsdbDebug("%s load %d statis blocks into buf, elapsed time:%.2fms", id, num, el);
+  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t doLoadSttFilesBlk(SSttBlockLoadInfo *pBlockLoadInfo, SLDataIter *pIter, int64_t suid,
@@ -428,19 +404,28 @@ static int32_t doLoadSttFilesBlk(SSttBlockLoadInfo *pBlockLoadInfo, SLDataIter *
     return code;
   }
 
+  // load the stt block info for each stt file block
   code = extractSttBlockInfo(pIter, pSttBlkArray, pBlockLoadInfo, suid);
   if (code != TSDB_CODE_SUCCESS) {
     tsdbError("load stt block info failed, code:%s, %s", tstrerror(code), idStr);
     return code;
   }
 
-  // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file
+  // load stt statistics block for all stt-blocks, to decide if the data of queried table exists in current stt file
-  code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pBlockLoadInfo->pSttStatisBlkArray);
+  TStatisBlkArray *pStatisBlkArray = NULL;
+  code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pStatisBlkArray);
   if (code != TSDB_CODE_SUCCESS) {
     tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), idStr);
     return code;
   }
 
+  // load statistics block for all tables in current stt file
+  code = loadSttStatisticsBlockData(pIter->pReader, pIter->pBlockLoadInfo, pStatisBlkArray, suid, idStr);
+  if (code != TSDB_CODE_SUCCESS) {
+    tsdbError("failed to load stt statistics block data, code:%s, %s", tstrerror(code), idStr);
+    return code;
+  }
+
   code = loadTombFn(pReader1, pIter->pReader, pIter->pBlockLoadInfo);
 
   double el = (taosGetTimestampUs() - st) / 1000.0;
@@ -448,19 +433,44 @@ static int32_t doLoadSttFilesBlk(SSttBlockLoadInfo *pBlockLoadInfo, SLDataIter *
   return code;
 }
 
+static int32_t uidComparFn(const void* p1, const void* p2) {
+  const uint64_t *pFirst = p1;
+  const uint64_t *pVal = p2;
+
+  if (*pFirst == *pVal) {
+    return 0;
+  } else {
+    return *pFirst < *pVal? -1:1;
+  }
+}
+
+static void setSttInfoForCurrentTable(SSttBlockLoadInfo *pLoadInfo, uint64_t uid, STimeWindow *pTimeWindow,
+                                      int64_t *numOfRows) {
+  if (pTimeWindow == NULL || taosArrayGetSize(pLoadInfo->info.pUid) == 0) {
+    return;
+  }
+
+  int32_t index = taosArraySearchIdx(pLoadInfo->info.pUid, &uid, uidComparFn, TD_EQ);
+  if (index >= 0) {
+    pTimeWindow->skey = *(int64_t *)taosArrayGet(pLoadInfo->info.pFirstKey, index);
+    pTimeWindow->ekey = *(int64_t *)taosArrayGet(pLoadInfo->info.pLastKey, index);
+
+    *numOfRows += *(int64_t*) taosArrayGet(pLoadInfo->info.pCount, index);
+  }
+}
+
 int32_t tLDataIterOpen2(SLDataIter *pIter, SSttFileReader *pSttFileReader, int32_t cid, int8_t backward,
-                        uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange,
+                        SMergeTreeConf *pConf, SSttBlockLoadInfo *pBlockLoadInfo, STimeWindow *pTimeWindow,
-                        SSttBlockLoadInfo *pBlockLoadInfo, const char *idStr, bool strictTimeRange,
+                        int64_t *numOfRows, const char *idStr) {
-                        _load_tomb_fn loadTombFn, void *pReader1) {
   int32_t code = TSDB_CODE_SUCCESS;
 
-  pIter->uid = uid;
+  pIter->uid = pConf->uid;
   pIter->cid = cid;
   pIter->backward = backward;
-  pIter->verRange.minVer = pRange->minVer;
+  pIter->verRange.minVer = pConf->verRange.minVer;
-  pIter->verRange.maxVer = pRange->maxVer;
+  pIter->verRange.maxVer = pConf->verRange.maxVer;
-  pIter->timeWindow.skey = pTimeWindow->skey;
+  pIter->timeWindow.skey = pConf->timewindow.skey;
-  pIter->timeWindow.ekey = pTimeWindow->ekey;
+  pIter->timeWindow.ekey = pConf->timewindow.ekey;
   pIter->pReader = pSttFileReader;
   pIter->pBlockLoadInfo = pBlockLoadInfo;
 
@@ -473,34 +483,29 @@ int32_t tLDataIterOpen2(SLDataIter *pIter, SSttFileReader *pSttFileReader, int32
   }
 
   if (!pBlockLoadInfo->sttBlockLoaded) {
-    code = doLoadSttFilesBlk(pBlockLoadInfo, pIter, suid, loadTombFn, pReader1, idStr);
+    code = doLoadSttFilesBlk(pBlockLoadInfo, pIter, pConf->suid, pConf->loadTombFn, pConf->pReader, idStr);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
   }
 
-//  bool exists = existsFromSttBlkStatis(pBlockLoadInfo, suid, uid, pIter->pReader);
+  setSttInfoForCurrentTable(pBlockLoadInfo, pConf->uid, pTimeWindow, numOfRows);
-//  if (!exists) {
-//    pIter->iSttBlk = -1;
-//    pIter->pSttBlk = NULL;
-//    return TSDB_CODE_SUCCESS;
-//   }
 
   // find the start block, actually we could load the position to avoid repeatly searching for the start position when
   // the skey is updated.
   size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
-  pIter->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward);
+  pIter->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, pConf->uid, backward);
   if (pIter->iSttBlk != -1) {
     pIter->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, pIter->iSttBlk);
     pIter->iRow = (pIter->backward) ? pIter->pSttBlk->nRow : -1;
 
-    if ((!backward) && ((strictTimeRange && pIter->pSttBlk->minKey >= pIter->timeWindow.ekey) ||
+    if ((!backward) && ((pConf->strictTimeRange && pIter->pSttBlk->minKey >= pIter->timeWindow.ekey) ||
-                        (!strictTimeRange && pIter->pSttBlk->minKey > pIter->timeWindow.ekey))) {
+                        (!pConf->strictTimeRange && pIter->pSttBlk->minKey > pIter->timeWindow.ekey))) {
       pIter->pSttBlk = NULL;
     }
 
-    if (backward && ((strictTimeRange && pIter->pSttBlk->maxKey <= pIter->timeWindow.skey) ||
+    if (backward && ((pConf->strictTimeRange && pIter->pSttBlk->maxKey <= pIter->timeWindow.skey) ||
-                     (!strictTimeRange && pIter->pSttBlk->maxKey < pIter->timeWindow.skey))) {
+                     (!pConf->strictTimeRange && pIter->pSttBlk->maxKey < pIter->timeWindow.skey))) {
       pIter->pSttBlk = NULL;
       pIter->ignoreEarlierTs = true;
     }
@@ -708,8 +713,6 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) {
   return (terrno == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL) && (pBlockData != NULL);
 }
 
-SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; }
-
 // SMergeTree =================================================
 static FORCE_INLINE int32_t tLDataIterCmprFn(const SRBTreeNode *p1, const SRBTreeNode *p2) {
   SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node));
@@ -737,7 +740,7 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR
   return -1 * tLDataIterCmprFn(p1, p2);
 }
 
-int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) {
+int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoForTable* pSttDataInfo) {
   int32_t code = TSDB_CODE_SUCCESS;
 
   pMTree->pIter = NULL;
@@ -758,17 +761,16 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) {
     goto _end;
   }
 
-  // add the list/iter placeholder
+  adjustSttDataIters(pConf->pSttFileBlockIterArray, pConf->pCurrentFileset);
-  adjustLDataIters(pConf->pSttFileBlockIterArray, pConf->pCurrentFileset);
 
   for (int32_t j = 0; j < numOfLevels; ++j) {
     SSttLvl *pSttLevel = ((STFileSet *)pConf->pCurrentFileset)->lvlArr->data[j];
-    SArray  *pList = taosArrayGetP(pConf->pSttFileBlockIterArray, j);
+    SArray * pList = taosArrayGetP(pConf->pSttFileBlockIterArray, j);
 
     for (int32_t i = 0; i < TARRAY2_SIZE(pSttLevel->fobjArr); ++i) {  // open all last file
       SLDataIter *pIter = taosArrayGetP(pList, i);
 
-      SSttFileReader    *pSttFileReader = pIter->pReader;
+      SSttFileReader *   pSttFileReader = pIter->pReader;
       SSttBlockLoadInfo *pLoadInfo = pIter->pBlockLoadInfo;
 
       // open stt file reader if not opened yet
@@ -790,10 +792,11 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) {
 
       memset(pIter, 0, sizeof(SLDataIter));
 
+      STimeWindow w = {0};
+      int64_t     numOfRows = 0;
+
       int64_t cid = pSttLevel->fobjArr->data[i]->f->cid;
-      code = tLDataIterOpen2(pIter, pSttFileReader, cid, pMTree->backward, pConf->suid, pConf->uid, &pConf->timewindow,
+      code = tLDataIterOpen2(pIter, pSttFileReader, cid, pMTree->backward, pConf, pLoadInfo, &w, &numOfRows, pMTree->idStr);
-                             &pConf->verRange, pLoadInfo, pMTree->idStr, pConf->strictTimeRange, pConf->loadTombFn,
-                             pConf->pReader);
       if (code != TSDB_CODE_SUCCESS) {
         goto _end;
       }
@@ -801,6 +804,12 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf) {
       bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr);
       if (hasVal) {
         tMergeTreeAddIter(pMTree, pIter);
+
+        // let's record the time window for current table of uid in the stt files
+        if (pSttDataInfo != NULL) {
+          taosArrayPush(pSttDataInfo->pTimeWindowList, &w);
+          pSttDataInfo->numOfRows += numOfRows;
+        }
       } else {
         if (!pMTree->ignoreEarlierTs) {
           pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs;

source/dnode/vnode/src/tsdb/tsdbRead2.c

@@ -25,6 +25,16 @@
 #define ASCENDING_TRAVERSE(o)        (o == TSDB_ORDER_ASC)
 #define getCurrentKeyInSttBlock(_r)  ((_r)->currentKey)
 
+typedef struct {
+  bool overlapWithNeighborBlock;
+  bool hasDupTs;
+  bool overlapWithDelInfo;
+  bool overlapWithSttBlock;
+  bool overlapWithKeyInBuf;
+  bool partiallyRequired;
+  bool moreThanCapcity;
+} SDataBlockToLoadInfo;
+
 static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter);
 static int32_t  buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity,
                                           STsdbReader* pReader);
@@ -596,6 +606,13 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
       return TSDB_CODE_OUT_OF_MEMORY;
     }
 
+    if (pScanInfo->filesetWindow.skey > pRecord->firstKey) {
+      pScanInfo->filesetWindow.skey = pRecord->firstKey;
+    }
+    if (pScanInfo->filesetWindow.ekey < pRecord->lastKey) {
+      pScanInfo->filesetWindow.ekey = pRecord->lastKey;
+    }
+
     pBlockNum->numOfBlocks += 1;
     if (taosArrayGetSize(pTableScanInfoList) == 0) {
       taosArrayPush(pTableScanInfoList, &pScanInfo);
@@ -614,7 +631,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
 
   double el = (taosGetTimestampUs() - st) / 1000.0;
   tsdbDebug(
-      "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed "
+      "load block of %d tables completed, blocks:%d in %d tables, stt-files:%d, block-info-size:%.2f Kb, elapsed "
       "time:%.2f ms %s",
       numOfTables, pBlockNum->numOfBlocks, (int32_t)taosArrayGetSize(pTableScanInfoList), pBlockNum->numOfSttFiles,
       sizeInDisk / 1000.0, el, pReader->idStr);
@@ -1226,78 +1243,6 @@ static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersio
          (pBlock->record.maxVer >= pVerRange->minVer) && (pBlock->record.minVer <= pVerRange->maxVer);
 }
 
-static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,
-                                       int32_t startIndex) {
-  size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);
-
-  for (int32_t i = startIndex; i < num; i += 1) {
-    TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);
-    if (p->ts >= pRecord->firstKey && p->ts <= pRecord->lastKey) {
-      if (p->version >= pRecord->minVer) {
-        return true;
-      }
-    } else if (p->ts < pRecord->firstKey) {  // p->ts < pBlock->minKey.ts
-      if (p->version >= pRecord->minVer) {
-        if (i < num - 1) {
-          TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);
-          if (pnext->ts >= pRecord->firstKey) {
-            return true;
-          }
-        } else {  // it must be the last point
-          ASSERT(p->version == 0);
-        }
-      }
-    } else {  // (p->ts > pBlock->maxKey.ts) {
-      return false;
-    }
-  }
-
-  return false;
-}
-
-static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) {
-  if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) {
-    return false;
-  }
-
-  // ts is not overlap
-  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);
-  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);
-  if (pRecord->firstKey > pLast->ts || pRecord->lastKey < pFirst->ts) {
-    return false;
-  }
-
-  // version is not overlap
-  if (ASCENDING_TRAVERSE(order)) {
-    return doCheckforDatablockOverlap(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex);
-  } else {
-    int32_t index = pBlockScanInfo->fileDelIndex;
-    while (1) {
-      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);
-      if (p->ts > pRecord->firstKey && index > 0) {
-        index -= 1;
-      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.
-        if (p->ts == pRecord->firstKey && p->version < pRecord->maxVer && index > 0) {
-          index -= 1;
-        }
-        break;
-      }
-    }
-
-    return doCheckforDatablockOverlap(pBlockScanInfo, pRecord, index);
-  }
-}
-
-typedef struct {
-  bool overlapWithNeighborBlock;
-  bool hasDupTs;
-  bool overlapWithDelInfo;
-  bool overlapWithLastBlock;
-  bool overlapWithKeyInBuf;
-  bool partiallyRequired;
-  bool moreThanCapcity;
-} SDataBlockToLoadInfo;
-
 static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo,
                                STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, STsdbReader* pReader) {
   SBrinRecord rec = {0};
@@ -1318,7 +1263,7 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo*
   ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT);
   if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) {
     int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey;
-    pInfo->overlapWithLastBlock =
+    pInfo->overlapWithSttBlock =
         !(pBlockInfo->record.lastKey < nextProcKeyInStt || pBlockInfo->record.firstKey > nextProcKeyInStt);
   }
 
@@ -1340,15 +1285,15 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlock
 
   bool loadDataBlock =
       (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf ||
-       info.moreThanCapcity || info.overlapWithDelInfo || info.overlapWithLastBlock);
+       info.moreThanCapcity || info.overlapWithDelInfo || info.overlapWithSttBlock);
 
   // log the reason why load the datablock for profile
   if (loadDataBlock) {
     tsdbDebug("%p uid:%" PRIu64
               " need to load the datablock, overlapneighbor:%d, hasDup:%d, partiallyRequired:%d, "
-              "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s",
+              "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithSttBlock:%d, %s",
               pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired,
-              info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithLastBlock,
+              info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithSttBlock,
               pReader->idStr);
   }
 
@@ -1360,7 +1305,7 @@ static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBloc
   SDataBlockToLoadInfo info = {0};
   getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader);
   bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf ||
-                            info.overlapWithDelInfo || info.overlapWithLastBlock);
+                            info.overlapWithDelInfo || info.overlapWithSttBlock);
   return isCleanFileBlock;
 }
 
@@ -2115,27 +2060,34 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum
   return true;
 }
 
-static bool initSttBlockReader(SSttBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
+static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
-  // the last block reader has been initialized for this table.
+  bool hasData = true;
-  if (pLBlockReader->uid == pScanInfo->uid) {
+
-    return hasDataInSttBlock(pLBlockReader);
+  // the stt block reader has been initialized for this table.
+  if (pSttBlockReader->uid == pScanInfo->uid) {
+    return hasDataInSttBlock(pSttBlockReader);
   }
 
-  if (pLBlockReader->uid != 0) {
+  if (pSttBlockReader->uid != 0) {
-    tMergeTreeClose(&pLBlockReader->mergeTree);
+    tMergeTreeClose(&pSttBlockReader->mergeTree);
   }
 
-  pLBlockReader->uid = pScanInfo->uid;
+  pSttBlockReader->uid = pScanInfo->uid;
 
-  STimeWindow w = pLBlockReader->window;
+  // second time init stt block reader
-  if (ASCENDING_TRAVERSE(pLBlockReader->order)) {
+  if (pScanInfo->cleanSttBlocks && pReader->info.execMode == READER_EXEC_ROWS) {
+    return !pScanInfo->sttBlockReturned;
+  }
+
+  STimeWindow w = pSttBlockReader->window;
+  if (ASCENDING_TRAVERSE(pSttBlockReader->order)) {
     w.skey = pScanInfo->sttKeyInfo.nextProcKey;
   } else {
     w.ekey = pScanInfo->sttKeyInfo.nextProcKey;
   }
 
   int64_t st = taosGetTimestampUs();
-  tsdbDebug("init last block reader, window:%" PRId64 "-%" PRId64 ", uid:%" PRIu64 ", %s", w.skey, w.ekey,
+  tsdbDebug("init stt block reader, window:%" PRId64 "-%" PRId64 ", uid:%" PRIu64 ", %s", w.skey, w.ekey,
             pScanInfo->uid, pReader->idStr);
 
   SMergeTreeConf conf = {
@@ -2143,20 +2095,22 @@ static bool initSttBlockReader(SSttBlockReader* pLBlockReader, STableBlockScanIn
       .suid = pReader->info.suid,
       .pTsdb = pReader->pTsdb,
       .timewindow = w,
-      .verRange = pLBlockReader->verRange,
+      .verRange = pSttBlockReader->verRange,
       .strictTimeRange = false,
       .pSchema = pReader->info.pSchema,
       .pCurrentFileset = pReader->status.pCurrentFileset,
-      .backward = (pLBlockReader->order == TSDB_ORDER_DESC),
+      .backward = (pSttBlockReader->order == TSDB_ORDER_DESC),
       .pSttFileBlockIterArray = pReader->status.pLDataIterArray,
       .pCols = pReader->suppInfo.colId,
       .numOfCols = pReader->suppInfo.numOfCols,
       .loadTombFn = loadSttTombDataForAll,
       .pReader = pReader,
       .idstr = pReader->idStr,
+      .rspRows = (pReader->info.execMode == READER_EXEC_ROWS),
   };
 
-  int32_t code = tMergeTreeOpen2(&pLBlockReader->mergeTree, &conf);
+  SSttDataInfoForTable info = {.pTimeWindowList = taosArrayInit(4, sizeof(STimeWindow))};
+  int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info);
   if (code != TSDB_CODE_SUCCESS) {
     return false;
   }
@@ -2164,13 +2118,44 @@ static bool initSttBlockReader(SSttBlockReader* pLBlockReader, STableBlockScanIn
   initMemDataIterator(pScanInfo, pReader);
   initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost);
 
-  code = nextRowFromSttBlocks(pLBlockReader, pScanInfo, &pReader->info.verRange);
+  if (conf.rspRows) {
+    pScanInfo->cleanSttBlocks =
+        isCleanSttBlock(info.pTimeWindowList, &pReader->info.window, pScanInfo, pReader->info.order);
+
+    if (pScanInfo->cleanSttBlocks) {
+      pScanInfo->numOfRowsInStt = info.numOfRows;
+      pScanInfo->sttWindow.skey = INT64_MAX;
+      pScanInfo->sttWindow.ekey = INT64_MIN;
+
+      // calculate the time window for data in stt files
+      for(int32_t i = 0; i < taosArrayGetSize(info.pTimeWindowList); ++i) {
+        STimeWindow* pWindow = taosArrayGet(info.pTimeWindowList, i);
+        if (pScanInfo->sttWindow.skey > pWindow->skey) {
+          pScanInfo->sttWindow.skey = pWindow->skey;
+        }
+
+        if (pScanInfo->sttWindow.ekey < pWindow->ekey) {
+          pScanInfo->sttWindow.ekey = pWindow->ekey;
+        }
+      }
+
+      pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pTimeWindowList)? STT_FILE_HAS_DATA:STT_FILE_NO_DATA;
+      pScanInfo->sttKeyInfo.nextProcKey = ASCENDING_TRAVERSE(pReader->info.order)? pScanInfo->sttWindow.skey:pScanInfo->sttWindow.ekey;
+      hasData = true;
+    } else {
+      hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange);
+    }
+  } else {
+    hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange);
+  }
+
+  taosArrayDestroy(info.pTimeWindowList);
 
   int64_t el = taosGetTimestampUs() - st;
   pReader->cost.initSttBlockReader += (el / 1000.0);
 
-  tsdbDebug("init last block reader completed, elapsed time:%" PRId64 "us %s", el, pReader->idStr);
+  tsdbDebug("init stt block reader completed, elapsed time:%" PRId64 "us %s", el, pReader->idStr);
-  return code;
+  return hasData;
 }
 
 static bool hasDataInSttBlock(SSttBlockReader* pSttBlockReader) { return pSttBlockReader->mergeTree.pIter != NULL; }
@@ -2361,18 +2346,15 @@ void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInf
 
 static int32_t buildComposedDataBlock(STsdbReader* pReader) {
   int32_t code = TSDB_CODE_SUCCESS;
-
-  SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
-
-  SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
-  SSttBlockReader*   pSttBlockReader = pReader->status.fileIter.pSttBlockReader;
-
   bool    asc = ASCENDING_TRAVERSE(pReader->info.order);
   int64_t st = taosGetTimestampUs();
   int32_t step = asc ? 1 : -1;
   double  el = 0;
-  SBrinRecord* pRecord = &pBlockInfo->record;
 
+  SSDataBlock*        pResBlock = pReader->resBlockInfo.pResBlock;
+  SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
+  SSttBlockReader*    pSttBlockReader = pReader->status.fileIter.pSttBlockReader;
+  SBrinRecord*        pRecord = &pBlockInfo->record;
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
 
   STableBlockScanInfo* pBlockScanInfo = NULL;
@@ -2711,8 +2693,8 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) {
       continue;
     }
 
-    bool hasDataInLastFile = initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
+    bool hasDataInSttFile = initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
-    if (!hasDataInLastFile) {
+    if (!hasDataInSttFile) {
       bool hasNexTable = moveToNextTable(pUidList, pStatus);
       if (!hasNexTable) {
         return TSDB_CODE_SUCCESS;
@@ -2721,12 +2703,37 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) {
       continue;
     }
 
+    // if only require the total rows, no need to load data from stt file if it is clean stt blocks
+    if (pReader->info.execMode == READER_EXEC_ROWS && pScanInfo->cleanSttBlocks) {
+      bool asc = ASCENDING_TRAVERSE(pReader->info.order);
+
+      SDataBlockInfo* pInfo = &pResBlock->info;
+      blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt);
+
+      pInfo->rows = pScanInfo->numOfRowsInStt;
+      pInfo->id.uid = pScanInfo->uid;
+      pInfo->dataLoad = 1;
+      pInfo->window = pScanInfo->sttWindow;
+
+      setComposedBlockFlag(pReader, true);
+
+      pScanInfo->sttKeyInfo.nextProcKey = asc ? pScanInfo->sttWindow.ekey + 1 : pScanInfo->sttWindow.skey - 1;
+      pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;
+      pScanInfo->lastProcKey = asc ? pScanInfo->sttWindow.ekey : pScanInfo->sttWindow.skey;
+      pScanInfo->sttBlockReturned = true;
+
+      pSttBlockReader->mergeTree.pIter = NULL;
+
+      tsdbDebug("%p uid:%" PRId64 " return clean stt block as one, brange:%" PRId64 "-%" PRId64 " rows:%" PRId64 " %s",
+                pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
+                pResBlock->info.rows, pReader->idStr);
+      return TSDB_CODE_SUCCESS;
+    }
+
     int64_t st = taosGetTimestampUs();
     while (1) {
-      bool hasBlockLData = hasDataInSttBlock(pSttBlockReader);
+      // no data in stt block and block, no need to proceed.
-
+      if (!hasDataInSttBlock(pSttBlockReader)) {
-      // no data in last block and block, no need to proceed.
-      if (hasBlockLData == false) {
         break;
       }
 
@@ -2771,14 +2778,13 @@ static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, STableBlockSc
 }
 
 static int32_t doBuildDataBlock(STsdbReader* pReader) {
-  int32_t code = TSDB_CODE_SUCCESS;
-
   SReaderStatus*       pStatus = &pReader->status;
   SDataBlockIter*      pBlockIter = &pStatus->blockIter;
   STableBlockScanInfo* pScanInfo = NULL;
   SFileDataBlockInfo*  pBlockInfo = getCurrentBlockInfo(pBlockIter);
   SSttBlockReader*     pSttBlockReader = pReader->status.fileIter.pSttBlockReader;
   bool                 asc = ASCENDING_TRAVERSE(pReader->info.order);
+  int32_t              code = TSDB_CODE_SUCCESS;
 
   if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) {
     setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order);
@@ -2836,13 +2842,14 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
 
       SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
 
-      tsdbDebug("load data in last block firstly %s", pReader->idStr);
+      tsdbDebug("load data in stt block firstly %s", pReader->idStr);
       int64_t st = taosGetTimestampUs();
 
-      // let's load data from stt files
+      // let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files
+      pScanInfo->cleanSttBlocks = false;
       initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
 
-      // no data in last block, no need to proceed.
+      // no data in stt block, no need to proceed.
       while (hasDataInSttBlock(pSttBlockReader)) {
         ASSERT(pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA);
 
@@ -2880,147 +2887,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
   return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code;
 }
 
-static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) {
-  int64_t    st = taosGetTimestampUs();
-  LRUHandle* handle = NULL;
-  int32_t    code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle);
-  if (code != TSDB_CODE_SUCCESS || handle == NULL) {
-    goto _end;
-  }
-
-#if 0
-  int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
-
-  SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle);
-  size_t  num = taosArrayGetSize(aBlockIdx);
-  if (num == 0) {
-    tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle);
-    return TSDB_CODE_SUCCESS;
-  }
-
-  SBlockIdx* pBlockIdx = NULL;
-  for (int32_t i = 0; i < num; ++i) {
-    pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i);
-    if (pBlockIdx->suid != pReader->info.suid) {
-      continue;
-    }
-
-    STableBlockScanInfo** p = tSimpleHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid));
-    if (p == NULL) {
-      continue;
-    }
-
-    STableBlockScanInfo* pScanInfo = *p;
-    SDataBlk             block = {0};
-    //    for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) {
-    //      tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block);
-    //      pReader->rowsNum += block.nRow;
-    //    }
-  }
-#endif
-
-_end:
-  tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle);
-  return code;
-}
-
-static int32_t doSumSttBlockRows(STsdbReader* pReader) {
-  int32_t            code = TSDB_CODE_SUCCESS;
-  SSttBlockReader*  pSttBlockReader = pReader->status.fileIter.pSttBlockReader;
-  SSttBlockLoadInfo* pBlockLoadInfo = NULL;
-#if 0
-  for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) {  // open all last file
-    pBlockLoadInfo = &pSttBlockReader->pInfo[i];
-
-    code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk);
-    if (code) {
-      return code;
-    }
-
-    size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);
-    if (size >= 1) {
-      SSttBlk* pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0);
-      SSttBlk* pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1);
-
-      // all identical
-      if (pStart->suid == pEnd->suid) {
-        if (pStart->suid != pReader->info.suid) {
-          // no qualified stt block existed
-          taosArrayClear(pBlockLoadInfo->aSttBlk);
-          continue;
-        }
-        for (int32_t j = 0; j < size; ++j) {
-          SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, j);
-          pReader->rowsNum += p->nRow;
-        }
-      } else {
-        for (int32_t j = 0; j < size; ++j) {
-          SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, j);
-          uint64_t s = p->suid;
-          if (s < pReader->info.suid) {
-            continue;
-          }
-
-          if (s == pReader->info.suid) {
-            pReader->rowsNum += p->nRow;
-          } else if (s > pReader->info.suid) {
-            break;
-          }
-        }
-      }
-    }
-  }
-#endif
-
-  return code;
-}
-
-static int32_t readRowsCountFromFiles(STsdbReader* pReader) {
-  int32_t code = TSDB_CODE_SUCCESS;
-
-  while (1) {
-    bool hasNext = false;
-    code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext);
-    if (code) {
-      return code;
-    }
-
-    if (!hasNext) {  // no data files on disk
-      break;
-    }
-
-    //    code = doSumFileBlockRows(pReader, pReader->pFileReader);
-    if (code != TSDB_CODE_SUCCESS) {
-      return code;
-    }
-
-    code = doSumSttBlockRows(pReader);
-    if (code != TSDB_CODE_SUCCESS) {
-      return code;
-    }
-  }
-
-  pReader->status.loadFromFile = false;
-
-  return code;
-}
-
-static int32_t readRowsCountFromMem(STsdbReader* pReader) {
-  int32_t code = TSDB_CODE_SUCCESS;
-  int64_t memNum = 0, imemNum = 0;
-  if (pReader->pReadSnap->pMem != NULL) {
-    tsdbMemTableCountRows(pReader->pReadSnap->pMem, pReader->status.pTableMap, &memNum);
-  }
-
-  if (pReader->pReadSnap->pIMem != NULL) {
-    tsdbMemTableCountRows(pReader->pReadSnap->pIMem, pReader->status.pTableMap, &imemNum);
-  }
-
-  pReader->rowsNum += memNum + imemNum;
-
-  return code;
-}
-
 static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t endKey) {
   SReaderStatus* pStatus = &pReader->status;
   STableUidList* pUidList = &pStatus->uidList;
@@ -3149,7 +3015,7 @@ static ERetrieveType doReadDataFromSttFiles(STsdbReader* pReader) {
       return TSDB_READ_RETURN;
     }
 
-    // all data blocks are checked in this last block file, now let's try the next file
+    // all data blocks are checked in this stt file, now let's try the next file set
     ASSERT(pReader->status.pTableIter == NULL);
     code = initForFirstBlockInFile(pReader, pBlockIter);
 
@@ -3875,7 +3741,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
     }
 
     if (row.type == TSDBROW_ROW_FMT) {
-      int64_t ts = row.pTSRow->ts;;
+      int64_t ts = row.pTSRow->ts;
       code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo);
 
       if (freeTSRow) {
@@ -3993,7 +3859,7 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
   int32_t code = TSDB_CODE_SUCCESS;
   if (pStatus->fileIter.numOfFiles == 0) {
     pStatus->loadFromFile = false;
-  } else if (READ_MODE_COUNT_ONLY == pReader->info.readMode) {
+//  } else if (READER_EXEC_DATA == pReader->info.readMode) {
     // DO NOTHING
   } else {
     code = initForFirstBlockInFile(pReader, pBlockIter);
@@ -4034,8 +3900,7 @@ static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) {
 
 // ====================================== EXPOSED APIs ======================================
 int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
-                        SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly,
+                        SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables) {
-                        SHashObj** pIgnoreTables) {
   STimeWindow window = pCond->twindows;
   SVnodeCfg*  pConf = &(((SVnode*)pVnode)->config);
 
@@ -4141,13 +4006,9 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi
   }
 
   pReader->flag = READER_STATUS_SUSPEND;
-
+  pReader->info.execMode = pCond->notLoadData? READER_EXEC_ROWS : READER_EXEC_DATA;
-  if (countOnly) {
-    pReader->info.readMode = READ_MODE_COUNT_ONLY;
-  }
 
   pReader->pIgnoreTables = pIgnoreTables;
-
   tsdbDebug("%p total numOfTable:%d, window:%" PRId64 " - %" PRId64 ", verRange:%" PRId64 " - %" PRId64
             " in this query %s",
             pReader, numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->info.verRange.minVer,
@@ -4257,8 +4118,6 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) {
   SReaderStatus*       pStatus = &pReader->status;
   STableBlockScanInfo* pBlockScanInfo = NULL;
 
-  pReader->status.suspendInvoked = true; // record the suspend status
-
   if (pStatus->loadFromFile) {
     SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
     if (pBlockInfo != NULL) {
@@ -4383,32 +4242,6 @@ _err:
   return code;
 }
 
-static bool tsdbReadRowsCountOnly(STsdbReader* pReader) {
-  int32_t      code = TSDB_CODE_SUCCESS;
-  SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock;
-
-  if (pReader->status.loadFromFile == false) {
-    return false;
-  }
-
-  code = readRowsCountFromFiles(pReader);
-  if (code != TSDB_CODE_SUCCESS) {
-    return false;
-  }
-
-  code = readRowsCountFromMem(pReader);
-  if (code != TSDB_CODE_SUCCESS) {
-    return false;
-  }
-
-  pBlock->info.rows = pReader->rowsNum;
-  pBlock->info.id.uid = 0;
-  pBlock->info.dataLoad = 0;
-  pReader->rowsNum = 0;
-
-  return pBlock->info.rows > 0;
-}
-
 static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) {
   SReaderStatus* pStatus = &pReader->status;
   int32_t code = TSDB_CODE_SUCCESS;
@@ -4489,9 +4322,6 @@ static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) {
     return code;
   }
 
-  if (READ_MODE_COUNT_ONLY == pReader->info.readMode) {
-    return tsdbReadRowsCountOnly(pReader);
-  }
   if (!pReader->bFilesetDelimited) {
     code = doTsdbNextDataBlockFilesFirst(pReader);
   } else {
@@ -4793,7 +4623,7 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) {
   }
 
   SReaderStatus* pStatus = &pTReader->status;
-  if (pStatus->composedDataBlock) {
+  if (pStatus->composedDataBlock || pReader->info.execMode == READER_EXEC_ROWS) {
     return pTReader->resBlockInfo.pResBlock;
   }
 
@@ -4830,9 +4660,9 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) {
 
   pReader->info.order = pCond->order;
   pReader->type = TIMEWINDOW_RANGE_CONTAINED;
+  pReader->info.window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows);
   pStatus->loadFromFile = true;
   pStatus->pTableIter = NULL;
-  pReader->info.window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows);
 
   // allocate buffer in order to load data blocks from file
   memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg));

source/dnode/vnode/src/tsdb/tsdbReadUtil.c

@@ -22,15 +22,9 @@
 #include "tsdbUtil2.h"
 #include "tsimplehash.h"
 
-int32_t uidComparFunc(const void* p1, const void* p2) {
+#define INIT_TIMEWINDOW(_w)  do { (_w)->skey = INT64_MAX; (_w)->ekey = INT64_MIN;} while(0);
-  uint64_t pu1 = *(uint64_t*)p1;
+
-  uint64_t pu2 = *(uint64_t*)p2;
+static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order);
-  if (pu1 == pu2) {
-    return 0;
-  } else {
-    return (pu1 < pu2) ? -1 : 1;
-  }
-}
 
 static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
   int32_t num = numOfTables / pBuf->numPerBucket;
@@ -61,6 +55,16 @@ static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t uidComparFunc(const void* p1, const void* p2) {
+  uint64_t pu1 = *(uint64_t*)p1;
+  uint64_t pu2 = *(uint64_t*)p2;
+  if (pu1 == pu2) {
+    return 0;
+  } else {
+    return (pu1 < pu2) ? -1 : 1;
+  }
+}
+
 int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
   if (numOfTables <= pBuf->numOfTables) {
     return TSDB_CODE_SUCCESS;
@@ -153,6 +157,9 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
     STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j);
 
     pScanInfo->uid = idList[j].uid;
+    INIT_TIMEWINDOW(&pScanInfo->sttWindow);
+    INIT_TIMEWINDOW(&pScanInfo->filesetWindow);
+
     pUidList->tableUidList[j] = idList[j].uid;
 
     if (ASCENDING_TRAVERSE(pTsdbReader->info.order)) {
@@ -243,6 +250,10 @@ static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) {
   taosArrayClear(pScanInfo->pBlockList);
   taosArrayClear(pScanInfo->pBlockIdxList);
   taosArrayClear(pScanInfo->pFileDelData);  // del data from each file set
+  pScanInfo->cleanSttBlocks = false;
+  pScanInfo->numOfRowsInStt = 0;
+  INIT_TIMEWINDOW(&pScanInfo->sttWindow);
+  INIT_TIMEWINDOW(&pScanInfo->filesetWindow);
   pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;
 }
 
@@ -403,12 +414,10 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
       blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i);
 
       taosArrayPush(pBlockIter->blockList, &blockInfo);
-
       STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};
       taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);
     }
-    taosArrayDestroy(pTableScanInfo->pBlockList);
+    pTableScanInfo->pBlockList = taosArrayDestroy(pTableScanInfo->pBlockList);
-    pTableScanInfo->pBlockList = NULL;
 
     int64_t et = taosGetTimestampUs();
     tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s",
@@ -457,8 +466,7 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
 
   for (int32_t i = 0; i < numOfTables; ++i) {
     STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i);
-    taosArrayDestroy(pTableScanInfo->pBlockList);
+    pTableScanInfo->pBlockList = taosArrayDestroy(pTableScanInfo->pBlockList);
-    pTableScanInfo->pBlockList = NULL;
   }
 
   int64_t et = taosGetTimestampUs();
@@ -488,7 +496,7 @@ typedef enum {
   BLK_CHECK_QUIT = 0x2,
 } ETombBlkCheckEnum;
 
-static void    loadNextStatisticsBlock(SSttFileReader* pSttFileReader, const SSttBlockLoadInfo* pBlockLoadInfo,
+static void    loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlock* pStatisBlock,
                                        const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i, int32_t* j);
 static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_t numOfTables, int32_t* j,
                                 ETombBlkCheckEnum* pRet) {
@@ -662,17 +670,17 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT
   }
 }
 
-int32_t getNumOfRowsInSttBlock(SSttFileReader *pSttFileReader, SSttBlockLoadInfo *pBlockLoadInfo, uint64_t suid,
+int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,
-                               const uint64_t* pUidList, int32_t numOfTables) {
+                               TStatisBlkArray* pStatisBlkArray, uint64_t suid, const uint64_t* pUidList,
+                               int32_t numOfTables) {
   int32_t num = 0;
 
-  const TStatisBlkArray *pStatisBlkArray = pBlockLoadInfo->pSttStatisBlkArray;
   if (TARRAY2_SIZE(pStatisBlkArray) <= 0) {
     return 0;
   }
 
   int32_t i = 0;
-  while((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].minTbid.suid < suid)) {
+  while((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].maxTbid.suid < suid)) {
     ++i;
   }
 
@@ -681,64 +689,65 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader *pSttFileReader, SSttBlockLoadInfo
   }
 
   SStatisBlk *p = &pStatisBlkArray->data[i];
-  if (pBlockLoadInfo->statisBlock == NULL) {
+  STbStatisBlock* pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock));
-    pBlockLoadInfo->statisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock));
+  tStatisBlockInit(pStatisBlock);
-    tStatisBlockInit(pBlockLoadInfo->statisBlock);
-  }
 
   int64_t st = taosGetTimestampMs();
-  tsdbSttFileReadStatisBlock(pSttFileReader, p, pBlockLoadInfo->statisBlock);
+  tsdbSttFileReadStatisBlock(pSttFileReader, p, pStatisBlock);
-  pBlockLoadInfo->statisBlockIndex = i;
 
   double el = (taosGetTimestampMs() - st) / 1000.0;
   pBlockLoadInfo->cost.loadStatisBlocks += 1;
   pBlockLoadInfo->cost.statisElapsedTime += el;
 
-  STbStatisBlock *pBlock = pBlockLoadInfo->statisBlock;
-
   int32_t index = 0;
-  while (index < TARRAY2_SIZE(pBlock->suid) && pBlock->suid->data[index] < suid) {
+  while (index < TARRAY2_SIZE(pStatisBlock->suid) && pStatisBlock->suid->data[index] < suid) {
     ++index;
   }
 
-  if (index >= TARRAY2_SIZE(pBlock->suid)) {
+  if (index >= TARRAY2_SIZE(pStatisBlock->suid)) {
+    tStatisBlockDestroy(pStatisBlock);
+    taosMemoryFreeClear(pStatisBlock);
     return num;
   }
 
   int32_t j = index;
   int32_t uidIndex = 0;
-  while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex <= numOfTables) {
+  while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex < numOfTables) {
     p = &pStatisBlkArray->data[i];
     if (p->minTbid.suid > suid) {
+      tStatisBlockDestroy(pStatisBlock);
+      taosMemoryFreeClear(pStatisBlock);
       return num;
     }
 
     uint64_t uid = pUidList[uidIndex];
 
-    if (pBlock->uid->data[j] == uid) {
+    if (pStatisBlock->uid->data[j] == uid) {
-      num += pBlock->count->data[j];
+      num += pStatisBlock->count->data[j];
       uidIndex += 1;
       j += 1;
-      loadNextStatisticsBlock(pSttFileReader, pBlockLoadInfo, pStatisBlkArray, pBlock->suid->size, &i, &j);
+      loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->suid->size, &i, &j);
-    } else if (pBlock->uid->data[j] < uid) {
+    } else if (pStatisBlock->uid->data[j] < uid) {
       j += 1;
-      loadNextStatisticsBlock(pSttFileReader, pBlockLoadInfo, pStatisBlkArray, pBlock->suid->size, &i, &j);
+      loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->suid->size, &i, &j);
     } else {
       uidIndex += 1;
     }
   }
 
+  tStatisBlockDestroy(pStatisBlock);
+  taosMemoryFreeClear(pStatisBlock);
   return num;
 }
 
 // load next stt statistics block
-static void loadNextStatisticsBlock(SSttFileReader* pSttFileReader, const SSttBlockLoadInfo* pBlockLoadInfo,
+static void loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlock* pStatisBlock,
                                     const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i, int32_t* j) {
   if ((*j) >= numOfRows) {
     (*i) += 1;
     (*j) = 0;
     if ((*i) < TARRAY2_SIZE(pStatisBlkArray)) {
-      tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pBlockLoadInfo->statisBlock);
+      tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock);
     }
   }
 }
@@ -762,7 +771,7 @@ void doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) {
   }
 }
 
-int32_t adjustLDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) {
+int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) {
   int32_t numOfLevels = pFileSet->lvlArr->size;
 
   // add the list/iter placeholder
@@ -791,7 +800,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
   }
 
   // add the list/iter placeholder
-  adjustLDataIters(pSttFileBlockIterArray, pFileSet);
+  adjustSttDataIters(pSttFileBlockIterArray, pFileSet);
 
   for (int32_t j = 0; j < numOfLevels; ++j) {
     SSttLvl* pSttLevel = pFileSet->lvlArr->data[j];
@@ -819,7 +828,8 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
       }
 
       // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file
-      int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pIter->pBlockLoadInfo->pSttStatisBlkArray);
+      TStatisBlkArray *pStatisBlkArray = NULL;
+      int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pStatisBlkArray);
       if (code != TSDB_CODE_SUCCESS) {
         tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr);
         continue;
@@ -829,9 +839,214 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
       STsdbReader* pReader = pConf->pReader;
       int32_t      numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
       uint64_t*    pUidList = pReader->status.uidList.tableUidList;
-      numOfRows += getNumOfRowsInSttBlock(pIter->pReader, pIter->pBlockLoadInfo, pConf->suid, pUidList, numOfTables);
+      numOfRows += getNumOfRowsInSttBlock(pIter->pReader, pIter->pBlockLoadInfo, pStatisBlkArray, pConf->suid, pUidList,
+                                          numOfTables);
     }
   }
 
   return numOfRows;
 }
+
+static bool overlapHelper(const STimeWindow* pLeft, TSKEY minKey, TSKEY maxKey) {
+  return (pLeft->ekey >= minKey) && (pLeft->skey <= maxKey);
+}
+
+static bool overlapWithTimeWindow(STimeWindow* p1, STimeWindow* pQueryWindow, STableBlockScanInfo* pBlockScanInfo,
+                                  int32_t order) {
+  // overlap with query window
+  if (!(p1->skey >= pQueryWindow->skey && p1->ekey <= pQueryWindow->ekey)) {
+    return true;
+  }
+
+  SIterInfo* pMemIter = &pBlockScanInfo->iter;
+  SIterInfo* pIMemIter = &pBlockScanInfo->iiter;
+
+  // overlap with mem data
+  if (pMemIter->hasVal) {
+    STbData* pTbData = pMemIter->iter->pTbData;
+    if (overlapHelper(p1, pTbData->minKey, pTbData->maxKey)) {
+      return true;
+    }
+  }
+
+  // overlap with imem data
+  if (pIMemIter->hasVal) {
+    STbData* pITbData = pIMemIter->iter->pTbData;
+    if (overlapHelper(p1, pITbData->minKey, pITbData->maxKey)) {
+      return true;
+    }
+  }
+
+  // overlap with data file block
+  STimeWindow* pFileWin = &pBlockScanInfo->filesetWindow;
+  if ((taosArrayGetSize(pBlockScanInfo->pBlockIdxList) > 0) && overlapHelper(p1, pFileWin->skey, pFileWin->ekey)) {
+    return true;
+  }
+
+  // overlap with deletion skyline
+  SBrinRecord record = {.firstKey = p1->skey, .lastKey = p1->ekey};
+  if (overlapWithDelSkylineWithoutVer(pBlockScanInfo, &record, order)) {
+    return true;
+  }
+
+  return false;
+}
+
+static int32_t sortUidComparFn(const void* p1, const void* p2) {
+  const STimeWindow* px1 = p1;
+  const STimeWindow* px2 = p2;
+  if (px1->skey == px2->skey) {
+    return 0;
+  } else {
+    return px1->skey < px2->skey? -1:1;
+  }
+}
+
+bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableBlockScanInfo *pScanInfo, int32_t order) {
+  // check if it overlap with del skyline
+  taosArraySort(pTimewindowList, sortUidComparFn);
+
+  int32_t num = taosArrayGetSize(pTimewindowList);
+  if (num == 0) {
+    return false;
+  }
+
+  STimeWindow* p = taosArrayGet(pTimewindowList, 0);
+  if (overlapWithTimeWindow(p, pQueryWindow, pScanInfo, order)) {
+    return false;
+  }
+
+  for (int32_t i = 0; i < num - 1; ++i) {
+    STimeWindow* p1 = taosArrayGet(pTimewindowList, i);
+    STimeWindow* p2 = taosArrayGet(pTimewindowList, i + 1);
+
+    if (p1->ekey >= p2->skey) {
+      return false;
+    }
+
+    bool overlap = overlapWithTimeWindow(p2, pQueryWindow, pScanInfo, order);
+    if (overlap) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,
+                                       int32_t startIndex) {
+  size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);
+
+  for (int32_t i = startIndex; i < num; i += 1) {
+    TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);
+    if (p->ts >= pRecord->firstKey && p->ts <= pRecord->lastKey) {
+      if (p->version >= pRecord->minVer) {
+        return true;
+      }
+    } else if (p->ts < pRecord->firstKey) {  // p->ts < pBlock->minKey.ts
+      if (p->version >= pRecord->minVer) {
+        if (i < num - 1) {
+          TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);
+          if (pnext->ts >= pRecord->firstKey) {
+            return true;
+          }
+        } else {  // it must be the last point
+          ASSERT(p->version == 0);
+        }
+      }
+    } else {  // (p->ts > pBlock->maxKey.ts) {
+      return false;
+    }
+  }
+
+  return false;
+}
+
+static bool doCheckDatablockOverlapWithoutVersion(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,
+                                                  int32_t startIndex) {
+  size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);
+
+  for (int32_t i = startIndex; i < num; i += 1) {
+    TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);
+    if (p->ts >= pRecord->firstKey && p->ts <= pRecord->lastKey) {
+      return true;
+    } else if (p->ts < pRecord->firstKey) {  // p->ts < pBlock->minKey.ts
+      if (i < num - 1) {
+        TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);
+        if (pnext->ts >= pRecord->firstKey) {
+          return true;
+        }
+      }
+    } else {  // (p->ts > pBlock->maxKey.ts) {
+      return false;
+    }
+  }
+
+  return false;
+}
+
+bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) {
+  if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) {
+    return false;
+  }
+
+  // ts is not overlap
+  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);
+  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);
+  if (pRecord->firstKey > pLast->ts || pRecord->lastKey < pFirst->ts) {
+    return false;
+  }
+
+  // version is not overlap
+  if (ASCENDING_TRAVERSE(order)) {
+    return doCheckDatablockOverlap(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex);
+  } else {
+    int32_t index = pBlockScanInfo->fileDelIndex;
+    while (1) {
+      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);
+      if (p->ts > pRecord->firstKey && index > 0) {
+        index -= 1;
+      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.
+        if (p->ts == pRecord->firstKey && p->version < pRecord->maxVer && index > 0) {
+          index -= 1;
+        }
+        break;
+      }
+    }
+
+    return doCheckDatablockOverlap(pBlockScanInfo, pRecord, index);
+  }
+}
+
+bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) {
+  if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) {
+    return false;
+  }
+
+  // ts is not overlap
+  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);
+  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);
+  if (pRecord->firstKey > pLast->ts || pRecord->lastKey < pFirst->ts) {
+    return false;
+  }
+
+  // version is not overlap
+  if (ASCENDING_TRAVERSE(order)) {
+    return doCheckDatablockOverlapWithoutVersion(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex);
+  } else {
+    int32_t index = pBlockScanInfo->fileDelIndex;
+    while (1) {
+      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);
+      if (p->ts > pRecord->firstKey && index > 0) {
+        index -= 1;
+      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.
+        if (p->ts == pRecord->firstKey && index > 0) {
+          index -= 1;
+        }
+        break;
+      }
+    }
+
+    return doCheckDatablockOverlapWithoutVersion(pBlockScanInfo, pRecord, index);
+  }
+}

source/dnode/vnode/src/tsdb/tsdbReadUtil.h

@@ -40,8 +40,7 @@ typedef enum {
 typedef struct STsdbReaderInfo {
   uint64_t      suid;
   STSchema*     pSchema;
-  EReadMode     readMode;
+  EExecMode     execMode;
-  uint64_t      rowsNum;
   STimeWindow   window;
   SVersionRange verRange;
   int16_t       order;
@@ -75,6 +74,11 @@ typedef struct SSttKeyInfo {
   int64_t       nextProcKey;
 } SSttKeyInfo;
 
+// clean stt file blocks:
+// 1. not overlap with stt blocks in other stt files of the same fileset
+// 2. not overlap with delete skyline
+// 3. not overlap with in-memory data (mem/imem)
+// 4. not overlap with data file blocks
 typedef struct STableBlockScanInfo {
   uint64_t    uid;
   TSKEY       lastProcKey;
@@ -89,6 +93,11 @@ typedef struct STableBlockScanInfo {
   int32_t     fileDelIndex;      // file block delete index
   int32_t     sttBlockDelIndex;  // delete index for last block
   bool        iterInit;          // whether to initialize the in-memory skip list iterator or not
+  bool        cleanSttBlocks;    // stt block is clean in current fileset
+  bool        sttBlockReturned;  // result block returned alreay
+  int64_t     numOfRowsInStt;
+  STimeWindow sttWindow;         // timestamp window for current stt files
+  STimeWindow filesetWindow;     // timestamp window for current file set
 } STableBlockScanInfo;
 
 typedef struct SResultBlockInfo {
@@ -146,6 +155,7 @@ typedef struct SBlockLoadSuppInfo {
   bool                smaValid;  // the sma on all queried columns are activated
 } SBlockLoadSuppInfo;
 
+// each blocks in stt file not overlaps with in-memory/data-file/tomb-files, and not overlap with any other blocks in stt-file
 typedef struct SSttBlockReader {
   STimeWindow        window;
   SVersionRange      verRange;
@@ -187,7 +197,6 @@ typedef struct SFileBlockDumpInfo {
 } SFileBlockDumpInfo;
 
 typedef struct SReaderStatus {
-  bool                  suspendInvoked;
   bool                  loadFromFile;       // check file stage
   bool                  composedDataBlock;  // the returned data block is a composed block or not
   SSHashObj*            pTableMap;          // SHash<STableBlockScanInfo>
@@ -272,12 +281,17 @@ bool    blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr);
 void    loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver);
 int32_t loadDataFileTombDataForAll(STsdbReader* pReader);
 int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo);
-int32_t getNumOfRowsInSttBlock(SSttFileReader *pSttFileReader, SSttBlockLoadInfo *pBlockLoadInfo, uint64_t suid,
+int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,
-                               const uint64_t* pUidList, int32_t numOfTables);
+                               TStatisBlkArray* pStatisBlkArray, uint64_t suid, const uint64_t* pUidList,
+                               int32_t numOfTables);
+
 void    destroyLDataIter(SLDataIter* pIter);
-int32_t adjustLDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet);
+int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet);
 int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf,
                               const char* pstr);
+bool    isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, int32_t order);
+bool    overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order);
+
 typedef struct {
   SArray* pTombData;
 } STableLoadInfo;

source/dnode/vnode/src/tsdb/tsdbSnapInfo.c

@@ -0,0 +1,627 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdb.h"
+#include "tsdbFS2.h"
+
+#define TSDB_SNAP_MSG_VER 1
+
+// fset partition
+static int32_t tsdbFSetPartCmprFn(STsdbFSetPartition* x, STsdbFSetPartition* y) {
+  if (x->fid < y->fid) return -1;
+  if (x->fid > y->fid) return 1;
+  return 0;
+}
+
+static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) {
+  if (x->minVer < y->minVer) return -1;
+  if (x->minVer > y->minVer) return 1;
+  if (x->maxVer < y->maxVer) return -1;
+  if (x->maxVer > y->maxVer) return 1;
+  return 0;
+}
+
+static int32_t tsdbTFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) {
+  if (x->fid < y->fid) return -1;
+  if (x->fid > y->fid) return 1;
+  return 0;
+}
+
+STsdbFSetPartition* tsdbFSetPartitionCreate() {
+  STsdbFSetPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbFSetPartition));
+  if (pSP == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) {
+    TARRAY2_INIT(&pSP->verRanges[i]);
+  }
+  return pSP;
+}
+
+void tsdbFSetPartitionClear(STsdbFSetPartition** ppSP) {
+  if (ppSP == NULL || ppSP[0] == NULL) {
+    return;
+  }
+  for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) {
+    TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL);
+  }
+  taosMemoryFree(ppSP[0]);
+  ppSP[0] = NULL;
+}
+
+static int32_t tsdbFTypeToFRangeType(tsdb_ftype_t ftype) {
+  switch (ftype) {
+    case TSDB_FTYPE_HEAD:
+      return TSDB_FSET_RANGE_TYP_HEAD;
+    case TSDB_FTYPE_DATA:
+      return TSDB_FSET_RANGE_TYP_DATA;
+    case TSDB_FTYPE_SMA:
+      return TSDB_FSET_RANGE_TYP_SMA;
+    case TSDB_FTYPE_TOMB:
+      return TSDB_FSET_RANGE_TYP_TOMB;
+    case TSDB_FTYPE_STT:
+      return TSDB_FSET_RANGE_TYP_STT;
+  }
+  return TSDB_FSET_RANGE_TYP_MAX;
+}
+
+static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition** ppSP) {
+  STsdbFSetPartition* p = tsdbFSetPartitionCreate();
+  if (p == NULL) {
+    goto _err;
+  }
+
+  p->fid = fset->fid;
+
+  int32_t code = 0;
+  int32_t typ = 0;
+  int32_t corrupt = false;
+  int32_t count = 0;
+  for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
+    if (fset->farr[ftype] == NULL) continue;
+    typ = tsdbFTypeToFRangeType(ftype);
+    ASSERT(typ < TSDB_FSET_RANGE_TYP_MAX);
+    STFile* f = fset->farr[ftype]->f;
+    if (f->maxVer > fset->maxVerValid) {
+      corrupt = true;
+      tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64
+                ", ftype: %d",
+                fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype);
+      continue;
+    }
+    count++;
+    SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer};
+    code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
+    ASSERT(code == 0);
+  }
+
+  typ = TSDB_FSET_RANGE_TYP_STT;
+  const SSttLvl* lvl;
+  TARRAY2_FOREACH(fset->lvlArr, lvl) {
+    STFileObj* fobj;
+    TARRAY2_FOREACH(lvl->fobjArr, fobj) {
+      STFile* f = fobj->f;
+      if (f->maxVer > fset->maxVerValid) {
+        corrupt = true;
+        tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64
+                  ", ftype: %d",
+                  fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ);
+        continue;
+      }
+      count++;
+      SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer};
+      code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
+      ASSERT(code == 0);
+    }
+  }
+  if (corrupt && count == 0) {
+    SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid};
+    code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
+    ASSERT(code == 0);
+  }
+  ppSP[0] = p;
+  return 0;
+
+_err:
+  tsdbFSetPartitionClear(&p);
+  return -1;
+}
+
+// fset partition list
+STsdbFSetPartList* tsdbFSetPartListCreate() {
+  STsdbFSetPartList* pList = taosMemoryCalloc(1, sizeof(STsdbFSetPartList));
+  if (pList == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  TARRAY2_INIT(pList);
+  return pList;
+}
+
+void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) {
+  if (ppList == NULL || ppList[0] == NULL) return;
+
+  TARRAY2_DESTROY(ppList[0], tsdbFSetPartitionClear);
+  taosMemoryFree(ppList[0]);
+  ppList[0] = NULL;
+}
+
+int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) {
+  TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray));
+  if (pDiff == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  TARRAY2_INIT(pDiff);
+
+  STsdbFSetPartition* part;
+  TARRAY2_FOREACH(pList, part) {
+    STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange));
+    if (r == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+    int64_t maxVerValid = -1;
+    int32_t typMax = TSDB_FSET_RANGE_TYP_MAX;
+    for (int32_t i = 0; i < typMax; i++) {
+      SVerRangeList* iList = &part->verRanges[i];
+      SVersionRange  vr = {0};
+      TARRAY2_FOREACH(iList, vr) {
+        if (vr.maxVer < vr.minVer) {
+          continue;
+        }
+        maxVerValid = TMAX(maxVerValid, vr.maxVer);
+      }
+    }
+    r->fid = part->fid;
+    r->sver = maxVerValid + 1;
+    r->ever = VERSION_MAX;
+    tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever);
+    int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbTFileSetRangeCmprFn);
+    ASSERT(code == 0);
+  }
+  ppRanges[0] = pDiff;
+
+  tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff));
+  return 0;
+
+_err:
+  if (pDiff) {
+    tsdbTFileSetRangeArrayDestroy(&pDiff);
+  }
+  return -1;
+}
+
+// serialization
+int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) {
+  int32_t hdrLen = sizeof(int32_t);
+  int32_t datLen = 0;
+
+  int8_t  msgVer = 1;
+  int32_t len = TARRAY2_SIZE(pList);
+  hdrLen += sizeof(msgVer);
+  hdrLen += sizeof(len);
+  datLen += hdrLen;
+
+  for (int32_t u = 0; u < len; u++) {
+    STsdbFSetPartition* p = TARRAY2_GET(pList, u);
+    int32_t             typMax = TSDB_FSET_RANGE_TYP_MAX;
+    int32_t             uItem = 0;
+    uItem += sizeof(STsdbFSetPartition);
+    uItem += sizeof(typMax);
+
+    for (int32_t i = 0; i < typMax; i++) {
+      int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]);
+      int32_t jItem = 0;
+      jItem += sizeof(SVersionRange);
+      jItem += sizeof(int64_t);
+      uItem += sizeof(iLen) + jItem * iLen;
+    }
+    datLen += uItem;
+  }
+  return datLen;
+}
+
+int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  int8_t  reserved8 = 0;
+  int16_t reserved16 = 0;
+  int64_t reserved64 = 0;
+
+  int8_t  msgVer = TSDB_SNAP_MSG_VER;
+  int32_t len = TARRAY2_SIZE(pList);
+
+  if (tStartEncode(&encoder) < 0) goto _err;
+  if (tEncodeI8(&encoder, msgVer) < 0) goto _err;
+  if (tEncodeI32(&encoder, len) < 0) goto _err;
+
+  for (int32_t u = 0; u < len; u++) {
+    STsdbFSetPartition* p = TARRAY2_GET(pList, u);
+    if (tEncodeI64(&encoder, p->fid) < 0) goto _err;
+    if (tEncodeI8(&encoder, p->stat) < 0) goto _err;
+    if (tEncodeI8(&encoder, reserved8) < 0) goto _err;
+    if (tEncodeI16(&encoder, reserved16) < 0) goto _err;
+
+    int32_t typMax = TSDB_FSET_RANGE_TYP_MAX;
+    if (tEncodeI32(&encoder, typMax) < 0) goto _err;
+
+    for (int32_t i = 0; i < typMax; i++) {
+      SVerRangeList* iList = &p->verRanges[i];
+      int32_t        iLen = TARRAY2_SIZE(iList);
+
+      if (tEncodeI32(&encoder, iLen) < 0) goto _err;
+      for (int32_t j = 0; j < iLen; j++) {
+        SVersionRange r = TARRAY2_GET(iList, j);
+        if (tEncodeI64(&encoder, r.minVer) < 0) goto _err;
+        if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err;
+        if (tEncodeI64(&encoder, reserved64) < 0) goto _err;
+      }
+    }
+  }
+
+  tEndEncode(&encoder);
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+
+_err:
+  tEncoderClear(&encoder);
+  return -1;
+}
+
+int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  int8_t  reserved8 = 0;
+  int16_t reserved16 = 0;
+  int64_t reserved64 = 0;
+
+  STsdbFSetPartition* p = NULL;
+
+  int8_t  msgVer = 0;
+  int32_t len = 0;
+  if (tStartDecode(&decoder) < 0) goto _err;
+  if (tDecodeI8(&decoder, &msgVer) < 0) goto _err;
+  if (msgVer != TSDB_SNAP_MSG_VER) goto _err;
+  if (tDecodeI32(&decoder, &len) < 0) goto _err;
+
+  for (int32_t u = 0; u < len; u++) {
+    p = tsdbFSetPartitionCreate();
+    if (p == NULL) goto _err;
+    if (tDecodeI64(&decoder, &p->fid) < 0) goto _err;
+    if (tDecodeI8(&decoder, &p->stat) < 0) goto _err;
+    if (tDecodeI8(&decoder, &reserved8) < 0) goto _err;
+    if (tDecodeI16(&decoder, &reserved16) < 0) goto _err;
+
+    int32_t typMax = 0;
+    if (tDecodeI32(&decoder, &typMax) < 0) goto _err;
+
+    for (int32_t i = 0; i < typMax; i++) {
+      SVerRangeList* iList = &p->verRanges[i];
+      int32_t        iLen = 0;
+      if (tDecodeI32(&decoder, &iLen) < 0) goto _err;
+      for (int32_t j = 0; j < iLen; j++) {
+        SVersionRange r = {0};
+        if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err;
+        if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err;
+        if (tDecodeI64(&decoder, &reserved64) < 0) goto _err;
+        TARRAY2_APPEND(iList, r);
+      }
+    }
+    TARRAY2_APPEND(pList, p);
+    p = NULL;
+  }
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+
+_err:
+  if (p) {
+    tsdbFSetPartitionClear(&p);
+  }
+  tDecoderClear(&decoder);
+  return -1;
+}
+
+// fs state
+static STsdbFSetPartList* tsdbSnapGetFSetPartList(STFileSystem* fs) {
+  STsdbFSetPartList* pList = tsdbFSetPartListCreate();
+  if (pList == NULL) {
+    return NULL;
+  }
+
+  int32_t code = 0;
+  taosThreadMutexLock(&fs->tsdb->mutex);
+  STFileSet* fset;
+  TARRAY2_FOREACH(fs->fSetArr, fset) {
+    STsdbFSetPartition* pItem = NULL;
+    if (tsdbTFileSetToFSetPartition(fset, &pItem) < 0) {
+      code = -1;
+      break;
+    }
+    ASSERT(pItem != NULL);
+    code = TARRAY2_SORT_INSERT(pList, pItem, tsdbFSetPartCmprFn);
+    ASSERT(code == 0);
+  }
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
+
+  if (code) {
+    TARRAY2_DESTROY(pList, tsdbFSetPartitionClear);
+    taosMemoryFree(pList);
+    pList = NULL;
+  }
+  return pList;
+}
+
+ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) {
+  if (!VND_IS_RSMA(pVnode)) {
+    return pVnode->pTsdb->pFS->fsstate;
+  }
+  for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) {
+    STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl);
+    if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) {
+      return TSDB_FS_STATE_INCOMPLETE;
+    }
+  }
+  return TSDB_FS_STATE_NORMAL;
+}
+
+// description
+typedef struct STsdbPartitionInfo {
+  int32_t            vgId;
+  int32_t            tsdbMaxCnt;
+  int32_t            subTyps[TSDB_RETENTION_MAX];
+  STsdbFSetPartList* pLists[TSDB_RETENTION_MAX];
+} STsdbPartitionInfo;
+
+static int32_t tsdbPartitionInfoInit(SVnode* pVnode, STsdbPartitionInfo* pInfo) {
+  int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2};
+  pInfo->vgId = TD_VID(pVnode);
+  pInfo->tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX);
+
+  ASSERT(sizeof(pInfo->subTyps) == sizeof(subTyps));
+  memcpy(pInfo->subTyps, (char*)subTyps, sizeof(subTyps));
+
+  // fset partition list
+  memset(pInfo->pLists, 0, sizeof(pInfo->pLists[0]) * TSDB_RETENTION_MAX);
+  for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) {
+    STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j);
+    pInfo->pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS);
+    if (pInfo->pLists[j] == NULL) return -1;
+  }
+  return 0;
+}
+
+static void tsdbPartitionInfoClear(STsdbPartitionInfo* pInfo) {
+  for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) {
+    if (pInfo->pLists[j] == NULL) continue;
+    tsdbFSetPartListDestroy(&pInfo->pLists[j]);
+  }
+}
+
+static int32_t tsdbPartitionInfoEstSize(STsdbPartitionInfo* pInfo) {
+  int32_t dataLen = 0;
+  for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) {
+    dataLen += sizeof(SSyncTLV);  // subTyps[j]
+    dataLen += tTsdbFSetPartListDataLenCalc(pInfo->pLists[j]);
+  }
+  return dataLen;
+}
+
+static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* buf, int32_t bufLen) {
+  int32_t tlen = 0;
+  int32_t offset = 0;
+  for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) {
+    SSyncTLV* pSubHead = (void*)((char*)buf + offset);
+    int32_t   valOffset = offset + sizeof(*pSubHead);
+    ASSERT(pSubHead->val == (char*)buf + valOffset);
+    if ((tlen = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j])) < 0) {
+      tsdbError("vgId:%d, failed to serialize fset partition list of tsdb %d since %s", pInfo->vgId, j, terrstr());
+      return -1;
+    }
+    pSubHead->typ = pInfo->subTyps[j];
+    pSubHead->len = tlen;
+    offset += sizeof(*pSubHead) + tlen;
+  }
+  return offset;
+}
+
+// tsdb replication opts
+static int32_t tTsdbRepOptsDataLenCalc(STsdbRepOpts* pInfo) {
+  int32_t hdrLen = sizeof(int32_t);
+  int32_t datLen = 0;
+
+  int8_t  msgVer = 0;
+  int64_t reserved64 = 0;
+  int16_t format = 0;
+  hdrLen += sizeof(msgVer);
+  datLen += hdrLen;
+  datLen += sizeof(format);
+  datLen += sizeof(reserved64);
+  datLen += sizeof(*pInfo);
+  return datLen;
+}
+
+int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  int64_t reserved64 = 0;
+  int8_t  msgVer = TSDB_SNAP_MSG_VER;
+
+  if (tStartEncode(&encoder) < 0) goto _err;
+  if (tEncodeI8(&encoder, msgVer) < 0) goto _err;
+  int16_t format = pOpts->format;
+  if (tEncodeI16(&encoder, format) < 0) goto _err;
+  if (tEncodeI64(&encoder, reserved64) < 0) goto _err;
+
+  tEndEncode(&encoder);
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+
+_err:
+  tEncoderClear(&encoder);
+  return -1;
+}
+
+int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  int64_t reserved64 = 0;
+  int8_t  msgVer = 0;
+
+  if (tStartDecode(&decoder) < 0) goto _err;
+  if (tDecodeI8(&decoder, &msgVer) < 0) goto _err;
+  if (msgVer != TSDB_SNAP_MSG_VER) goto _err;
+  int16_t format = 0;
+  if (tDecodeI16(&decoder, &format) < 0) goto _err;
+  pOpts->format = format;
+  if (tDecodeI64(&decoder, &reserved64) < 0) goto _err;
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+
+_err:
+  tDecoderClear(&decoder);
+  return -1;
+}
+
+static int32_t tsdbRepOptsEstSize(STsdbRepOpts* pOpts) {
+  int32_t dataLen = 0;
+  dataLen += sizeof(SSyncTLV);
+  dataLen += tTsdbRepOptsDataLenCalc(pOpts);
+  return dataLen;
+}
+
+static int32_t tsdbRepOptsSerialize(STsdbRepOpts* pOpts, void* buf, int32_t bufLen) {
+  SSyncTLV* pSubHead = buf;
+  int32_t   offset = 0;
+  int32_t   tlen = 0;
+  if ((tlen = tSerializeTsdbRepOpts(pSubHead->val, bufLen, pOpts)) < 0) {
+    return -1;
+  }
+  pSubHead->typ = SNAP_DATA_RAW;
+  pSubHead->len = tlen;
+  offset += sizeof(*pSubHead) + tlen;
+  return offset;
+}
+
+// snap info
+static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, STsdbRepOpts* pInfo) {
+  if (!pSnap->data) return 0;
+  int32_t code = -1;
+
+  SSyncTLV* pHead = (void*)pSnap->data;
+  int32_t   offset = 0;
+
+  while (offset + sizeof(*pHead) < pHead->len) {
+    SSyncTLV* pField = (void*)(pHead->val + offset);
+    offset += sizeof(*pField) + pField->len;
+    void*   buf = pField->val;
+    int32_t bufLen = pField->len;
+
+    switch (pField->typ) {
+      case SNAP_DATA_TSDB:
+      case SNAP_DATA_RSMA1:
+      case SNAP_DATA_RSMA2: {
+      } break;
+      case SNAP_DATA_RAW: {
+        if (tDeserializeTsdbRepOpts(buf, bufLen, pInfo) < 0) {
+          terrno = TSDB_CODE_INVALID_DATA_FMT;
+          tsdbError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
+          goto _out;
+        }
+      } break;
+      default:
+        tsdbError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), pField->typ);
+        goto _out;
+    }
+  }
+
+  code = 0;
+_out:
+  return code;
+}
+
+int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
+  ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY);
+  STsdbPartitionInfo  partitionInfo = {0};
+  int                 code = -1;
+  STsdbPartitionInfo* pInfo = &partitionInfo;
+
+  if (tsdbPartitionInfoInit(pVnode, pInfo) != 0) {
+    goto _out;
+  }
+
+  // deal with snap info for reply
+  STsdbRepOpts opts = {.format = TSDB_SNAP_REP_FMT_RAW};
+  if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
+    STsdbRepOpts leaderOpts = {0};
+    if (tsdbSnapPrepDealWithSnapInfo(pVnode, pSnap, &leaderOpts) < 0) {
+      tsdbError("vgId:%d, failed to deal with snap info for reply since %s", TD_VID(pVnode), terrstr());
+      goto _out;
+    }
+    opts.format = TMIN(opts.format, leaderOpts.format);
+  }
+
+  // info data realloc
+  const int32_t headLen = sizeof(SSyncTLV);
+  int32_t       bufLen = headLen;
+  bufLen += tsdbPartitionInfoEstSize(pInfo);
+  bufLen += tsdbRepOptsEstSize(&opts);
+  if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) {
+    tsdbError("vgId:%d, failed to realloc memory for data of snap info. bytes:%d", TD_VID(pVnode), bufLen);
+    goto _out;
+  }
+
+  // serialization
+  char*   buf = (void*)pSnap->data;
+  int32_t offset = headLen;
+  int32_t tlen = 0;
+
+  if ((tlen = tsdbPartitionInfoSerialize(pInfo, buf + offset, bufLen - offset)) < 0) {
+    tsdbError("vgId:%d, failed to serialize tsdb partition info since %s", TD_VID(pVnode), terrstr());
+    goto _out;
+  }
+  offset += tlen;
+  ASSERT(offset <= bufLen);
+
+  if ((tlen = tsdbRepOptsSerialize(&opts, buf + offset, bufLen - offset)) < 0) {
+    tsdbError("vgId:%d, failed to serialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
+    goto _out;
+  }
+  offset += tlen;
+  ASSERT(offset <= bufLen);
+
+  // set header of info data
+  SSyncTLV* pHead = pSnap->data;
+  pHead->typ = pSnap->type;
+  pHead->len = offset - headLen;
+
+  tsdbInfo("vgId:%d, tsdb snap info prepared. type:%s, val length:%d", TD_VID(pVnode), TMSG_INFO(pHead->typ),
+           pHead->len);
+  code = 0;
+_out:
+  tsdbPartitionInfoClear(pInfo);
+  return code;
+}

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -32,12 +32,12 @@ struct STsdbSnapReader {
   uint8_t* aBuf[5];
   SSkmInfo skmTb[1];
 
-  TSnapRangeArray* fsrArr;
+  TFileSetRangeArray* fsrArr;
 
   // context
   struct {
     int32_t      fsrArrIdx;
-    STSnapRange* fsr;
+    STFileSetRange* fsr;
     bool         isDataDone;
     bool         isTombDone;
   } ctx[1];
@@ -331,7 +331,7 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** dat
 
     if (!(reader->blockData->nRow % 16)) {
       int64_t nData = tBlockDataSize(reader->blockData);
-      if (nData >= 1 * 1024 * 1024) {
+      if (nData >= TSDB_SNAP_DATA_PAYLOAD_SIZE) {
         break;
       }
     }
@@ -437,14 +437,14 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type,
   reader[0]->ever = ever;
   reader[0]->type = type;
 
-  code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TSnapRangeArray*)pRanges, &reader[0]->fsrArr);
+  code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TFileSetRangeArray*)pRanges, &reader[0]->fsrArr);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
   if (code) {
     tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode),
               __func__, lino, tstrerror(code), sver, ever, type);
-    tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr);
+    tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr);
     taosMemoryFree(reader[0]);
     reader[0] = NULL;
   } else {
@@ -472,7 +472,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) {
   TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose);
   tsdbDataFileReaderClose(&reader[0]->dataReader);
 
-  tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr);
+  tsdbFSDestroyRefRangedSnapshot(&reader[0]->fsrArr);
   tDestroyTSchema(reader[0]->skmTb->pTSchema);
 
   for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) {
@@ -1061,7 +1061,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang
   writer[0]->compactVersion = INT64_MAX;
   writer[0]->now = taosGetTimestampMs();
 
-  code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TSnapRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr);
+  code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TFileSetRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
@@ -1125,7 +1125,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) {
   tsdbDataFileReaderClose(&writer[0]->ctx->dataReader);
 
   TARRAY2_DESTROY(writer[0]->fopArr, NULL);
-  tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr);
+  tsdbFSDestroyCopyRangedSnapshot(&writer[0]->fsetArr);
 
   for (int32_t i = 0; i < ARRAY_SIZE(writer[0]->aBuf); ++i) {
     tFree(writer[0]->aBuf[i]);
@@ -1167,439 +1167,3 @@ _exit:
   }
   return code;
 }
-
-// snap part
-static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) {
-  if (x->fid < y->fid) return -1;
-  if (x->fid > y->fid) return 1;
-  return 0;
-}
-
-static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) {
-  if (x->minVer < y->minVer) return -1;
-  if (x->minVer > y->minVer) return 1;
-  if (x->maxVer < y->maxVer) return -1;
-  if (x->maxVer > y->maxVer) return 1;
-  return 0;
-}
-
-static int32_t tsdbSnapRangeCmprFn(STSnapRange* x, STSnapRange* y) {
-  if (x->fid < y->fid) return -1;
-  if (x->fid > y->fid) return 1;
-  return 0;
-}
-
-STsdbSnapPartition* tsdbSnapPartitionCreate() {
-  STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition));
-  if (pSP == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-  for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) {
-    TARRAY2_INIT(&pSP->verRanges[i]);
-  }
-  return pSP;
-}
-
-void tsdbSnapPartitionClear(STsdbSnapPartition** ppSP) {
-  if (ppSP == NULL || ppSP[0] == NULL) {
-    return;
-  }
-  for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) {
-    TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL);
-  }
-  taosMemoryFree(ppSP[0]);
-  ppSP[0] = NULL;
-}
-
-static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) {
-  switch (ftype) {
-    case TSDB_FTYPE_HEAD:
-      return TSDB_SNAP_RANGE_TYP_HEAD;
-    case TSDB_FTYPE_DATA:
-      return TSDB_SNAP_RANGE_TYP_DATA;
-    case TSDB_FTYPE_SMA:
-      return TSDB_SNAP_RANGE_TYP_SMA;
-    case TSDB_FTYPE_TOMB:
-      return TSDB_SNAP_RANGE_TYP_TOMB;
-    case TSDB_FTYPE_STT:
-      return TSDB_SNAP_RANGE_TYP_STT;
-  }
-  return TSDB_SNAP_RANGE_TYP_MAX;
-}
-
-static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP) {
-  STsdbSnapPartition* p = tsdbSnapPartitionCreate();
-  if (p == NULL) {
-    goto _err;
-  }
-
-  p->fid = fset->fid;
-
-  int32_t code = 0;
-  int32_t typ = 0;
-  int32_t corrupt = false;
-  int32_t count = 0;
-  for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
-    if (fset->farr[ftype] == NULL) continue;
-    typ = tsdbFTypeToSRangeTyp(ftype);
-    ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX);
-    STFile* f = fset->farr[ftype]->f;
-    if (f->maxVer > fset->maxVerValid) {
-      corrupt = true;
-      tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64
-                ", ftype: %d",
-                fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype);
-      continue;
-    }
-    count++;
-    SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer};
-    code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
-    ASSERT(code == 0);
-  }
-
-  typ = TSDB_SNAP_RANGE_TYP_STT;
-  const SSttLvl* lvl;
-  TARRAY2_FOREACH(fset->lvlArr, lvl) {
-    STFileObj* fobj;
-    TARRAY2_FOREACH(lvl->fobjArr, fobj) {
-      STFile* f = fobj->f;
-      if (f->maxVer > fset->maxVerValid) {
-        corrupt = true;
-        tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64
-                  ", ftype: %d",
-                  fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ);
-        continue;
-      }
-      count++;
-      SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer};
-      code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
-      ASSERT(code == 0);
-    }
-  }
-  if (corrupt && count == 0) {
-    SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid};
-    code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
-    ASSERT(code == 0);
-  }
-  ppSP[0] = p;
-  return 0;
-
-_err:
-  tsdbSnapPartitionClear(&p);
-  return -1;
-}
-
-STsdbSnapPartList* tsdbSnapPartListCreate() {
-  STsdbSnapPartList* pList = taosMemoryCalloc(1, sizeof(STsdbSnapPartList));
-  if (pList == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-  TARRAY2_INIT(pList);
-  return pList;
-}
-
-static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) {
-  STsdbSnapPartList* pList = tsdbSnapPartListCreate();
-  if (pList == NULL) {
-    return NULL;
-  }
-
-  int32_t code = 0;
-  taosThreadMutexLock(&fs->tsdb->mutex);
-  STFileSet* fset;
-  TARRAY2_FOREACH(fs->fSetArr, fset) {
-    STsdbSnapPartition* pItem = NULL;
-    if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) {
-      code = -1;
-      break;
-    }
-    ASSERT(pItem != NULL);
-    code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn);
-    ASSERT(code == 0);
-  }
-  taosThreadMutexUnlock(&fs->tsdb->mutex);
-
-  if (code) {
-    TARRAY2_DESTROY(pList, tsdbSnapPartitionClear);
-    taosMemoryFree(pList);
-    pList = NULL;
-  }
-  return pList;
-}
-
-int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) {
-  int32_t hdrLen = sizeof(int32_t);
-  int32_t datLen = 0;
-
-  int8_t  msgVer = 1;
-  int32_t len = TARRAY2_SIZE(pList);
-  hdrLen += sizeof(msgVer);
-  hdrLen += sizeof(len);
-  datLen += hdrLen;
-
-  for (int32_t u = 0; u < len; u++) {
-    STsdbSnapPartition* p = TARRAY2_GET(pList, u);
-    int32_t             typMax = TSDB_SNAP_RANGE_TYP_MAX;
-    int32_t             uItem = 0;
-    uItem += sizeof(STsdbSnapPartition);
-    uItem += sizeof(typMax);
-
-    for (int32_t i = 0; i < typMax; i++) {
-      int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]);
-      int32_t jItem = 0;
-      jItem += sizeof(SVersionRange);
-      jItem += sizeof(int64_t);
-      uItem += sizeof(iLen) + jItem * iLen;
-    }
-    datLen += uItem;
-  }
-  return datLen;
-}
-
-int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) {
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
-
-  int8_t  reserved8 = 0;
-  int16_t reserved16 = 0;
-  int64_t reserved64 = 0;
-
-  int8_t  msgVer = 1;
-  int32_t len = TARRAY2_SIZE(pList);
-
-  if (tStartEncode(&encoder) < 0) goto _err;
-  if (tEncodeI8(&encoder, msgVer) < 0) goto _err;
-  if (tEncodeI32(&encoder, len) < 0) goto _err;
-
-  for (int32_t u = 0; u < len; u++) {
-    STsdbSnapPartition* p = TARRAY2_GET(pList, u);
-    if (tEncodeI64(&encoder, p->fid) < 0) goto _err;
-    if (tEncodeI8(&encoder, p->stat) < 0) goto _err;
-    if (tEncodeI8(&encoder, reserved8) < 0) goto _err;
-    if (tEncodeI16(&encoder, reserved16) < 0) goto _err;
-
-    int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX;
-    if (tEncodeI32(&encoder, typMax) < 0) goto _err;
-
-    for (int32_t i = 0; i < typMax; i++) {
-      SVerRangeList* iList = &p->verRanges[i];
-      int32_t        iLen = TARRAY2_SIZE(iList);
-
-      if (tEncodeI32(&encoder, iLen) < 0) goto _err;
-      for (int32_t j = 0; j < iLen; j++) {
-        SVersionRange r = TARRAY2_GET(iList, j);
-        if (tEncodeI64(&encoder, r.minVer) < 0) goto _err;
-        if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err;
-        if (tEncodeI64(&encoder, reserved64) < 0) goto _err;
-      }
-    }
-  }
-
-  tEndEncode(&encoder);
-  int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
-  return tlen;
-
-_err:
-  tEncoderClear(&encoder);
-  return -1;
-}
-
-int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
-
-  int8_t  reserved8 = 0;
-  int16_t reserved16 = 0;
-  int64_t reserved64 = 0;
-
-  STsdbSnapPartition* p = NULL;
-
-  int8_t  msgVer = 0;
-  int32_t len = 0;
-  if (tStartDecode(&decoder) < 0) goto _err;
-  if (tDecodeI8(&decoder, &msgVer) < 0) goto _err;
-  if (tDecodeI32(&decoder, &len) < 0) goto _err;
-
-  for (int32_t u = 0; u < len; u++) {
-    p = tsdbSnapPartitionCreate();
-    if (p == NULL) goto _err;
-    if (tDecodeI64(&decoder, &p->fid) < 0) goto _err;
-    if (tDecodeI8(&decoder, &p->stat) < 0) goto _err;
-    if (tDecodeI8(&decoder, &reserved8) < 0) goto _err;
-    if (tDecodeI16(&decoder, &reserved16) < 0) goto _err;
-
-    int32_t typMax = 0;
-    if (tDecodeI32(&decoder, &typMax) < 0) goto _err;
-
-    for (int32_t i = 0; i < typMax; i++) {
-      SVerRangeList* iList = &p->verRanges[i];
-      int32_t        iLen = 0;
-      if (tDecodeI32(&decoder, &iLen) < 0) goto _err;
-      for (int32_t j = 0; j < iLen; j++) {
-        SVersionRange r = {0};
-        if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err;
-        if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err;
-        if (tDecodeI64(&decoder, &reserved64) < 0) goto _err;
-        TARRAY2_APPEND(iList, r);
-      }
-    }
-    TARRAY2_APPEND(pList, p);
-    p = NULL;
-  }
-
-  tEndDecode(&decoder);
-  tDecoderClear(&decoder);
-  return 0;
-
-_err:
-  if (p) {
-    tsdbSnapPartitionClear(&p);
-  }
-  tDecoderClear(&decoder);
-  return -1;
-}
-
-int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** ppRanges) {
-  TSnapRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TSnapRangeArray));
-  if (pDiff == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-  TARRAY2_INIT(pDiff);
-
-  STsdbSnapPartition* part;
-  TARRAY2_FOREACH(pList, part) {
-    STSnapRange* r = taosMemoryCalloc(1, sizeof(STSnapRange));
-    if (r == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      goto _err;
-    }
-    int64_t maxVerValid = -1;
-    int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX;
-    for (int32_t i = 0; i < typMax; i++) {
-      SVerRangeList* iList = &part->verRanges[i];
-      SVersionRange  vr = {0};
-      TARRAY2_FOREACH(iList, vr) {
-        if (vr.maxVer < vr.minVer) {
-          continue;
-        }
-        maxVerValid = TMAX(maxVerValid, vr.maxVer);
-      }
-    }
-    r->fid = part->fid;
-    r->sver = maxVerValid + 1;
-    r->ever = VERSION_MAX;
-    tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever);
-    int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbSnapRangeCmprFn);
-    ASSERT(code == 0);
-  }
-  ppRanges[0] = pDiff;
-
-  tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff));
-  return 0;
-
-_err:
-  if (pDiff) {
-    tsdbSnapRangeArrayDestroy(&pDiff);
-  }
-  return -1;
-}
-
-void tsdbSnapRangeArrayDestroy(TSnapRangeArray** ppSnap) {
-  if (ppSnap && ppSnap[0]) {
-    TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear);
-    taosMemoryFree(ppSnap[0]);
-    ppSnap[0] = NULL;
-  }
-}
-
-void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) {
-  if (ppList == NULL || ppList[0] == NULL) return;
-
-  TARRAY2_DESTROY(ppList[0], tsdbSnapPartitionClear);
-  taosMemoryFree(ppList[0]);
-  ppList[0] = NULL;
-}
-
-ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) {
-  if (!VND_IS_RSMA(pVnode)) {
-    return pVnode->pTsdb->pFS->fsstate;
-  }
-  for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) {
-    STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl);
-    if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) {
-      return TSDB_FS_STATE_INCOMPLETE;
-    }
-  }
-  return TSDB_FS_STATE_NORMAL;
-}
-
-int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) {
-  int                code = -1;
-  int32_t            tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX);
-  int32_t            subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2};
-  STsdbSnapPartList* pLists[TSDB_RETENTION_MAX] = {0};
-
-  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
-    STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j);
-    pLists[j] = tsdbGetSnapPartList(pTsdb->pFS);
-    if (pLists[j] == NULL) goto _out;
-  }
-
-  // estimate bufLen and prepare
-  int32_t bufLen = sizeof(SSyncTLV);  // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY
-  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
-    bufLen += sizeof(SSyncTLV);  // subTyps[j]
-    bufLen += tTsdbSnapPartListDataLenCalc(pLists[j]);
-  }
-
-  tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen);
-
-  void* data = taosMemoryRealloc(pSnap->data, bufLen);
-  if (data == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen);
-    goto _out;
-  }
-  pSnap->data = data;
-
-  // header
-  SSyncTLV* head = data;
-  head->len = 0;
-  head->typ = pSnap->type;
-  int32_t offset = sizeof(SSyncTLV);
-  int32_t tlen = 0;
-
-  // fill snapshot info
-  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
-    if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
-    }
-
-    //  subHead
-    SSyncTLV* subHead = (void*)((char*)data + offset);
-    subHead->typ = subTyps[j];
-    ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV));
-
-    if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) {
-      tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr());
-      goto _out;
-    }
-    subHead->len = tlen;
-    offset += sizeof(SSyncTLV) + tlen;
-  }
-
-  head->len = offset - sizeof(SSyncTLV);
-  ASSERT(offset <= bufLen);
-  code = 0;
-
-_out:
-  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
-    if (pLists[j] == NULL) continue;
-    tsdbSnapPartListDestroy(&pLists[j]);
-  }
-
-  return code;
-}

source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c

@@ -0,0 +1,599 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdb.h"
+#include "tsdbDataFileRAW.h"
+#include "tsdbFS2.h"
+#include "tsdbFSetRAW.h"
+
+static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader);
+
+// reader
+typedef struct SDataFileRAWReaderIter {
+  int32_t count;
+  int32_t idx;
+} SDataFileRAWReaderIter;
+
+typedef struct STsdbSnapRAWReader {
+  STsdb*  tsdb;
+  int64_t ever;
+  int8_t  type;
+
+  TFileSetArray* fsetArr;
+
+  // context
+  struct {
+    int32_t    fsetArrIdx;
+    STFileSet* fset;
+    bool       isDataDone;
+  } ctx[1];
+
+  // reader
+  SDataFileRAWReaderArray dataReaderArr[1];
+
+  // iter
+  SDataFileRAWReaderIter dataIter[1];
+} STsdbSnapRAWReader;
+
+int32_t tsdbSnapRAWReaderOpen(STsdb* tsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  reader[0] = taosMemoryCalloc(1, sizeof(STsdbSnapRAWReader));
+  if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+
+  reader[0]->tsdb = tsdb;
+  reader[0]->ever = ever;
+  reader[0]->type = type;
+
+  code = tsdbFSCreateRefSnapshot(tsdb->pFS, &reader[0]->fsetArr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s, sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__,
+              lino, tstrerror(code), ever, type);
+    tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr);
+    taosMemoryFree(reader[0]);
+    reader[0] = NULL;
+  } else {
+    tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, type);
+  }
+  return code;
+}
+
+int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** reader) {
+  if (reader[0] == NULL) return 0;
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STsdb* tsdb = reader[0]->tsdb;
+
+  TARRAY2_DESTROY(reader[0]->dataReaderArr, tsdbDataFileRAWReaderClose);
+  tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr);
+  taosMemoryFree(reader[0]);
+  reader[0] = NULL;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
+  } else {
+    tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadFileSetOpenReader(STsdbSnapRAWReader* reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  // data
+  for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ftype++) {
+    if (reader->ctx->fset->farr[ftype] == NULL) {
+      continue;
+    }
+    STFileObj*               fobj = reader->ctx->fset->farr[ftype];
+    SDataFileRAWReader*      dataReader;
+    SDataFileRAWReaderConfig config = {
+        .tsdb = reader->tsdb,
+        .szPage = reader->tsdb->pVnode->config.tsdbPageSize,
+        .file = fobj->f[0],
+    };
+    code = tsdbDataFileRAWReaderOpen(NULL, &config, &dataReader);
+    TSDB_CHECK_CODE(code, lino, _exit);
+
+    code = TARRAY2_APPEND(reader->dataReaderArr, dataReader);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  // stt
+  SSttLvl* lvl;
+  TARRAY2_FOREACH(reader->ctx->fset->lvlArr, lvl) {
+    STFileObj* fobj;
+    TARRAY2_FOREACH(lvl->fobjArr, fobj) {
+      SDataFileRAWReader*      dataReader;
+      SDataFileRAWReaderConfig config = {
+          .tsdb = reader->tsdb,
+          .szPage = reader->tsdb->pVnode->config.tsdbPageSize,
+          .file = fobj->f[0],
+      };
+      code = tsdbDataFileRAWReaderOpen(NULL, &config, &dataReader);
+      TSDB_CHECK_CODE(code, lino, _exit);
+
+      code = TARRAY2_APPEND(reader->dataReaderArr, dataReader);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  TARRAY2_CLEAR(reader->dataReaderArr, tsdbDataFileRAWReaderClose);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadFileSetOpenIter(STsdbSnapRAWReader* reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  reader->dataIter->count = TARRAY2_SIZE(reader->dataReaderArr);
+  reader->dataIter->idx = 0;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) {
+  reader->dataIter->count = 0;
+  reader->dataIter->idx = 0;
+  return 0;
+}
+
+static int64_t tsdbSnapRAWReadPeek(SDataFileRAWReader* reader) {
+  int64_t size = TMIN(reader->config->file.size - reader->ctx->offset, TSDB_SNAP_DATA_PAYLOAD_SIZE);
+  return size;
+}
+
+static SDataFileRAWReader* tsdbSnapRAWReaderIterNext(STsdbSnapRAWReader* reader) {
+  ASSERT(reader->dataIter->idx <= reader->dataIter->count);
+
+  while (reader->dataIter->idx < reader->dataIter->count) {
+    SDataFileRAWReader* dataReader = TARRAY2_GET(reader->dataReaderArr, reader->dataIter->idx);
+    ASSERT(dataReader);
+    if (dataReader->ctx->offset < dataReader->config->file.size) {
+      return dataReader;
+    }
+    reader->dataIter->idx++;
+  }
+  return NULL;
+}
+
+static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, SSnapDataHdr** ppData) {
+  int32_t code = 0;
+  int32_t lino = 0;
+  int8_t  type = reader->type;
+  ppData[0] = NULL;
+
+  SDataFileRAWReader* dataReader = tsdbSnapRAWReaderIterNext(reader);
+  if (dataReader == NULL) {
+    return 0;
+  }
+
+  // prepare
+  int64_t dataLength = tsdbSnapRAWReadPeek(dataReader);
+  ASSERT(dataLength > 0);
+
+  void* pBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + sizeof(STsdbDataRAWBlockHeader) + dataLength);
+  if (pBuf == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  SSnapDataHdr* pHdr = pBuf;
+  pHdr->type = type;
+  pHdr->size = sizeof(STsdbDataRAWBlockHeader) + dataLength;
+
+  // read
+  STsdbDataRAWBlockHeader* pBlock = (void*)pHdr->data;
+  pBlock->offset = dataReader->ctx->offset;
+  pBlock->dataLength = dataLength;
+
+  code = tsdbDataFileRAWReadBlockData(dataReader, pBlock);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  // finish
+  dataReader->ctx->offset += pBlock->dataLength;
+  ASSERT(dataReader->ctx->offset <= dataReader->config->file.size);
+  ppData[0] = pBuf;
+
+_exit:
+  if (code) {
+    taosMemoryFree(pBuf);
+    pBuf = NULL;
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadData(STsdbSnapRAWReader* reader, uint8_t** ppData) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbSnapRAWReadNext(reader, (SSnapDataHdr**)ppData);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadBegin(STsdbSnapRAWReader* reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  ASSERT(reader->ctx->fset == NULL);
+
+  if (reader->ctx->fsetArrIdx < TARRAY2_SIZE(reader->fsetArr)) {
+    reader->ctx->fset = TARRAY2_GET(reader->fsetArr, reader->ctx->fsetArrIdx++);
+    reader->ctx->isDataDone = false;
+
+    code = tsdbSnapRAWReadFileSetOpenReader(reader);
+    TSDB_CHECK_CODE(code, lino, _exit);
+
+    code = tsdbSnapRAWReadFileSetOpenIter(reader);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadEnd(STsdbSnapRAWReader* reader) {
+  tsdbSnapRAWReadFileSetCloseIter(reader);
+  tsdbSnapRAWReadFileSetCloseReader(reader);
+  reader->ctx->fset = NULL;
+  return 0;
+}
+
+int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* reader, uint8_t** data) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  data[0] = NULL;
+
+  for (;;) {
+    if (reader->ctx->fset == NULL) {
+      code = tsdbSnapRAWReadBegin(reader);
+      TSDB_CHECK_CODE(code, lino, _exit);
+
+      if (reader->ctx->fset == NULL) {
+        break;
+      }
+    }
+
+    if (!reader->ctx->isDataDone) {
+      code = tsdbSnapRAWReadData(reader, data);
+      TSDB_CHECK_CODE(code, lino, _exit);
+      if (data[0]) {
+        goto _exit;
+      } else {
+        reader->ctx->isDataDone = true;
+      }
+    }
+
+    code = tsdbSnapRAWReadEnd(reader);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  } else {
+    tsdbDebug("vgId:%d %s done", TD_VID(reader->tsdb->pVnode), __func__);
+  }
+  return code;
+}
+
+// writer
+struct STsdbSnapRAWWriter {
+  STsdb*  tsdb;
+  int64_t sver;
+  int64_t ever;
+  int32_t minutes;
+  int8_t  precision;
+  int32_t minRow;
+  int32_t maxRow;
+  int8_t  cmprAlg;
+  int64_t commitID;
+  int32_t szPage;
+  int64_t compactVersion;
+  int64_t now;
+
+  TFileSetArray* fsetArr;
+  TFileOpArray   fopArr[1];
+
+  struct {
+    bool       fsetWriteBegin;
+    int32_t    fid;
+    STFileSet* fset;
+    SDiskID    did;
+    int64_t    cid;
+    int64_t    level;
+
+    // writer
+    SFSetRAWWriter* fsetWriter;
+  } ctx[1];
+};
+
+int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  // start to write
+  writer[0] = taosMemoryCalloc(1, sizeof(*writer[0]));
+  if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+
+  writer[0]->tsdb = pTsdb;
+  writer[0]->ever = ever;
+  writer[0]->minutes = pTsdb->keepCfg.days;
+  writer[0]->precision = pTsdb->keepCfg.precision;
+  writer[0]->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
+  writer[0]->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
+  writer[0]->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
+  writer[0]->commitID = tsdbFSAllocEid(pTsdb->pFS);
+  writer[0]->szPage = pTsdb->pVnode->config.tsdbPageSize;
+  writer[0]->compactVersion = INT64_MAX;
+  writer[0]->now = taosGetTimestampMs();
+
+  code = tsdbFSCreateCopySnapshot(pTsdb->pFS, &writer[0]->fsetArr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+  } else {
+    tsdbInfo("vgId:%d %s done, sver:0, ever:%" PRId64, TD_VID(pTsdb->pVnode), __func__, ever);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteFileSetOpenIter(STsdbSnapRAWWriter* writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteFileSetCloseIter(STsdbSnapRAWWriter* writer) { return 0; }
+
+static int32_t tsdbSnapRAWWriteFileSetOpenWriter(STsdbSnapRAWWriter* writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  SFSetRAWWriterConfig config = {
+      .tsdb = writer->tsdb,
+      .szPage = writer->szPage,
+      .fid = writer->ctx->fid,
+      .cid = writer->commitID,
+      .did = writer->ctx->did,
+      .level = writer->ctx->level,
+  };
+
+  code = tsdbFSetRAWWriterOpen(&config, &writer->ctx->fsetWriter);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteFileSetCloseWriter(STsdbSnapRAWWriter* writer) {
+  return tsdbFSetRAWWriterClose(&writer->ctx->fsetWriter, 0, writer->fopArr);
+}
+
+static int32_t tsdbSnapRAWWriteFileSetBegin(STsdbSnapRAWWriter* writer, int32_t fid) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  ASSERT(writer->ctx->fsetWriteBegin == false);
+
+  STFileSet* fset = &(STFileSet){.fid = fid};
+
+  writer->ctx->fid = fid;
+  STFileSet** fsetPtr = TARRAY2_SEARCH(writer->fsetArr, &fset, tsdbTFileSetCmprFn, TD_EQ);
+  writer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr;
+
+  int32_t level = tsdbFidLevel(fid, &writer->tsdb->keepCfg, taosGetTimestampSec());
+  if (tfsAllocDisk(writer->tsdb->pVnode->pTfs, level, &writer->ctx->did)) {
+    code = TSDB_CODE_NO_AVAIL_DISK;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did);
+
+  code = tsdbSnapRAWWriteFileSetOpenWriter(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->level = level;
+  writer->ctx->fsetWriteBegin = true;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteFileSetEnd(STsdbSnapRAWWriter* writer) {
+  if (!writer->ctx->fsetWriteBegin) return 0;
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  // close write
+  code = tsdbSnapRAWWriteFileSetCloseWriter(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->fsetWriteBegin = false;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbSnapRAWWriteFileSetEnd(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  code = tsdbFSEditBegin(writer->tsdb->pFS, writer->fopArr, TSDB_FEDIT_COMMIT);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  } else {
+    tsdbDebug("vgId:%d %s done", TD_VID(writer->tsdb->pVnode), __func__);
+  }
+  return code;
+}
+
+int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** writer, int8_t rollback) {
+  if (writer[0] == NULL) return 0;
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STsdb* tsdb = writer[0]->tsdb;
+
+  if (rollback) {
+    code = tsdbFSEditAbort(writer[0]->tsdb->pFS);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  } else {
+    taosThreadMutexLock(&writer[0]->tsdb->mutex);
+
+    code = tsdbFSEditCommit(writer[0]->tsdb->pFS);
+    if (code) {
+      taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+
+    writer[0]->tsdb->pFS->fsstate = TSDB_FS_STATE_NORMAL;
+
+    taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
+  }
+
+  TARRAY2_DESTROY(writer[0]->fopArr, NULL);
+  tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr);
+
+  taosMemoryFree(writer[0]);
+  writer[0] = NULL;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
+  } else {
+    tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbDataRAWBlockHeader* bHdr) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteData(STsdbSnapRAWWriter* writer, SSnapDataHdr* hdr) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STsdbDataRAWBlockHeader* bHdr = (void*)hdr->data;
+  int32_t                  fid = bHdr->file.fid;
+  if (!writer->ctx->fsetWriteBegin || fid != writer->ctx->fid) {
+    code = tsdbSnapRAWWriteFileSetEnd(writer);
+    TSDB_CHECK_CODE(code, lino, _exit);
+
+    code = tsdbSnapRAWWriteFileSetBegin(writer, fid);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  code = tsdbSnapRAWWriteTimeSeriesData(writer, bHdr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* writer, SSnapDataHdr* hdr) {
+  ASSERT(hdr->type == SNAP_DATA_RAW);
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbSnapRAWWriteData(writer, hdr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s, type:%d index:%" PRId64 " size:%" PRId64,
+              TD_VID(writer->tsdb->pVnode), __func__, lino, tstrerror(code), hdr->type, hdr->index, hdr->size);
+  } else {
+    tsdbDebug("vgId:%d %s done, type:%d index:%" PRId64 " size:%" PRId64, TD_VID(writer->tsdb->pVnode), __func__,
+              hdr->type, hdr->index, hdr->size);
+  }
+  return code;
+}

source/dnode/vnode/src/vnd/vnodeAsync.c

@@ -177,6 +177,8 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
 }
 
 static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
+  while (async->queue[0].next != &async->queue[0] || async->queue[1].next != &async->queue[1] ||
+         async->queue[2].next != &async->queue[2]) {
     for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) {
       while (async->queue[i].next != &async->queue[i]) {
         SVATask *task = async->queue[i].next;
@@ -185,6 +187,7 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
         vnodeAsyncTaskDone(async, task);
       }
     }
+  }
   return 0;
 }
 

source/dnode/vnode/src/vnd/vnodeInitApi.c

@@ -42,7 +42,7 @@ void initStorageAPI(SStorageAPI* pAPI) {
 
 void initTsdbReaderAPI(TsdReader* pReader) {
   pReader->tsdReaderOpen = (int32_t(*)(void*, SQueryTableDataCond*, void*, int32_t, SSDataBlock*, void**, const char*,
-                                       bool, SHashObj**))tsdbReaderOpen2;
+                                       SHashObj**))tsdbReaderOpen2;
   pReader->tsdReaderClose = tsdbReaderClose2;
 
   pReader->tsdNextDataBlock = tsdbNextDataBlock2;

source/dnode/vnode/src/vnd/vnodeSnapshot.c

@@ -16,6 +16,26 @@
 #include "tsdb.h"
 #include "vnd.h"
 
+static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) {
+  int32_t            code = -1;
+  STsdbFSetPartList *pList = tsdbFSetPartListCreate();
+  if (pList == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _out;
+  }
+  if (tDeserializeTsdbFSetPartList(buf, bufLen, pList) < 0) {
+    terrno = TSDB_CODE_INVALID_DATA_FMT;
+    goto _out;
+  }
+  if (tsdbFSetPartListToRangeDiff(pList, ppRanges) < 0) {
+    goto _out;
+  }
+  code = 0;
+_out:
+  tsdbFSetPartListDestroy(&pList);
+  return code;
+}
+
 // SVSnapReader ========================================================
 struct SVSnapReader {
   SVnode *pVnode;
@@ -29,8 +49,12 @@ struct SVSnapReader {
   SMetaSnapReader *pMetaReader;
   // tsdb
   int8_t           tsdbDone;
-  TSnapRangeArray *pRanges;
+  TFileSetRangeArray *pRanges;
   STsdbSnapReader *pTsdbReader;
+  // tsdb raw
+  int8_t              tsdbRAWDone;
+  STsdbSnapRAWReader *pTsdbRAWReader;
+
   // tq
   int8_t              tqHandleDone;
   STqSnapReader      *pTqSnapReader;
@@ -45,31 +69,11 @@ struct SVSnapReader {
   SStreamStateReader *pStreamStateReader;
   // rsma
   int8_t           rsmaDone;
-  TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
+  TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
   SRSmaSnapReader *pRsmaReader;
 };
 
-static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TSnapRangeArray **ppRanges) {
+static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) {
-  int32_t            code = -1;
-  STsdbSnapPartList *pList = tsdbSnapPartListCreate();
-  if (pList == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    goto _out;
-  }
-  if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) {
-    terrno = TSDB_CODE_INVALID_DATA_FMT;
-    goto _out;
-  }
-  if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) {
-    goto _out;
-  }
-  code = 0;
-_out:
-  tsdbSnapPartListDestroy(&pList);
-  return code;
-}
-
-static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) {
   ASSERTS(sizeof(pReader->pRsmaRanges) / sizeof(pReader->pRsmaRanges[0]) == 2, "Unexpected array size");
   switch (tsdbTyp) {
     case SNAP_DATA_TSDB:
@@ -83,18 +87,20 @@ static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int
   }
 }
 
-static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) {
+static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) {
   SVnode *pVnode = pReader->pVnode;
   int32_t code = -1;
 
   if (pParam->data) {
+    // decode
     SSyncTLV *datHead = (void *)pParam->data;
     if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
       terrno = TSDB_CODE_INVALID_DATA_FMT;
       goto _out;
     }
 
-    TSnapRangeArray **ppRanges = NULL;
+    STsdbRepOpts         tsdbOpts = {0};
+    TFileSetRangeArray **ppRanges = NULL;
     int32_t              offset = 0;
 
     while (offset + sizeof(SSyncTLV) < datHead->len) {
@@ -102,18 +108,45 @@ static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam *
       offset += sizeof(SSyncTLV) + subField->len;
       void   *buf = subField->val;
       int32_t bufLen = subField->len;
+
+      switch (subField->typ) {
+        case SNAP_DATA_TSDB:
+        case SNAP_DATA_RSMA1:
+        case SNAP_DATA_RSMA2: {
           ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ);
           if (ppRanges == NULL) {
-        vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
+            vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
             goto _out;
           }
           if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
             vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
             goto _out;
           }
+        } break;
+        case SNAP_DATA_RAW: {
+          if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) {
+            vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
+            goto _out;
+          }
+        } break;
+        default:
+          vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
+          goto _out;
       }
     }
 
+    // toggle snap replication mode
+    vInfo("vgId:%d, vnode snap reader supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format);
+    if (pReader->sver == 0 && tsdbOpts.format == TSDB_SNAP_REP_FMT_RAW) {
+      pReader->tsdbDone = true;
+    } else {
+      pReader->tsdbRAWDone = true;
+    }
+
+    ASSERT(pReader->tsdbDone != pReader->tsdbRAWDone);
+    vInfo("vgId:%d, vnode snap writer enabled replication mode: %s", TD_VID(pVnode),
+          (pReader->tsdbDone ? "raw" : "normal"));
+  }
   code = 0;
 _out:
   return code;
@@ -135,7 +168,7 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader
   pReader->ever = ever;
 
   // snapshot info
-  if (vnodeSnapReaderDoSnapInfo(pReader, pParam) < 0) {
+  if (vnodeSnapReaderDealWithSnapInfo(pReader, pParam) < 0) {
     goto _err;
   }
 
@@ -152,9 +185,9 @@ _err:
 static void vnodeSnapReaderDestroyTsdbRanges(SVSnapReader *pReader) {
   int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2};
   for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) {
-    TSnapRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]);
+    TFileSetRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]);
     if (ppRanges == NULL) continue;
-    tsdbSnapRangeArrayDestroy(ppRanges);
+    tsdbTFileSetRangeArrayDestroy(ppRanges);
   }
 }
 
@@ -170,6 +203,10 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) {
     tsdbSnapReaderClose(&pReader->pTsdbReader);
   }
 
+  if (pReader->pTsdbRAWReader) {
+    tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader);
+  }
+
   if (pReader->pMetaReader) {
     metaSnapReaderClose(&pReader->pMetaReader);
   }
@@ -285,6 +322,28 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
     }
   }
 
+  if (!pReader->tsdbRAWDone) {
+    // open if not
+    if (pReader->pTsdbRAWReader == NULL) {
+      ASSERT(pReader->sver == 0);
+      code = tsdbSnapRAWReaderOpen(pReader->pVnode->pTsdb, pReader->ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
+      if (code) goto _err;
+    }
+
+    code = tsdbSnapRAWRead(pReader->pTsdbRAWReader, ppData);
+    if (code) {
+      goto _err;
+    } else {
+      if (*ppData) {
+        goto _exit;
+      } else {
+        pReader->tsdbRAWDone = 1;
+        code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader);
+        if (code) goto _err;
+      }
+    }
+  }
+
   // TQ ================
   vInfo("vgId:%d tq transform start", vgId);
   if (!pReader->tqHandleDone) {
@@ -455,8 +514,10 @@ struct SVSnapWriter {
   // meta
   SMetaSnapWriter *pMetaSnapWriter;
   // tsdb
-  TSnapRangeArray *pRanges;
+  TFileSetRangeArray *pRanges;
   STsdbSnapWriter *pTsdbSnapWriter;
+  // tsdb raw
+  STsdbSnapRAWWriter *pTsdbSnapRAWWriter;
   // tq
   STqSnapWriter      *pTqSnapWriter;
   STqOffsetWriter    *pTqOffsetWriter;
@@ -465,11 +526,11 @@ struct SVSnapWriter {
   SStreamTaskWriter  *pStreamTaskWriter;
   SStreamStateWriter *pStreamStateWriter;
   // rsma
-  TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
+  TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
   SRSmaSnapWriter *pRsmaSnapWriter;
 };
 
-TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) {
+TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) {
   ASSERTS(sizeof(pWriter->pRsmaRanges) / sizeof(pWriter->pRsmaRanges[0]) == 2, "Unexpected array size");
   switch (tsdbTyp) {
     case SNAP_DATA_TSDB:
@@ -483,7 +544,7 @@ TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t ts
   }
 }
 
-static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) {
+static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) {
   SVnode *pVnode = pWriter->pVnode;
   int32_t code = -1;
 
@@ -494,7 +555,8 @@ static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *
       goto _out;
     }
 
-    TSnapRangeArray **ppRanges = NULL;
+    STsdbRepOpts         tsdbOpts = {0};
+    TFileSetRangeArray **ppRanges = NULL;
     int32_t           offset = 0;
 
     while (offset + sizeof(SSyncTLV) < datHead->len) {
@@ -502,16 +564,34 @@ static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *
       offset += sizeof(SSyncTLV) + subField->len;
       void   *buf = subField->val;
       int32_t bufLen = subField->len;
+
+      switch (subField->typ) {
+        case SNAP_DATA_TSDB:
+        case SNAP_DATA_RSMA1:
+        case SNAP_DATA_RSMA2: {
           ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ);
           if (ppRanges == NULL) {
-        vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
+            vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
             goto _out;
           }
           if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
             vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
             goto _out;
           }
+        } break;
+        case SNAP_DATA_RAW: {
+          if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) {
+            vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
+            goto _out;
           }
+        } break;
+        default:
+          vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
+          goto _out;
+      }
+    }
+
+    vInfo("vgId:%d, vnode snap writer supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format);
   }
 
   code = 0;
@@ -558,7 +638,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter
   pWriter->commitID = ++pVnode->state.commitID;
 
   // snapshot info
-  if (vnodeSnapWriterDoSnapInfo(pWriter, pParam) < 0) {
+  if (vnodeSnapWriterDealWithSnapInfo(pWriter, pParam) < 0) {
     goto _err;
   }
 
@@ -576,9 +656,9 @@ _err:
 static void vnodeSnapWriterDestroyTsdbRanges(SVSnapWriter *pWriter) {
   int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2};
   for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) {
-    TSnapRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]);
+    TFileSetRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]);
     if (ppRanges == NULL) continue;
-    tsdbSnapRangeArrayDestroy(ppRanges);
+    tsdbTFileSetRangeArrayDestroy(ppRanges);
   }
 }
 
@@ -593,6 +673,10 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
     tsdbSnapWriterPrepareClose(pWriter->pTsdbSnapWriter);
   }
 
+  if (pWriter->pTsdbSnapRAWWriter) {
+    tsdbSnapRAWWriterPrepareClose(pWriter->pTsdbSnapRAWWriter);
+  }
+
   if (pWriter->pRsmaSnapWriter) {
     rsmaSnapWriterPrepareClose(pWriter->pRsmaSnapWriter);
   }
@@ -629,6 +713,11 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
     if (code) goto _exit;
   }
 
+  if (pWriter->pTsdbSnapRAWWriter) {
+    code = tsdbSnapRAWWriterClose(&pWriter->pTsdbSnapRAWWriter, rollback);
+    if (code) goto _exit;
+  }
+
   if (pWriter->pTqSnapWriter) {
     code = tqSnapWriterClose(&pWriter->pTqSnapWriter, rollback);
     if (code) goto _exit;
@@ -752,6 +841,17 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
       code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pHdr);
       if (code) goto _err;
     } break;
+    case SNAP_DATA_RAW: {
+      // tsdb
+      if (pWriter->pTsdbSnapRAWWriter == NULL) {
+        ASSERT(pWriter->sver == 0);
+        code = tsdbSnapRAWWriterOpen(pVnode->pTsdb, pWriter->ever, &pWriter->pTsdbSnapRAWWriter);
+        if (code) goto _err;
+      }
+
+      code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr);
+      if (code) goto _err;
+    } break;
     case SNAP_DATA_TQ_HANDLE: {
       // tq handle
       if (pWriter->pTqSnapWriter == NULL) {

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -804,7 +804,7 @@ int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) {
   }
 
   if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
-    code = tsdbSnapGetDetails(pVnode, pSnap);
+    code = tsdbSnapPrepDescription(pVnode, pSnap);
   }
   return code;
 }

source/libs/executor/inc/executorInt.h

@@ -636,6 +636,7 @@ typedef struct SStreamEventAggOperatorInfo {
   bool                isHistoryOp;
   SArray*             historyWins;
   bool                reCkBlock;
+  bool                recvGetAll;
   SSDataBlock*        pCheckpointRes;
   SFilterInfo*        pStartCondInfo;
   SFilterInfo*        pEndCondInfo;
@@ -837,6 +838,8 @@ void     compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeW
 int32_t  releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI);
 void     resetWinRange(STimeWindow* winRange);
 bool     checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts);
+int64_t  getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval);
+void     resetUnCloseSessionWinInfo(SSHashObj* winMap);
 
 int32_t encodeSSessionKey(void** buf, SSessionKey* key);
 void*   decodeSSessionKey(void* buf, SSessionKey* key);

source/libs/executor/src/executil.c

@@ -1734,6 +1734,11 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi
   pCond->endVersion = -1;
   pCond->skipRollup = readHandle->skipRollup;
 
+  // allowed read stt file optimization mode
+  pCond->notLoadData = (pTableScanNode->dataRequired == FUNC_DATA_REQUIRED_NOT_LOAD) &&
+                       (pTableScanNode->scan.node.pConditions == NULL) &&
+                       (pTableScanNode->interval == 0);
+
   int32_t j = 0;
   for (int32_t i = 0; i < pCond->numOfCols; ++i) {
     STargetNode* pNode = (STargetNode*)nodesListGetNode(pTableScanNode->scan.pScanCols, i);

source/libs/executor/src/executor.c

@@ -1232,7 +1232,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       if (pScanBaseInfo->dataReader == NULL) {
         int32_t code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(
             pScanBaseInfo->readHandle.vnode, &pScanBaseInfo->cond, &keyInfo, 1, pScanInfo->pResBlock,
-            (void**)&pScanBaseInfo->dataReader, id, false, NULL);
+            (void**)&pScanBaseInfo->dataReader, id, NULL);
         if (code != TSDB_CODE_SUCCESS) {
           qError("prepare read tsdb snapshot failed, uid:%" PRId64 ", code:%s %s", pOffset->uid, tstrerror(code), id);
           terrno = code;
@@ -1291,7 +1291,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
       int32_t        size = tableListGetSize(pTableListInfo);
 
       pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL,
-                                                    (void**)&pInfo->dataReader, NULL, false, NULL);
+                                                    (void**)&pInfo->dataReader, NULL, NULL);
 
       cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
       strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName);

source/libs/executor/src/scanoperator.c

@@ -889,7 +889,7 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) {
     ASSERT(pInfo->base.dataReader == NULL);
   
     int32_t code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock,
-                                  (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly, &pInfo->pIgnoreTables);
+                                  (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), &pInfo->pIgnoreTables);
     if (code != TSDB_CODE_SUCCESS) {
       T_LONG_JMP(pTaskInfo->env, code);
     }
@@ -1061,7 +1061,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
   pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader;
   initResultSizeInfo(&pOperator->resultInfo, 4096);
   pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
-  //  blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
 
   code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
   if (code != TSDB_CODE_SUCCESS) {
@@ -1183,7 +1182,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
   SSDataBlock* pBlock = pTableScanInfo->pResBlock;
   STsdbReader* pReader = NULL;
   int32_t      code = pAPI->tsdReader.tsdReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock,
-                                     (void**)&pReader, GET_TASKID(pTaskInfo), false, NULL);
+                                     (void**)&pReader, GET_TASKID(pTaskInfo), NULL);
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
     T_LONG_JMP(pTaskInfo->env, code);
@@ -3427,7 +3426,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   int32_t numOfTable = tableEndIdx - tableStartIdx + 1;
   STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx);
   pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock,
-                                (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables);
+                                (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), &pInfo->mSkipTables);
   if (pInfo->filesetDelimited) {
     pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader);
   }

source/libs/executor/src/streameventwindowoperator.c

@@ -176,7 +176,7 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
 
   for (int32_t i = start; i < rows; ++i) {
     if (pTsData[i] >= maxTs) {
-      return i - 1 - start;
+      return i - start;
     }
 
     if (pWin->skey > pTsData[i]) {
@@ -323,6 +323,9 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
       tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
       doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin);
       releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
+      SSessionKey tmpSeInfo = {0};
+      getSessionHashKey(&curWin.winInfo.sessionWin, &tmpSeInfo);
+      tSimpleHashPut(pStDeleted, &tmpSeInfo, sizeof(SSessionKey), NULL, 0);
       continue;
     }
     code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,
@@ -483,6 +486,11 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
       return pInfo->pCheckpointRes;
     }
 
+    if (pInfo->recvGetAll) {
+      pInfo->recvGetAll = false;
+      resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);
+    }
+
     setOperatorCompleted(pOperator);
     return NULL;
   }
@@ -507,6 +515,7 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
       deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted);
       continue;
     } else if (pBlock->info.type == STREAM_GET_ALL) {
+      pInfo->recvGetAll = true;
       getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated);
       continue;
     } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {
@@ -669,6 +678,7 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
       .calTrigger = pEventNode->window.triggerType,
       .maxTs = INT64_MIN,
       .minTs = INT64_MAX,
+      .deleteMark = getDeleteMark(&pEventNode->window, 0),
   };
 
   initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
@@ -717,6 +727,7 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
 
   pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
   pInfo->reCkBlock = false;
+  pInfo->recvGetAll = false;
 
   // for stream
   void*   buff = NULL;

source/libs/executor/src/streamtimewindowoperator.c

@@ -634,9 +634,6 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i
   for (int32_t i = 0; i < size; i++) {
     SWinKey*    winKey = taosArrayGet(wins, i);
     STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
-    if (isOverdue(nextWin.ekey, &pInfo->twAggSup) && pInfo->ignoreExpiredData) {
-      continue;
-    }
     void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
     if (!chIds) {
       SPullWindowInfo pull = {
@@ -801,7 +798,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
   }
   while (1) {
     bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
-    if ((!IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData &&
+    if ((!IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData && pSDataBlock->info.type != STREAM_PULL_DATA &&
          checkExpiredData(&pInfo->stateStore, pInfo->pUpdateInfo, &pInfo->twAggSup, pSDataBlock->info.id.uid,
                           nextWin.ekey)) ||
         !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
@@ -1345,12 +1342,12 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
   return buildIntervalResult(pOperator);
 }
 
-static int64_t getDeleteMark(SIntervalPhysiNode* pIntervalPhyNode) {
+int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval) {
-  if (pIntervalPhyNode->window.deleteMark <= 0) {
+  if (pWinPhyNode->deleteMark <= 0) {
     return DEAULT_DELETE_MARK;
   }
-  int64_t deleteMark = TMAX(pIntervalPhyNode->window.deleteMark, pIntervalPhyNode->window.watermark);
+  int64_t deleteMark = TMAX(pWinPhyNode->deleteMark, pWinPhyNode->watermark);
-  deleteMark = TMAX(deleteMark, pIntervalPhyNode->interval);
+  deleteMark = TMAX(deleteMark, interval);
   return deleteMark;
 }
 
@@ -1442,7 +1439,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
       .calTrigger = pIntervalPhyNode->window.triggerType,
       .maxTs = INT64_MIN,
       .minTs = INT64_MAX,
-      .deleteMark = getDeleteMark(pIntervalPhyNode),
+      .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval),
       .deleteMarkSaved = 0,
       .calTriggerSaved = 0,
   };
@@ -2565,7 +2562,7 @@ void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {
   taosMemoryFree(buf);
 }
 
-static void resetUnCloseSessionWinInfo(SSHashObj* winMap) {
+void resetUnCloseSessionWinInfo(SSHashObj* winMap) {
   void*   pIte = NULL;
   int32_t iter = 0;
   while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) {
@@ -3963,7 +3960,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
                                          .calTrigger = pIntervalPhyNode->window.triggerType,
                                          .maxTs = INT64_MIN,
                                          .minTs = INT64_MAX,
-                                         .deleteMark = getDeleteMark(pIntervalPhyNode)};
+                                         .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};
 
   ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay");
 

source/libs/executor/src/sysscanoperator.c

@@ -2304,7 +2304,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
     void*  pList = tableListGetInfo(pTableListInfo, 0);
 
     code = readHandle->api.tsdReader.tsdReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock,
-                                                   (void**)&pInfo->pHandle, pTaskInfo->id.str, false, NULL);
+                                                   (void**)&pInfo->pHandle, pTaskInfo->id.str, NULL);
     cleanupQueryTableDataCond(&cond);
     if (code != 0) {
       goto _error;

source/libs/sync/inc/syncReplication.h

@@ -56,6 +56,10 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode);
 
 int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
 
+int32_t syncSnapSendMsg(SSyncSnapshotSender* pSender, int32_t seq, void* pBlock, int32_t len, int32_t typ);
+int32_t syncSnapSendRsp(SSyncSnapshotReceiver* pReceiver, SyncSnapshotSend* pMsg, void* pBlock, int32_t len,
+                        int32_t typ, int32_t code);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/sync/src/syncSnapshot.c

@@ -23,8 +23,6 @@
 #include "syncReplication.h"
 #include "syncUtil.h"
 
-int32_t syncSnapSendMsg(SSyncSnapshotSender *pSender, int32_t seq, void *pBlock, int32_t len, int32_t typ);
-
 static void syncSnapBufferReset(SSyncSnapBuffer *pBuf) {
   taosThreadMutexLock(&pBuf->mutex);
   for (int64_t i = pBuf->start; i < pBuf->end; ++i) {
@@ -123,6 +121,11 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
   if (pSender->pSndBuf) {
     syncSnapBufferDestroy(&pSender->pSndBuf);
   }
+
+  if (pSender->snapshotParam.data) {
+    taosMemoryFree(pSender->snapshotParam.data);
+    pSender->snapshotParam.data = NULL;
+  }
   // free sender
   taosMemoryFree(pSender);
 }
@@ -153,7 +156,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) {
   pSender->lastSendTime = taosGetTimestampMs();
   pSender->finish = false;
 
-  // Get full snapshot info
+  // Get snapshot info
   SSyncNode *pSyncNode = pSender->pSyncNode;
   SSnapshot  snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT};
   if (pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo) != 0) {
@@ -161,11 +164,10 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) {
     goto _out;
   }
 
-  int dataLen = 0;
   void   *pData = snapInfo.data;
-  int32_t type = 0;
+  int32_t type = (pData) ? snapInfo.type : 0;
+  int32_t dataLen = 0;
   if (pData) {
-    type = snapInfo.type;
     SSyncTLV *datHead = pData;
     if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT) {
       sSError(pSender, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ);
@@ -347,9 +349,6 @@ _out:;
   return code;
 }
 
-// return 0, start ok
-// return 1, last snapshot finish ok
-// return -1, error
 int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) {
   SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, pDestId);
   if (pSender == NULL) {
@@ -380,6 +379,7 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) {
   return 0;
 }
 
+// receiver
 SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId) {
   bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) &&
                    (pSyncNode->pFsm->FpSnapshotDoWrite != NULL);
@@ -509,8 +509,6 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p
   sRInfo(pReceiver, "snapshot receiver start, from dnode:%d.", DID(&pReceiver->fromId));
 }
 
-// just set start = false
-// FpSnapshotStopWrite should not be called
 void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
   sRDebug(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter);
 
@@ -531,7 +529,6 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
   syncSnapBufferReset(pReceiver->pRcvBuf);
 }
 
-// when recv last snapshot block, apply data into snapshot
 static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) {
   int32_t code = 0;
   if (pReceiver->pWriter != NULL) {
@@ -590,8 +587,6 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
   return 0;
 }
 
-// apply data block
-// update progress
 static int32_t snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) {
   if (pMsg->seq != pReceiver->ack + 1) {
     sRError(pReceiver, "snapshot receiver invalid seq, ack:%d seq:%d", pReceiver->ack, pMsg->seq);
@@ -644,6 +639,50 @@ SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) {
   return snapStart;
 }
 
+static int32_t syncSnapReceiverExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotReceiver *pReceiver,
+                                             SyncSnapshotSend *pMsg, SSnapshot *pInfo) {
+  ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT);
+  int32_t code = 0;
+
+  // copy snap info from leader
+  void *data = taosMemoryCalloc(1, pMsg->dataLen);
+  if (data == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    code = terrno;
+    goto _out;
+  }
+  pInfo->data = data;
+  data = NULL;
+  memcpy(pInfo->data, pMsg->data, pMsg->dataLen);
+
+  // exchange snap info
+  pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, pInfo);
+  SSyncTLV *datHead = pInfo->data;
+  if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
+    sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ);
+    code = TSDB_CODE_INVALID_DATA_FMT;
+    goto _out;
+  }
+  int32_t dataLen = sizeof(SSyncTLV) + datHead->len;
+
+  // save exchanged snap info
+  SSnapshotParam *pParam = &pReceiver->snapshotParam;
+  data = taosMemoryRealloc(pParam->data, dataLen);
+  if (data == NULL) {
+    sError("vgId:%d, failed to realloc memory for snapshot prep due to %s. dataLen:%d", pSyncNode->vgId,
+           strerror(errno), dataLen);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    code = terrno;
+    goto _out;
+  }
+  pParam->data = data;
+  data = NULL;
+  memcpy(pParam->data, pInfo->data, dataLen);
+
+_out:
+  return code;
+}
+
 static int32_t syncNodeOnSnapshotPrep(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
   SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
   int64_t                timeNow = taosGetTimestampMs();
@@ -686,77 +725,27 @@ _START_RECEIVER:
     snapshotReceiverStop(pReceiver);
   }
 
-  snapshotReceiverStart(pReceiver, pMsg);  // set start-time same with sender
+  snapshotReceiverStart(pReceiver, pMsg);
 
-_SEND_REPLY:
+_SEND_REPLY:;
-    // build msg
-    ;  // make complier happy
 
   SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT_REPLY};
   int32_t   dataLen = 0;
-  if (pMsg->dataLen > 0) {
+  if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT) {
-    void *data = taosMemoryCalloc(1, pMsg->dataLen);
+    if (syncSnapReceiverExchgSnapInfo(pSyncNode, pReceiver, pMsg, &snapInfo) != 0) {
-    if (data == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      code = terrno;
       goto _out;
     }
-    memcpy(data, pMsg->data, pMsg->dataLen);
-    snapInfo.data = data;
-    data = NULL;
-    pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo);
-
     SSyncTLV *datHead = snapInfo.data;
-    if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
-      sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ);
-      code = TSDB_CODE_INVALID_DATA_FMT;
-      goto _out;
-    }
     dataLen = sizeof(SSyncTLV) + datHead->len;
   }
 
-  SRpcMsg rpcMsg = {0};
+  // send response
-  if (syncBuildSnapshotSendRsp(&rpcMsg, dataLen, pSyncNode->vgId) != 0) {
+  int32_t type = (snapInfo.data) ? snapInfo.type : 0;
-    sRError(pReceiver, "snapshot receiver failed to build resp since %s", terrstr());
+  if (syncSnapSendRsp(pReceiver, pMsg, snapInfo.data, dataLen, type, code) != 0) {
     code = terrno;
     goto _out;
   }
 
-  SyncSnapshotRsp *pRspMsg = rpcMsg.pCont;
-  pRspMsg->srcId = pSyncNode->myRaftId;
-  pRspMsg->destId = pMsg->srcId;
-  pRspMsg->term = raftStoreGetTerm(pSyncNode);
-  pRspMsg->lastIndex = pMsg->lastIndex;
-  pRspMsg->lastTerm = pMsg->lastTerm;
-  pRspMsg->startTime = pMsg->startTime;
-  pRspMsg->ack = pMsg->seq;  // receiver maybe already closed
-  pRspMsg->code = code;
-  pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode);
-
-  if (snapInfo.data) {
-    pRspMsg->payloadType = snapInfo.type;
-    memcpy(pRspMsg->data, snapInfo.data, dataLen);
-
-    // save snapshot info
-    SSnapshotParam *pParam = &pReceiver->snapshotParam;
-    void           *data = taosMemoryRealloc(pParam->data, dataLen);
-    if (data == NULL) {
-      sError("vgId:%d, failed to realloc memory for snapshot prep due to %s. dataLen:%d", pSyncNode->vgId,
-             strerror(errno), dataLen);
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      code = terrno;
-      goto _out;
-    }
-    pParam->data = data;
-    memcpy(pParam->data, snapInfo.data, dataLen);
-  }
-
-  // send msg
-  if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) {
-    sRError(pReceiver, "failed to send resp since %s", terrstr());
-    code = terrno;
-  }
-
 _out:
   if (snapInfo.data) {
     taosMemoryFree(snapInfo.data);
@@ -793,38 +782,20 @@ _SEND_REPLY:
     code = terrno;
   }
 
-  // build msg
+  // send response
-  SRpcMsg rpcMsg = {0};
+  if (syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, code) != 0) {
-  if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId) != 0) {
-    sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr());
-    return -1;
-  }
-
-  SyncSnapshotRsp *pRspMsg = rpcMsg.pCont;
-  pRspMsg->srcId = pSyncNode->myRaftId;
-  pRspMsg->destId = pMsg->srcId;
-  pRspMsg->term = raftStoreGetTerm(pSyncNode);
-  pRspMsg->lastIndex = pMsg->lastIndex;
-  pRspMsg->lastTerm = pMsg->lastTerm;
-  pRspMsg->startTime = pMsg->startTime;
-  pRspMsg->ack = pReceiver->ack;  // receiver maybe already closed
-  pRspMsg->code = code;
-  pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start;
-
-  // send msg
-  if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) {
-    sRError(pReceiver, "failed to send snapshot receiver resp since %s", terrstr());
     return -1;
   }
 
   return code;
 }
 
-static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, int32_t code) {
+int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, void *pBlock, int32_t blockLen,
+                        int32_t type, int32_t code) {
   SSyncNode *pSyncNode = pReceiver->pSyncNode;
   // build msg
   SRpcMsg rpcMsg = {0};
-  if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId)) {
+  if (syncBuildSnapshotSendRsp(&rpcMsg, blockLen, pSyncNode->vgId)) {
     sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr());
     return -1;
   }
@@ -832,13 +803,18 @@ static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSen
   SyncSnapshotRsp *pRspMsg = rpcMsg.pCont;
   pRspMsg->srcId = pSyncNode->myRaftId;
   pRspMsg->destId = pMsg->srcId;
-  pRspMsg->term = raftStoreGetTerm(pSyncNode);
+  pRspMsg->term = pMsg->term;
   pRspMsg->lastIndex = pMsg->lastIndex;
   pRspMsg->lastTerm = pMsg->lastTerm;
   pRspMsg->startTime = pMsg->startTime;
   pRspMsg->ack = pMsg->seq;
   pRspMsg->code = code;
   pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start;
+  pRspMsg->payloadType = type;
+
+  if (pBlock != NULL && blockLen > 0) {
+    memcpy(pRspMsg->data, pBlock, blockLen);
+  }
 
   // send msg
   if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) {
@@ -872,7 +848,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot
     ppMsg[0] = NULL;
     pRcvBuf->end = TMAX(pMsg->seq + 1, pRcvBuf->end);
   } else if (pMsg->seq < pRcvBuf->start) {
-    syncSnapSendRsp(pReceiver, pMsg, code);
+    syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, code);
     goto _out;
   }
 
@@ -892,7 +868,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot
       }
     }
     pRcvBuf->start = seq + 1;
-    syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], code);
+    syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], NULL, 0, 0, code);
     pRcvBuf->entryDeleteCb(pRcvBuf->entries[seq % pRcvBuf->size]);
     pRcvBuf->entries[seq % pRcvBuf->size] = NULL;
     if (code) goto _out;
@@ -915,7 +891,7 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend
   if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) {
     terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
     sRError(pReceiver, "failed to receive snapshot data since %s.", terrstr());
-    return syncSnapSendRsp(pReceiver, pMsg, terrno);
+    return syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, terrno);
   }
 
   return syncSnapBufferRecv(pReceiver, ppMsg);
@@ -971,26 +947,6 @@ _SEND_REPLY:;
   return code;
 }
 
-// receiver on message
-//
-// condition 1, recv SYNC_SNAPSHOT_SEQ_PREP
-//              if receiver already start
-//                    if sender.start-time > receiver.start-time, restart receiver(reply snapshot start)
-//                    if sender.start-time = receiver.start-time, maybe duplicate msg
-//                    if sender.start-time < receiver.start-time, ignore
-//              else
-//                    waiting for clock match
-//                    start receiver(reply snapshot start)
-//
-// condition 2, recv SYNC_SNAPSHOT_SEQ_BEGIN
-//              a. create writer with <begin, end>
-//
-// condition 3, recv SYNC_SNAPSHOT_SEQ_END, finish receiver(apply snapshot data, update commit index, maybe reconfig)
-//
-// condition 4, recv SYNC_SNAPSHOT_SEQ_FORCE_CLOSE, force close
-//
-// condition 5, got data, update ack
-//
 int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) {
   SyncSnapshotSend **ppMsg = (SyncSnapshotSend **)&pRpcMsg->pCont;
   SyncSnapshotSend      *pMsg = ppMsg[0];
@@ -1074,6 +1030,32 @@ _out:;
   return code;
 }
 
+static int32_t syncSnapSenderExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
+  ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY);
+
+  SSyncTLV *datHead = (void *)pMsg->data;
+  if (datHead->typ != pMsg->payloadType) {
+    sSError(pSender, "unexpected data type in data of SyncSnapshotRsp. typ: %d", datHead->typ);
+    terrno = TSDB_CODE_INVALID_DATA_FMT;
+    return -1;
+  }
+  int32_t dataLen = sizeof(SSyncTLV) + datHead->len;
+
+  SSnapshotParam *pParam = &pSender->snapshotParam;
+  void           *data = taosMemoryRealloc(pParam->data, dataLen);
+  if (data == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  memcpy(data, pMsg->data, dataLen);
+
+  pParam->data = data;
+  data = NULL;
+  sSInfo(pSender, "data of snapshot param. len: %d", datHead->len);
+  return 0;
+}
+
+// sender
 static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
   SSnapshot snapshot = {0};
   pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
@@ -1090,14 +1072,9 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend
 
   // start reader
   if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
-    SSyncTLV *datHead = (void *)pMsg->data;
+    if (syncSnapSenderExchgSnapInfo(pSyncNode, pSender, pMsg) != 0) {
-    if (datHead->typ != pMsg->payloadType) {
-      sSError(pSender, "unexpected data type in data of SyncSnapshotRsp. typ: %d", datHead->typ);
-      terrno = TSDB_CODE_INVALID_DATA_FMT;
       return -1;
     }
-    pSender->snapshotParam.data = (void *)pMsg->data;
-    sSInfo(pSender, "data of snapshot param. len: %d", datHead->len);
   }
 
   int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader);
@@ -1131,7 +1108,7 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp
     goto _out;
   }
 
-  if (pSender->pReader == NULL || pSender->finish) {
+  if (pSender->pReader == NULL || pSender->finish || !snapshotSenderIsStart(pSender)) {
     code = terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
     goto _out;
   }
@@ -1182,12 +1159,6 @@ _out:
   return code;
 }
 
-// sender on message
-//
-// condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender
-// condition 2 sender receives ack, set seq = ack + 1, send msg from seq
-// condition 3 sender receives error msg, just print error log
-//
 int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) {
   SyncSnapshotRsp **ppMsg = (SyncSnapshotRsp **)&pRpcMsg->pCont;
   SyncSnapshotRsp  *pMsg = ppMsg[0];

source/libs/sync/src/syncUtil.c

@@ -487,3 +487,13 @@ void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteRepl
   sNInfo(pSyncNode, "send sync-request-vote-reply to dnode:%d {term:%" PRId64 ", grant:%d}, %s", DID(&pMsg->destId),
          pMsg->term, pMsg->voteGranted, s);
 }
+
+int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) {
+  void* data = taosMemoryRealloc(pSnap->data, size);
+  if (data == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  pSnap->data = data;
+  return 0;
+}

source/libs/transport/src/thttp.c

@@ -34,6 +34,7 @@ typedef struct SHttpModule {
   SAsyncPool* asyncPool;
   TdThread    thread;
   SHashObj*   connStatusTable;
+  int8_t      quit;
 } SHttpModule;
 
 typedef struct SHttpMsg {
@@ -166,7 +167,7 @@ _OVER:
 static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) {
   uint32_t ip = taosGetIpv4FromFqdn(server);
   if (ip == 0xffffffff) {
-    tError("http-report failed to get http server:%s since %s", server, errno == 0 ? "invalid http server" : terrstr());
+    tError("http-report failed to resolving domain names: %s", server);
     return -1;
   }
   char buf[128] = {0};
@@ -190,12 +191,32 @@ static void httpDestroyMsg(SHttpMsg* msg) {
   taosMemoryFree(msg->cont);
   taosMemoryFree(msg);
 }
+
+static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) {
+  SHttpMsg *msg = NULL, *quitMsg = NULL;
+  if (atomic_load_8(&http->quit) == 0) {
+    return;
+  }
+
+  while (!QUEUE_IS_EMPTY(&item->qmsg)) {
+    queue* h = QUEUE_HEAD(&item->qmsg);
+    QUEUE_REMOVE(h);
+    msg = QUEUE_DATA(h, SHttpMsg, q);
+    if (!msg->quit) {
+      httpDestroyMsg(msg);
+    } else {
+      quitMsg = msg;
+    }
+  }
+  if (quitMsg != NULL) {
+    QUEUE_PUSH(&item->qmsg, &quitMsg->q);
+  }
+}
 static void httpAsyncCb(uv_async_t* handle) {
   SAsyncItem*  item = handle->data;
   SHttpModule* http = item->pThrd;
 
   SHttpMsg *msg = NULL, *quitMsg = NULL;
-
   queue     wq;
   QUEUE_INIT(&wq);
 
@@ -203,6 +224,7 @@ static void httpAsyncCb(uv_async_t* handle) {
   int32_t        count = 0;
 
   taosThreadMutexLock(&item->mtx);
+  httpMayDiscardMsg(http, item);
 
   while (!QUEUE_IS_EMPTY(&item->qmsg) && count++ < BATCH_SIZE) {
     queue* h = QUEUE_HEAD(&item->qmsg);
@@ -497,9 +519,10 @@ static void transHttpDestroyHandle(void* handle) { taosMemoryFree(handle); }
 static void transHttpEnvInit() {
   httpRefMgt = taosOpenRef(1, transHttpDestroyHandle);
 
-  SHttpModule* http = taosMemoryMalloc(sizeof(SHttpModule));
+  SHttpModule* http = taosMemoryCalloc(1, sizeof(SHttpModule));
   http->loop = taosMemoryMalloc(sizeof(uv_loop_t));
   http->connStatusTable = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  http->quit = 0;
 
   uv_loop_init(http->loop);
 
@@ -526,6 +549,8 @@ void transHttpEnvDestroy() {
     return;
   }
   SHttpModule* load = taosAcquireRef(httpRefMgt, httpRef);
+
+  atomic_store_8(&load->quit, 1);
   httpSendQuit();
   taosThreadJoin(load->thread, NULL);
 

tests/system-test/0-others/test_create_same_name_db.py

@@ -0,0 +1,38 @@
+import time
+import os
+import platform
+import taos
+import threading
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+
+
+class TDTestCase:
+    """This test case is used to veirfy TD-25762
+    """
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+        self.db_name = "db"
+
+    def run(self):
+        try:
+            # create same name database multiple times
+            for i in range(100):
+                tdLog.debug(f"round {str(i+1)} create database {self.db_name}")
+                tdSql.execute(f"create database {self.db_name}")
+                tdLog.debug(f"round {str(i+1)} drop database {self.db_name}")
+                tdSql.execute(f"drop database {self.db_name}")
+        except Exception as ex:
+            tdLog.exit(str(ex))
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())