From b2f3ff2741367e409905afb954695fa1987870e4 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 4 Jul 2022 06:14:39 +0000
Subject: [PATCH 01/50] more

---
 source/dnode/vnode/inc/vnode.h             | 13 +++--
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 59 ++++++++++++++++++----
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 47 +++++++++++++++++
 3 files changed, 105 insertions(+), 14 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 3418a62f0f..0b06261f0f 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -43,6 +43,7 @@ typedef struct SVnode           SVnode;
 typedef struct STsdbCfg         STsdbCfg;  // todo: remove
 typedef struct SVnodeCfg        SVnodeCfg;
 typedef struct SVSnapshotReader SVSnapshotReader;
+typedef struct SVSnapshotWriter SVSnapshotWriter;
 
 extern const SVnodeCfg vnodeCfgDefault;
 
@@ -66,9 +67,6 @@ void    vnodeStop(SVnode *pVnode);
 int64_t vnodeGetSyncHandle(SVnode *pVnode);
 void    vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
 void    vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId);
-int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever);
-int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
-int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
 int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
 int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
 int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list);
@@ -159,7 +157,14 @@ int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, SStreamReader *pHandle);
 // sma
 int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
 
-// need to reposition
+// SVSnapshotReader
+int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever);
+int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
+int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
+// SVSnapshotWriter;
+int32_t vnodeSnapshotWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapshotWriter **ppWriter);
+int32_t vnodeSnapshotWrite(SVSnapshotWriter *pWriter, uint8_t *pData, uint32_t nData);
+int32_t vnodeSnapshotWriterClose(SVSnapshotWriter *pWriter, int8_t rollback);
 
 // structs
 struct STsdbCfg {
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 79989a5560..c6d1377591 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -16,21 +16,60 @@
 #include "tsdb.h"
 
 struct STsdbSnapshotReader {
-  STsdb* pTsdb;
-  // TODO
+  STsdb*  pTsdb;
+  int64_t sver;
+  int64_t ever;
+  // for data file
+  SDataFReader* pDataFReader;
+  // for del file
+  SDelFReader* pDelFReader;
 };
 
-int32_t tsdbSnapshotReaderOpen(STsdb* pTsdb, STsdbSnapshotReader** ppReader, int64_t sver, int64_t ever) {
-  // TODO
-  return 0;
-}
+typedef struct STsdbSnapshotWriter {
+  STsdb*  pTsdb;
+  int64_t sver;
+  int64_t ever;
+  // for data file
+  SDataFWriter* pDataFWriter;
+  // for del file
+  SDelFWriter* pDelFWriter;
+} STsdbSnapshotWriter;
 
-int32_t tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader) {
-  // TODO
-  return 0;
+int32_t tsdbSnapshotReaderOpen(STsdb* pTsdb, STsdbSnapshotReader** ppReader, int64_t sver, int64_t ever) {
+  int32_t              code = 0;
+  STsdbSnapshotReader* pReader = NULL;
+
+  // alloc
+  pReader = (STsdbSnapshotReader*)taosMemoryCalloc(1, sizeof(*pReader));
+  if (pReader == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pReader->pTsdb = pTsdb;
+  pReader->sver = sver;
+  pReader->ever = ever;
+
+  *ppReader = pReader;
+  return code;
+
+_err:
+  tsdbError("vgId:%d snapshot reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  *ppReader = NULL;
+  return code;
 }
 
 int32_t tsdbSnapshotRead(STsdbSnapshotReader* pReader, void** ppData, uint32_t* nData) {
+  int32_t code = 0;
   // TODO
-  return 0;
+  return code;
+
+_err:
+  tsdbError("vgId:%d snapshot read failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader) {
+  int32_t code = 0;
+  taosMemoryFree(pReader);
+  return code;
 }
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index baa8422307..8d2a316d64 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -27,6 +27,13 @@ struct SVSnapshotReader {
   int32_t              nData;
 };
 
+struct SVSnapshotWriter {
+  SVnode *pVnode;
+  int64_t sver;
+  int64_t ever;
+};
+
+// SVSnapshotReader ========================================================
 int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever) {
   SVSnapshotReader *pReader = NULL;
 
@@ -106,4 +113,44 @@ int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32
 
   code = TSDB_CODE_VND_READ_END;
   return code;
+}
+
+// SVSnapshotWriter ========================================================
+int32_t vnodeSnapshotWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapshotWriter **ppWriter) {
+  int32_t           code = 0;
+  SVSnapshotWriter *pWriter = NULL;
+
+  // alloc
+  pWriter = (SVSnapshotWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->pVnode = pVnode;
+  pWriter->sver = sver;
+  pWriter->ever = ever;
+
+  return code;
+
+_err:
+  return code;
+}
+
+int32_t vnodeSnapshotWrite(SVSnapshotWriter *pWriter, uint8_t *pData, uint32_t nData) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t vnodeSnapshotWriterClose(SVSnapshotWriter *pWriter, int8_t rollback) {
+  int32_t code = 0;
+
+  if (!rollback) {
+    // apply the change
+  } else {
+    // rollback the change
+  }
+
+  taosMemoryFree(pWriter);
+  return code;
 }
\ No newline at end of file

From eb888684e31887a560c214ea5b17e11b1ae9eead Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 4 Jul 2022 06:47:41 +0000
Subject: [PATCH 02/50] more refact

---
 source/dnode/vnode/inc/vnode.h             | 26 ++++-----
 source/dnode/vnode/src/inc/vnodeInt.h      | 34 ++++++------
 source/dnode/vnode/src/meta/metaSnapshot.c | 16 +++---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 25 +++++----
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 62 +++++++++++-----------
 5 files changed, 84 insertions(+), 79 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 0b06261f0f..348c792dd3 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -39,11 +39,11 @@ extern "C" {
 #endif
 
 // vnode
-typedef struct SVnode           SVnode;
-typedef struct STsdbCfg         STsdbCfg;  // todo: remove
-typedef struct SVnodeCfg        SVnodeCfg;
-typedef struct SVSnapshotReader SVSnapshotReader;
-typedef struct SVSnapshotWriter SVSnapshotWriter;
+typedef struct SVnode       SVnode;
+typedef struct STsdbCfg     STsdbCfg;  // todo: remove
+typedef struct SVnodeCfg    SVnodeCfg;
+typedef struct SVSnapReader SVSnapReader;
+typedef struct SVSnapWriter SVSnapWriter;
 
 extern const SVnodeCfg vnodeCfgDefault;
 
@@ -157,14 +157,14 @@ int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, SStreamReader *pHandle);
 // sma
 int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
 
-// SVSnapshotReader
-int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever);
-int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader);
-int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData);
-// SVSnapshotWriter;
-int32_t vnodeSnapshotWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapshotWriter **ppWriter);
-int32_t vnodeSnapshotWrite(SVSnapshotWriter *pWriter, uint8_t *pData, uint32_t nData);
-int32_t vnodeSnapshotWriterClose(SVSnapshotWriter *pWriter, int8_t rollback);
+// SVSnapReader
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, SVSnapReader **ppReader, int64_t sver, int64_t ever);
+int32_t vnodeSnapReaderClose(SVSnapReader *pReader);
+int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nData);
+// SVSnapWriter;
+int32_t vnodeSnapshotWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter);
+int32_t vnodeSnapshotWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
+int32_t vnodeSnapshotWriterClose(SVSnapWriter *pWriter, int8_t rollback);
 
 // structs
 struct STsdbCfg {
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 8d77a41c8a..7676f2f6e8 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -49,17 +49,17 @@
 extern "C" {
 #endif
 
-typedef struct SVnodeInfo          SVnodeInfo;
-typedef struct SMeta               SMeta;
-typedef struct SSma                SSma;
-typedef struct STsdb               STsdb;
-typedef struct STQ                 STQ;
-typedef struct SVState             SVState;
-typedef struct SVBufPool           SVBufPool;
-typedef struct SQWorker            SQHandle;
-typedef struct STsdbKeepCfg        STsdbKeepCfg;
-typedef struct SMetaSnapshotReader SMetaSnapshotReader;
-typedef struct STsdbSnapshotReader STsdbSnapshotReader;
+typedef struct SVnodeInfo      SVnodeInfo;
+typedef struct SMeta           SMeta;
+typedef struct SSma            SSma;
+typedef struct STsdb           STsdb;
+typedef struct STQ             STQ;
+typedef struct SVState         SVState;
+typedef struct SVBufPool       SVBufPool;
+typedef struct SQWorker        SQHandle;
+typedef struct STsdbKeepCfg    STsdbKeepCfg;
+typedef struct SMetaSnapReader SMetaSnapReader;
+typedef struct STsdbSnapReader STsdbSnapReader;
 
 #define VNODE_META_DIR  "meta"
 #define VNODE_TSDB_DIR  "tsdb"
@@ -108,9 +108,9 @@ STSma*          metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid);
 STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy);
 SArray*         metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid);
 SArray*         metaGetSmaTbUids(SMeta* pMeta);
-int32_t         metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppReader, int64_t sver, int64_t ever);
-int32_t         metaSnapshotReaderClose(SMetaSnapshotReader* pReader);
-int32_t         metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nData);
+int32_t         metaSnapReaderOpen(SMeta* pMeta, SMetaSnapReader** ppReader, int64_t sver, int64_t ever);
+int32_t         metaSnapReaderClose(SMetaSnapReader* pReader);
+int32_t         metaSnapRead(SMetaSnapReader* pReader, void** ppData, uint32_t* nData);
 void*           metaGetIdx(SMeta* pMeta);
 void*           metaGetIvtIdx(SMeta* pMeta);
 int             metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList);
@@ -130,9 +130,9 @@ int32_t     tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* p
 int32_t     tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
 STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
                                 void* pMemRef);
-int32_t     tsdbSnapshotReaderOpen(STsdb* pTsdb, STsdbSnapshotReader** ppReader, int64_t sver, int64_t ever);
-int32_t     tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader);
-int32_t     tsdbSnapshotRead(STsdbSnapshotReader* pReader, void** ppData, uint32_t* nData);
+int32_t     tsdbSnapReaderOpen(STsdb* pTsdb, STsdbSnapReader** ppReader, int64_t sver, int64_t ever);
+int32_t     tsdbSnapReaderClose(STsdbSnapReader* pReader);
+int32_t     tsdbSnapRead(STsdbSnapReader* pReader, void** ppData, uint32_t* nData);
 
 // tq
 int     tqInit();
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 5757039d55..146d24e998 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -15,19 +15,19 @@
 
 #include "meta.h"
 
-struct SMetaSnapshotReader {
+struct SMetaSnapReader {
   SMeta*  pMeta;
   TBC*    pTbc;
   int64_t sver;
   int64_t ever;
 };
 
-int32_t metaSnapshotReaderOpen(SMeta* pMeta, SMetaSnapshotReader** ppReader, int64_t sver, int64_t ever) {
-  int32_t              code = 0;
-  int32_t              c = 0;
-  SMetaSnapshotReader* pMetaReader = NULL;
+int32_t metaSnapReaderOpen(SMeta* pMeta, SMetaSnapReader** ppReader, int64_t sver, int64_t ever) {
+  int32_t          code = 0;
+  int32_t          c = 0;
+  SMetaSnapReader* pMetaReader = NULL;
 
-  pMetaReader = (SMetaSnapshotReader*)taosMemoryCalloc(1, sizeof(*pMetaReader));
+  pMetaReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pMetaReader));
   if (pMetaReader == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
@@ -53,7 +53,7 @@ _err:
   return code;
 }
 
-int32_t metaSnapshotReaderClose(SMetaSnapshotReader* pReader) {
+int32_t metaSnapReaderClose(SMetaSnapReader* pReader) {
   if (pReader) {
     tdbTbcClose(pReader->pTbc);
     taosMemoryFree(pReader);
@@ -61,7 +61,7 @@ int32_t metaSnapshotReaderClose(SMetaSnapshotReader* pReader) {
   return 0;
 }
 
-int32_t metaSnapshotRead(SMetaSnapshotReader* pReader, void** ppData, uint32_t* nDatap) {
+int32_t metaSnapRead(SMetaSnapReader* pReader, void** ppData, uint32_t* nDatap) {
   const void* pKey = NULL;
   const void* pData = NULL;
   int32_t     nKey = 0;
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index c6d1377591..57eebfc2b2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -15,7 +15,7 @@
 
 #include "tsdb.h"
 
-struct STsdbSnapshotReader {
+struct STsdbSnapReader {
   STsdb*  pTsdb;
   int64_t sver;
   int64_t ever;
@@ -25,22 +25,23 @@ struct STsdbSnapshotReader {
   SDelFReader* pDelFReader;
 };
 
-typedef struct STsdbSnapshotWriter {
+typedef struct STsdbSnapWriter {
   STsdb*  pTsdb;
   int64_t sver;
   int64_t ever;
   // for data file
+  int32_t       iDFileSet;
   SDataFWriter* pDataFWriter;
   // for del file
   SDelFWriter* pDelFWriter;
-} STsdbSnapshotWriter;
+} STsdbSnapWriter;
 
-int32_t tsdbSnapshotReaderOpen(STsdb* pTsdb, STsdbSnapshotReader** ppReader, int64_t sver, int64_t ever) {
-  int32_t              code = 0;
-  STsdbSnapshotReader* pReader = NULL;
+int32_t tsdbSnapReaderOpen(STsdb* pTsdb, STsdbSnapReader** ppReader, int64_t sver, int64_t ever) {
+  int32_t          code = 0;
+  STsdbSnapReader* pReader = NULL;
 
   // alloc
-  pReader = (STsdbSnapshotReader*)taosMemoryCalloc(1, sizeof(*pReader));
+  pReader = (STsdbSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
   if (pReader == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
@@ -58,9 +59,13 @@ _err:
   return code;
 }
 
-int32_t tsdbSnapshotRead(STsdbSnapshotReader* pReader, void** ppData, uint32_t* nData) {
+int32_t tsdbSnapRead(STsdbSnapReader* pReader, void** ppData, uint32_t* nData) {
   int32_t code = 0;
-  // TODO
+
+  // read data file
+
+  // read del file
+
   return code;
 
 _err:
@@ -68,7 +73,7 @@ _err:
   return code;
 }
 
-int32_t tsdbSnapshotReaderClose(STsdbSnapshotReader* pReader) {
+int32_t tsdbSnapReaderClose(STsdbSnapReader* pReader) {
   int32_t code = 0;
   taosMemoryFree(pReader);
   return code;
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 8d2a316d64..2de1d37d70 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -15,29 +15,29 @@
 
 #include "vnodeInt.h"
 
-struct SVSnapshotReader {
-  SVnode              *pVnode;
-  int64_t              sver;
-  int64_t              ever;
-  int8_t               isMetaEnd;
-  int8_t               isTsdbEnd;
-  SMetaSnapshotReader *pMetaReader;
-  STsdbSnapshotReader *pTsdbReader;
-  void                *pData;
-  int32_t              nData;
+struct SVSnapReader {
+  SVnode          *pVnode;
+  int64_t          sver;
+  int64_t          ever;
+  int8_t           isMetaEnd;
+  int8_t           isTsdbEnd;
+  SMetaSnapReader *pMetaReader;
+  STsdbSnapReader *pTsdbReader;
+  void            *pData;
+  int32_t          nData;
 };
 
-struct SVSnapshotWriter {
+struct SVSnapWriter {
   SVnode *pVnode;
   int64_t sver;
   int64_t ever;
 };
 
-// SVSnapshotReader ========================================================
-int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int64_t sver, int64_t ever) {
-  SVSnapshotReader *pReader = NULL;
+// SVSnapReader ========================================================
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, SVSnapReader **ppReader, int64_t sver, int64_t ever) {
+  SVSnapReader *pReader = NULL;
 
-  pReader = (SVSnapshotReader *)taosMemoryCalloc(1, sizeof(*pReader));
+  pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader));
   if (pReader == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
@@ -48,13 +48,13 @@ int32_t vnodeSnapshotReaderOpen(SVnode *pVnode, SVSnapshotReader **ppReader, int
   pReader->isMetaEnd = 0;
   pReader->isTsdbEnd = 0;
 
-  if (metaSnapshotReaderOpen(pVnode->pMeta, &pReader->pMetaReader, sver, ever) < 0) {
+  if (metaSnapReaderOpen(pVnode->pMeta, &pReader->pMetaReader, sver, ever) < 0) {
     taosMemoryFree(pReader);
     goto _err;
   }
 
-  if (tsdbSnapshotReaderOpen(pVnode->pTsdb, &pReader->pTsdbReader, sver, ever) < 0) {
-    metaSnapshotReaderClose(pReader->pMetaReader);
+  if (tsdbSnapReaderOpen(pVnode->pTsdb, &pReader->pTsdbReader, sver, ever) < 0) {
+    metaSnapReaderClose(pReader->pMetaReader);
     taosMemoryFree(pReader);
     goto _err;
   }
@@ -68,21 +68,21 @@ _err:
   return -1;
 }
 
-int32_t vnodeSnapshotReaderClose(SVSnapshotReader *pReader) {
+int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
   if (pReader) {
     vnodeFree(pReader->pData);
-    tsdbSnapshotReaderClose(pReader->pTsdbReader);
-    metaSnapshotReaderClose(pReader->pMetaReader);
+    tsdbSnapReaderClose(pReader->pTsdbReader);
+    metaSnapReaderClose(pReader->pMetaReader);
     taosMemoryFree(pReader);
   }
   return 0;
 }
 
-int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32_t *nData) {
+int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nData) {
   int32_t code = 0;
 
   if (!pReader->isMetaEnd) {
-    code = metaSnapshotRead(pReader->pMetaReader, &pReader->pData, &pReader->nData);
+    code = metaSnapRead(pReader->pMetaReader, &pReader->pData, &pReader->nData);
     if (code) {
       if (code == TSDB_CODE_VND_READ_END) {
         pReader->isMetaEnd = 1;
@@ -97,7 +97,7 @@ int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32
   }
 
   if (!pReader->isTsdbEnd) {
-    code = tsdbSnapshotRead(pReader->pTsdbReader, &pReader->pData, &pReader->nData);
+    code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData, &pReader->nData);
     if (code) {
       if (code == TSDB_CODE_VND_READ_END) {
         pReader->isTsdbEnd = 1;
@@ -115,13 +115,13 @@ int32_t vnodeSnapshotRead(SVSnapshotReader *pReader, const void **ppData, uint32
   return code;
 }
 
-// SVSnapshotWriter ========================================================
-int32_t vnodeSnapshotWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapshotWriter **ppWriter) {
-  int32_t           code = 0;
-  SVSnapshotWriter *pWriter = NULL;
+// SVSnapWriter ========================================================
+int32_t vnodeSnapshotWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) {
+  int32_t       code = 0;
+  SVSnapWriter *pWriter = NULL;
 
   // alloc
-  pWriter = (SVSnapshotWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
+  pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
   if (pWriter == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
@@ -136,13 +136,13 @@ _err:
   return code;
 }
 
-int32_t vnodeSnapshotWrite(SVSnapshotWriter *pWriter, uint8_t *pData, uint32_t nData) {
+int32_t vnodeSnapshotWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
   int32_t code = 0;
   // TODO
   return code;
 }
 
-int32_t vnodeSnapshotWriterClose(SVSnapshotWriter *pWriter, int8_t rollback) {
+int32_t vnodeSnapshotWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
   int32_t code = 0;
 
   if (!rollback) {

From fd98b9082f7674bfa0696e3fd9176dcf0cd1417e Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 4 Jul 2022 07:00:21 +0000
Subject: [PATCH 03/50] more code

---
 source/dnode/vnode/inc/vnode.h             | 2 +-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 348c792dd3..412efbb5f5 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -158,7 +158,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, SStreamReader *pHandle);
 int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
 
 // SVSnapReader
-int32_t vnodeSnapReaderOpen(SVnode *pVnode, SVSnapReader **ppReader, int64_t sver, int64_t ever);
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader);
 int32_t vnodeSnapReaderClose(SVSnapReader *pReader);
 int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nData);
 // SVSnapWriter;
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 2de1d37d70..ab661b0a54 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -34,7 +34,7 @@ struct SVSnapWriter {
 };
 
 // SVSnapReader ========================================================
-int32_t vnodeSnapReaderOpen(SVnode *pVnode, SVSnapReader **ppReader, int64_t sver, int64_t ever) {
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) {
   SVSnapReader *pReader = NULL;
 
   pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader));

From 0408ba414c03cf7fe93257d177ba5aa50880e3b2 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 4 Jul 2022 07:21:04 +0000
Subject: [PATCH 04/50] more work

---
 source/dnode/vnode/inc/vnode.h             |  2 +-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 51 +++++++++++-----------
 2 files changed, 26 insertions(+), 27 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 412efbb5f5..4a974e605d 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -163,8 +163,8 @@ int32_t vnodeSnapReaderClose(SVSnapReader *pReader);
 int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nData);
 // SVSnapWriter;
 int32_t vnodeSnapshotWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter);
-int32_t vnodeSnapshotWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
 int32_t vnodeSnapshotWriterClose(SVSnapWriter *pWriter, int8_t rollback);
+int32_t vnodeSnapshotWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
 
 // structs
 struct STsdbCfg {
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index ab661b0a54..27e736d622 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -16,12 +16,14 @@
 #include "vnodeInt.h"
 
 struct SVSnapReader {
-  SVnode          *pVnode;
-  int64_t          sver;
-  int64_t          ever;
-  int8_t           isMetaEnd;
-  int8_t           isTsdbEnd;
+  SVnode *pVnode;
+  int64_t sver;
+  int64_t ever;
+  // meta
+  int8_t           metaDone;
   SMetaSnapReader *pMetaReader;
+  // tsdb
+  int8_t           tsdbDone;
   STsdbSnapReader *pTsdbReader;
   void            *pData;
   int32_t          nData;
@@ -35,6 +37,7 @@ struct SVSnapWriter {
 
 // SVSnapReader ========================================================
 int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) {
+  int32_t       code = 0;
   SVSnapReader *pReader = NULL;
 
   pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader));
@@ -45,8 +48,6 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapRe
   pReader->pVnode = pVnode;
   pReader->sver = sver;
   pReader->ever = ever;
-  pReader->isMetaEnd = 0;
-  pReader->isTsdbEnd = 0;
 
   if (metaSnapReaderOpen(pVnode->pMeta, &pReader->pMetaReader, sver, ever) < 0) {
     taosMemoryFree(pReader);
@@ -59,33 +60,31 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapRe
     goto _err;
   }
 
-_exit:
   *ppReader = pReader;
-  return 0;
+  return code;
 
 _err:
   *ppReader = NULL;
-  return -1;
+  return code;
 }
 
 int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
-  if (pReader) {
-    vnodeFree(pReader->pData);
-    tsdbSnapReaderClose(pReader->pTsdbReader);
-    metaSnapReaderClose(pReader->pMetaReader);
-    taosMemoryFree(pReader);
-  }
+  vnodeFree(pReader->pData);
+  tsdbSnapReaderClose(pReader->pTsdbReader);
+  metaSnapReaderClose(pReader->pMetaReader);
+  taosMemoryFree(pReader);
+
   return 0;
 }
 
 int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nData) {
   int32_t code = 0;
 
-  if (!pReader->isMetaEnd) {
+  if (!pReader->metaDone) {
     code = metaSnapRead(pReader->pMetaReader, &pReader->pData, &pReader->nData);
     if (code) {
       if (code == TSDB_CODE_VND_READ_END) {
-        pReader->isMetaEnd = 1;
+        pReader->metaDone = 1;
       } else {
         return code;
       }
@@ -96,11 +95,11 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nDat
     }
   }
 
-  if (!pReader->isTsdbEnd) {
+  if (!pReader->tsdbDone) {
     code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData, &pReader->nData);
     if (code) {
       if (code == TSDB_CODE_VND_READ_END) {
-        pReader->isTsdbEnd = 1;
+        pReader->tsdbDone = 1;
       } else {
         return code;
       }
@@ -136,12 +135,6 @@ _err:
   return code;
 }
 
-int32_t vnodeSnapshotWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
 int32_t vnodeSnapshotWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
   int32_t code = 0;
 
@@ -153,4 +146,10 @@ int32_t vnodeSnapshotWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
 
   taosMemoryFree(pWriter);
   return code;
+}
+
+int32_t vnodeSnapshotWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
+  int32_t code = 0;
+  // TODO
+  return code;
 }
\ No newline at end of file

From 05af34a52797dda3119180db978e05ad0d926f9a Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 4 Jul 2022 07:26:52 +0000
Subject: [PATCH 05/50] more code

---
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 27e736d622..0deb1b75c6 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -69,12 +69,14 @@ _err:
 }
 
 int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
+  int32_t code = 0;
+
   vnodeFree(pReader->pData);
   tsdbSnapReaderClose(pReader->pTsdbReader);
   metaSnapReaderClose(pReader->pMetaReader);
   taosMemoryFree(pReader);
 
-  return 0;
+  return code;
 }
 
 int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nData) {

From 56913abdbff43a3fa5d7df25411364f2d8f0ada0 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 4 Jul 2022 10:55:01 +0000
Subject: [PATCH 06/50] more work

---
 source/dnode/vnode/src/inc/vnodeInt.h      |  4 +-
 source/dnode/vnode/src/meta/metaSnapshot.c |  2 +-
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c |  2 +-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 44 +++++++++++-----------
 4 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index b82d782941..a7b8d951d6 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -109,7 +109,7 @@ STSma*          metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid);
 STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy);
 SArray*         metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid);
 SArray*         metaGetSmaTbUids(SMeta* pMeta);
-int32_t         metaSnapReaderOpen(SMeta* pMeta, SMetaSnapReader** ppReader, int64_t sver, int64_t ever);
+int32_t         metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader);
 int32_t         metaSnapReaderClose(SMetaSnapReader* pReader);
 int32_t         metaSnapRead(SMetaSnapReader* pReader, void** ppData, uint32_t* nData);
 void*           metaGetIdx(SMeta* pMeta);
@@ -131,7 +131,7 @@ int32_t     tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* p
 int32_t     tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
 STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
                                 void* pMemRef);
-int32_t     tsdbSnapReaderOpen(STsdb* pTsdb, STsdbSnapReader** ppReader, int64_t sver, int64_t ever);
+int32_t     tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader);
 int32_t     tsdbSnapReaderClose(STsdbSnapReader* pReader);
 int32_t     tsdbSnapRead(STsdbSnapReader* pReader, void** ppData, uint32_t* nData);
 
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 146d24e998..321f979f0d 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -22,7 +22,7 @@ struct SMetaSnapReader {
   int64_t ever;
 };
 
-int32_t metaSnapReaderOpen(SMeta* pMeta, SMetaSnapReader** ppReader, int64_t sver, int64_t ever) {
+int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) {
   int32_t          code = 0;
   int32_t          c = 0;
   SMetaSnapReader* pMetaReader = NULL;
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 57eebfc2b2..9eebd3c36a 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -36,7 +36,7 @@ typedef struct STsdbSnapWriter {
   SDelFWriter* pDelFWriter;
 } STsdbSnapWriter;
 
-int32_t tsdbSnapReaderOpen(STsdb* pTsdb, STsdbSnapReader** ppReader, int64_t sver, int64_t ever) {
+int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader) {
   int32_t          code = 0;
   STsdbSnapReader* pReader = NULL;
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 0deb1b75c6..2444d1b0c6 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -13,8 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "vnd.h"
 
+// SVSnapReader ========================================================
 struct SVSnapReader {
   SVnode *pVnode;
   int64_t sver;
@@ -29,13 +30,6 @@ struct SVSnapReader {
   int32_t          nData;
 };
 
-struct SVSnapWriter {
-  SVnode *pVnode;
-  int64_t sver;
-  int64_t ever;
-};
-
-// SVSnapReader ========================================================
 int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) {
   int32_t       code = 0;
   SVSnapReader *pReader = NULL;
@@ -49,21 +43,17 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapRe
   pReader->sver = sver;
   pReader->ever = ever;
 
-  if (metaSnapReaderOpen(pVnode->pMeta, &pReader->pMetaReader, sver, ever) < 0) {
-    taosMemoryFree(pReader);
-    goto _err;
-  }
+  code = metaSnapReaderOpen(pVnode->pMeta, sver, ever, &pReader->pMetaReader);
+  if (code) goto _err;
 
-  if (tsdbSnapReaderOpen(pVnode->pTsdb, &pReader->pTsdbReader, sver, ever) < 0) {
-    metaSnapReaderClose(pReader->pMetaReader);
-    taosMemoryFree(pReader);
-    goto _err;
-  }
+  code = tsdbSnapReaderOpen(pVnode->pTsdb, sver, ever, &pReader->pTsdbReader);
+  if (code) goto _err;
 
   *ppReader = pReader;
   return code;
 
 _err:
+  vError("vgId:%d vnode snapshot reader open failed since %s", TD_VID(pVnode), tstrerror(code));
   *ppReader = NULL;
   return code;
 }
@@ -72,8 +62,8 @@ int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
   int32_t code = 0;
 
   vnodeFree(pReader->pData);
-  tsdbSnapReaderClose(pReader->pTsdbReader);
-  metaSnapReaderClose(pReader->pMetaReader);
+  if (pReader->pTsdbReader) tsdbSnapReaderClose(pReader->pTsdbReader);
+  if (pReader->pMetaReader) metaSnapReaderClose(pReader->pMetaReader);
   taosMemoryFree(pReader);
 
   return code;
@@ -93,7 +83,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nDat
     } else {
       *ppData = pReader->pData;
       *nData = pReader->nData;
-      return code;
+      goto _exit;
     }
   }
 
@@ -108,15 +98,27 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nDat
     } else {
       *ppData = pReader->pData;
       *nData = pReader->nData;
-      return code;
+      goto _exit;
     }
   }
 
   code = TSDB_CODE_VND_READ_END;
+
+_exit:
+  return code;
+
+_err:
+  vError("vgId:% snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code));
   return code;
 }
 
 // SVSnapWriter ========================================================
+struct SVSnapWriter {
+  SVnode *pVnode;
+  int64_t sver;
+  int64_t ever;
+};
+
 int32_t vnodeSnapshotWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) {
   int32_t       code = 0;
   SVSnapWriter *pWriter = NULL;

From 5891d97a0d4f21ecc066fcb43c8313ca059da054 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 4 Jul 2022 12:04:40 +0000
Subject: [PATCH 07/50] more work

---
 source/dnode/vnode/CMakeLists.txt          | 16 ++++----
 source/dnode/vnode/inc/vnode.h             |  4 +-
 source/dnode/vnode/src/inc/vnodeInt.h      | 24 +++++++-----
 source/dnode/vnode/src/meta/metaSnapshot.c | 42 +++++++++++---------
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c |  7 ++--
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 16 ++++----
 source/dnode/vnode/src/vnd/vnodeUtil.c     | 45 ----------------------
 7 files changed, 60 insertions(+), 94 deletions(-)
 delete mode 100644 source/dnode/vnode/src/vnd/vnodeUtil.c

diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index 24c4f2912c..a15f6fe4ee 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -3,6 +3,7 @@ add_library(vnode STATIC "")
 target_sources(
     vnode
     PRIVATE
+
     # vnode
     "src/vnd/vnodeOpen.c"
     "src/vnd/vnodeBufPool.c"
@@ -14,7 +15,6 @@ target_sources(
     "src/vnd/vnodeSvr.c"
     "src/vnd/vnodeSync.c"
     "src/vnd/vnodeSnapshot.c"
-    "src/vnd/vnodeUtil.c"
 
     # meta
     "src/meta/metaOpen.c"
@@ -66,7 +66,6 @@ target_include_directories(
     PUBLIC "inc"
     PRIVATE "src/inc"
     PUBLIC "${TD_SOURCE_DIR}/include/libs/scalar"
-    
 )
 target_link_libraries(
     vnode
@@ -80,18 +79,19 @@ target_link_libraries(
     PUBLIC executor
     PUBLIC scheduler
     PUBLIC tdb
-    #PUBLIC bdb
-    #PUBLIC scalar
+
+    # PUBLIC bdb
+    # PUBLIC scalar
     PUBLIC transport
     PUBLIC stream
     PUBLIC index
 )
 target_compile_definitions(vnode PUBLIC -DMETA_REFACT)
-if (${BUILD_WITH_INVERTEDINDEX})
-  add_definitions(-DUSE_INVERTED_INDEX) 
+
+if(${BUILD_WITH_INVERTEDINDEX})
+    add_definitions(-DUSE_INVERTED_INDEX)
 endif(${BUILD_WITH_INVERTEDINDEX})
+
 if(${BUILD_TEST})
     add_subdirectory(test)
 endif(${BUILD_TEST})
-
-
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index dd0456ea42..499ca635dc 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -159,8 +159,8 @@ int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *
 // SVSnapReader
 int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader);
 int32_t vnodeSnapReaderClose(SVSnapReader *pReader);
-int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nData);
-// SVSnapWriter;
+int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData);
+// SVSnapWriter
 int32_t vnodeSnapshotWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter);
 int32_t vnodeSnapshotWriterClose(SVSnapWriter *pWriter, int8_t rollback);
 int32_t vnodeSnapshotWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index ac1b01f749..9f7b1ed783 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -59,7 +59,9 @@ typedef struct SVBufPool       SVBufPool;
 typedef struct SQWorker        SQHandle;
 typedef struct STsdbKeepCfg    STsdbKeepCfg;
 typedef struct SMetaSnapReader SMetaSnapReader;
+typedef struct SMetaSnapWriter SMetaSnapWriter;
 typedef struct STsdbSnapReader STsdbSnapReader;
+typedef struct STsdbSnapWriter STsdbSnapWriter;
 
 #define VNODE_META_DIR  "meta"
 #define VNODE_TSDB_DIR  "tsdb"
@@ -72,10 +74,8 @@ typedef struct STsdbSnapReader STsdbSnapReader;
 #define VNODE_RSMA2_DIR "rsma2"
 
 // vnd.h
-void*   vnodeBufPoolMalloc(SVBufPool* pPool, int size);
-void    vnodeBufPoolFree(SVBufPool* pPool, void* p);
-int32_t vnodeRealloc(void** pp, int32_t size);
-void    vnodeFree(void* p);
+void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
+void  vnodeBufPoolFree(SVBufPool* pPool, void* p);
 
 // meta
 typedef struct SMCtbCursor SMCtbCursor;
@@ -109,9 +109,6 @@ STSma*          metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid);
 STSmaWrapper*   metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy);
 SArray*         metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid);
 SArray*         metaGetSmaTbUids(SMeta* pMeta);
-int32_t         metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader);
-int32_t         metaSnapReaderClose(SMetaSnapReader* pReader);
-int32_t         metaSnapRead(SMetaSnapReader* pReader, void** ppData, uint32_t* nData);
 void*           metaGetIdx(SMeta* pMeta);
 void*           metaGetIvtIdx(SMeta* pMeta);
 int             metaTtlSmaller(SMeta* pMeta, uint64_t time, SArray* uidList);
@@ -131,9 +128,6 @@ int32_t     tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* p
 int32_t     tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
 STsdbReader tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
                                 void* pMemRef);
-int32_t     tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader);
-int32_t     tsdbSnapReaderClose(STsdbSnapReader* pReader);
-int32_t     tsdbSnapRead(STsdbSnapReader* pReader, void** ppData, uint32_t* nData);
 
 // tq
 int     tqInit();
@@ -180,6 +174,16 @@ int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore);
 void    tdUidStoreDestory(STbUidStore* pStore);
 void*   tdUidStoreFree(STbUidStore* pStore);
 
+// SMetaSnapReader ========================================
+int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader);
+int32_t metaSnapReaderClose(SMetaSnapReader** ppReader);
+int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData, int64_t* nDatap);
+// SMetaSnapWriter ========================================
+// STsdbSnapReader ========================================
+int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader);
+int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader);
+int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData, int64_t* nData);
+// STsdbSnapWriter ========================================
 typedef struct {
   int8_t  streamType;  // sma or other
   int8_t  dstType;
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 321f979f0d..9252f8d23d 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -15,53 +15,57 @@
 
 #include "meta.h"
 
+// SMetaSnapReader ========================================
 struct SMetaSnapReader {
   SMeta*  pMeta;
-  TBC*    pTbc;
   int64_t sver;
   int64_t ever;
+  TBC*    pTbc;
 };
 
 int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) {
   int32_t          code = 0;
   int32_t          c = 0;
-  SMetaSnapReader* pMetaReader = NULL;
+  SMetaSnapReader* pMetaSnapReader = NULL;
 
-  pMetaReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pMetaReader));
-  if (pMetaReader == NULL) {
+  // alloc
+  pMetaSnapReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pMetaSnapReader));
+  if (pMetaSnapReader == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
-  pMetaReader->pMeta = pMeta;
-  pMetaReader->sver = sver;
-  pMetaReader->ever = ever;
-  code = tdbTbcOpen(pMeta->pTbDb, &pMetaReader->pTbc, NULL);
+  pMetaSnapReader->pMeta = pMeta;
+  pMetaSnapReader->sver = sver;
+  pMetaSnapReader->ever = ever;
+
+  // impl
+  code = tdbTbcOpen(pMeta->pTbDb, &pMetaSnapReader->pTbc, NULL);
   if (code) {
     goto _err;
   }
 
-  code = tdbTbcMoveTo(pMetaReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
+  code = tdbTbcMoveTo(pMetaSnapReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
   if (code) {
     goto _err;
   }
 
-  *ppReader = pMetaReader;
+  *ppReader = pMetaSnapReader;
   return code;
 
 _err:
+  metaError("vgId:%d meta snap reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
   *ppReader = NULL;
   return code;
 }
 
-int32_t metaSnapReaderClose(SMetaSnapReader* pReader) {
-  if (pReader) {
-    tdbTbcClose(pReader->pTbc);
-    taosMemoryFree(pReader);
-  }
+int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) {
+  tdbTbcClose((*ppReader)->pTbc);
+  taosMemoryFree(*ppReader);
+  *ppReader = NULL;
   return 0;
 }
 
-int32_t metaSnapRead(SMetaSnapReader* pReader, void** ppData, uint32_t* nDatap) {
+int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData, int64_t* nDatap) {
   const void* pKey = NULL;
   const void* pData = NULL;
   int32_t     nKey = 0;
@@ -82,7 +86,7 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, void** ppData, uint32_t* nDatap)
   }
 
   // copy the data
-  if (vnodeRealloc(ppData, nData) < 0) {
+  if (tRealloc(ppData, nData) < 0) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     return code;
   }
@@ -90,4 +94,6 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, void** ppData, uint32_t* nDatap)
   memcpy(*ppData, pData, nData);
   *nDatap = nData;
   return code;
-}
\ No newline at end of file
+}
+
+// SMetaSnapWriter ========================================
\ No newline at end of file
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 9eebd3c36a..d1f3e49291 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -15,6 +15,7 @@
 
 #include "tsdb.h"
 
+// STsdbSnapReader ========================================
 struct STsdbSnapReader {
   STsdb*  pTsdb;
   int64_t sver;
@@ -59,7 +60,7 @@ _err:
   return code;
 }
 
-int32_t tsdbSnapRead(STsdbSnapReader* pReader, void** ppData, uint32_t* nData) {
+int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData, int64_t* nData) {
   int32_t code = 0;
 
   // read data file
@@ -73,8 +74,8 @@ _err:
   return code;
 }
 
-int32_t tsdbSnapReaderClose(STsdbSnapReader* pReader) {
+int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
   int32_t code = 0;
-  taosMemoryFree(pReader);
+  taosMemoryFree(ppReader);
   return code;
 }
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 2444d1b0c6..fbc1a08401 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -26,8 +26,8 @@ struct SVSnapReader {
   // tsdb
   int8_t           tsdbDone;
   STsdbSnapReader *pTsdbReader;
-  void            *pData;
-  int32_t          nData;
+  uint8_t         *pData;
+  int64_t          nData;
 };
 
 int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) {
@@ -61,15 +61,15 @@ _err:
 int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
   int32_t code = 0;
 
-  vnodeFree(pReader->pData);
-  if (pReader->pTsdbReader) tsdbSnapReaderClose(pReader->pTsdbReader);
-  if (pReader->pMetaReader) metaSnapReaderClose(pReader->pMetaReader);
+  tFree(pReader->pData);
+  if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader);
+  if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader);
   taosMemoryFree(pReader);
 
   return code;
 }
 
-int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nData) {
+int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) {
   int32_t code = 0;
 
   if (!pReader->metaDone) {
@@ -78,7 +78,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nDat
       if (code == TSDB_CODE_VND_READ_END) {
         pReader->metaDone = 1;
       } else {
-        return code;
+        goto _err;
       }
     } else {
       *ppData = pReader->pData;
@@ -93,7 +93,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, const void **ppData, uint32_t *nDat
       if (code == TSDB_CODE_VND_READ_END) {
         pReader->tsdbDone = 1;
       } else {
-        return code;
+        goto _err;
       }
     } else {
       *ppData = pReader->pData;
diff --git a/source/dnode/vnode/src/vnd/vnodeUtil.c b/source/dnode/vnode/src/vnd/vnodeUtil.c
deleted file mode 100644
index cd942099bc..0000000000
--- a/source/dnode/vnode/src/vnd/vnodeUtil.c
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "vnd.h"
-
-int32_t vnodeRealloc(void** pp, int32_t size) {
-  uint8_t* p = NULL;
-  int32_t  csize = 0;
-
-  if (*pp) {
-    p = (uint8_t*)(*pp) - sizeof(int32_t);
-    csize = *(int32_t*)p;
-  }
-
-  if (csize >= size) {
-    return 0;
-  }
-
-  p = (uint8_t*)taosMemoryRealloc(p, size);
-  if (p == NULL) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  *(int32_t*)p = size;
-  *pp = p + sizeof(int32_t);
-
-  return 0;
-}
-
-void vnodeFree(void* p) {
-  if (p) {
-    taosMemoryFree(((uint8_t*)p) - sizeof(int32_t));
-  }
-}
\ No newline at end of file

From 032339949edb84b89962eb613b9d6c5e576e4372 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 4 Jul 2022 12:24:34 +0000
Subject: [PATCH 08/50] more work

---
 source/dnode/vnode/src/inc/vnodeInt.h      |  9 ++++++++-
 source/dnode/vnode/src/meta/metaSnapshot.c | 12 +++++++-----
 source/dnode/vnode/src/vnd/vnodeSnapshot.c |  7 +++----
 3 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 9f7b1ed783..4850aab2e0 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -62,6 +62,7 @@ typedef struct SMetaSnapReader SMetaSnapReader;
 typedef struct SMetaSnapWriter SMetaSnapWriter;
 typedef struct STsdbSnapReader STsdbSnapReader;
 typedef struct STsdbSnapWriter STsdbSnapWriter;
+typedef struct SSnapDataHdr    SSnapDataHdr;
 
 #define VNODE_META_DIR  "meta"
 #define VNODE_TSDB_DIR  "tsdb"
@@ -177,7 +178,7 @@ void*   tdUidStoreFree(STbUidStore* pStore);
 // SMetaSnapReader ========================================
 int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader);
 int32_t metaSnapReaderClose(SMetaSnapReader** ppReader);
-int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData, int64_t* nDatap);
+int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData);
 // SMetaSnapWriter ========================================
 // STsdbSnapReader ========================================
 int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader);
@@ -292,6 +293,12 @@ struct SSma {
 // sma
 void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 
+struct SSnapDataHdr {
+  int8_t  type;
+  int64_t size;
+  uint8_t data[];
+};
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 9252f8d23d..013ec19411 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -65,7 +65,7 @@ int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) {
   return 0;
 }
 
-int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData, int64_t* nDatap) {
+int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
   const void* pKey = NULL;
   const void* pData = NULL;
   int32_t     nKey = 0;
@@ -79,20 +79,22 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData, int64_t* nDatap
     }
 
     if (((STbDbKey*)pData)->version < pReader->sver) {
+      tdbTbcMoveToNext(pReader->pTbc);
       continue;
     }
 
+    tdbTbcMoveToNext(pReader->pTbc);
     break;
   }
 
   // copy the data
-  if (tRealloc(ppData, nData) < 0) {
+  if (tRealloc(ppData, sizeof(SSnapDataHdr) + nData) < 0) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     return code;
   }
-
-  memcpy(*ppData, pData, nData);
-  *nDatap = nData;
+  ((SSnapDataHdr*)(*ppData))->type = 0;  // TODO: use macro
+  ((SSnapDataHdr*)(*ppData))->size = nData;
+  memcpy(((SSnapDataHdr*)(*ppData))->data, pData, nData);
   return code;
 }
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index fbc1a08401..f760d9174e 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -27,7 +27,6 @@ struct SVSnapReader {
   int8_t           tsdbDone;
   STsdbSnapReader *pTsdbReader;
   uint8_t         *pData;
-  int64_t          nData;
 };
 
 int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) {
@@ -73,7 +72,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
   int32_t code = 0;
 
   if (!pReader->metaDone) {
-    code = metaSnapRead(pReader->pMetaReader, &pReader->pData, &pReader->nData);
+    code = metaSnapRead(pReader->pMetaReader, &pReader->pData);
     if (code) {
       if (code == TSDB_CODE_VND_READ_END) {
         pReader->metaDone = 1;
@@ -82,7 +81,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
       }
     } else {
       *ppData = pReader->pData;
-      *nData = pReader->nData;
+      *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
       goto _exit;
     }
   }
@@ -97,7 +96,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
       }
     } else {
       *ppData = pReader->pData;
-      *nData = pReader->nData;
+      *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
       goto _exit;
     }
   }

From b7e2b7398eca174deb8730784a29cc56d39be942 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Mon, 4 Jul 2022 13:07:13 +0000
Subject: [PATCH 09/50] more work

---
 source/dnode/vnode/src/inc/vnodeInt.h      |   2 +-
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 141 ++++++++++++++++++---
 source/dnode/vnode/src/vnd/vnodeSnapshot.c |   2 +-
 3 files changed, 127 insertions(+), 18 deletions(-)

diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 4850aab2e0..70a59ca2d7 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -183,7 +183,7 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData);
 // STsdbSnapReader ========================================
 int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader);
 int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader);
-int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData, int64_t* nData);
+int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData);
 // STsdbSnapWriter ========================================
 typedef struct {
   int8_t  streamType;  // sma or other
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index d1f3e49291..e5500accab 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -21,21 +21,91 @@ struct STsdbSnapReader {
   int64_t sver;
   int64_t ever;
   // for data file
+  int8_t        dataDone;
+  int32_t       fid;
   SDataFReader* pDataFReader;
+  int32_t       iBlockIdx;
+  SArray*       aBlockIdx;  // SArray<SBlockIdx>
   // for del file
+  int8_t       delDone;
   SDelFReader* pDelFReader;
+  int32_t      iDelIdx;
+  SArray*      aDelIdx;   // SArray<SDelIdx>
+  SArray*      aDelData;  // SArray<SDelData>
 };
 
-typedef struct STsdbSnapWriter {
-  STsdb*  pTsdb;
-  int64_t sver;
-  int64_t ever;
-  // for data file
-  int32_t       iDFileSet;
-  SDataFWriter* pDataFWriter;
-  // for del file
-  SDelFWriter* pDelFWriter;
-} STsdbSnapWriter;
+static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
+  int32_t code = 0;
+
+  if (pReader->pDataFReader == NULL) {
+    code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, NULL);
+    if (code) goto _err;
+
+    code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx, NULL);
+    if (code) goto _err;
+
+    pReader->iBlockIdx = 0;
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d snap read data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
+  int32_t   code = 0;
+  STsdb*    pTsdb = pReader->pTsdb;
+  SDelFile* pDelFile = pTsdb->fs->nState->pDelFile;
+
+  if (pReader->pDelFReader == NULL) {
+    if (pDelFile == NULL) {
+      code = TSDB_CODE_VND_READ_END;
+      goto _exit;
+    }
+
+    // open
+    code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb, NULL);
+    if (code) goto _err;
+
+    code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx, NULL);
+    if (code) goto _err;
+
+    pReader->iDelIdx = 0;
+  }
+
+  while (pReader->iDelIdx < taosArrayGetSize(pReader->aDelIdx)) {
+    SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pReader->aDelIdx, pReader->iDelIdx);
+    int8_t   overlap = 0;
+
+    code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData, NULL);
+    if (code) goto _err;
+
+    for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) {
+      SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData);
+
+      if (pDelData->version >= pReader->sver && pDelData->version <= pReader->ever) {
+        // encode the data to sync (todo)
+        overlap = 1;
+      }
+    }
+
+    if (overlap) {
+      // prepare the data
+      goto _exit;
+    }
+  }
+
+  code = TSDB_CODE_VND_READ_END;
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d snap read del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
 
 int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader) {
   int32_t          code = 0;
@@ -60,13 +130,46 @@ _err:
   return code;
 }
 
-int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData, int64_t* nData) {
+int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
+  int32_t code = 0;
+  taosMemoryFree(ppReader);
+  return code;
+}
+
+int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) {
   int32_t code = 0;
 
   // read data file
+  if (!pReader->dataDone) {
+    code = tsdbSnapReadData(pReader, ppData);
+    if (code) {
+      if (code == TSDB_CODE_VND_READ_END) {
+        pReader->dataDone = 1;
+      } else {
+        goto _err;
+      }
+    } else {
+      goto _exit;
+    }
+  }
 
   // read del file
+  if (!pReader->delDone) {
+    code = tsdbSnapReadDel(pReader, ppData);
+    if (code) {
+      if (code == TSDB_CODE_VND_READ_END) {
+        pReader->delDone = 1;
+      } else {
+        goto _err;
+      }
+    } else {
+      goto _exit;
+    }
+  }
 
+  code = TSDB_CODE_VND_READ_END;
+
+_exit:
   return code;
 
 _err:
@@ -74,8 +177,14 @@ _err:
   return code;
 }
 
-int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
-  int32_t code = 0;
-  taosMemoryFree(ppReader);
-  return code;
-}
+// STsdbSnapReader ========================================
+struct STsdbSnapWriter {
+  STsdb*  pTsdb;
+  int64_t sver;
+  int64_t ever;
+  // for data file
+  int32_t       iDFileSet;
+  SDataFWriter* pDataFWriter;
+  // for del file
+  SDelFWriter* pDelFWriter;
+};
\ No newline at end of file
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index f760d9174e..410e66e464 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -87,7 +87,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
   }
 
   if (!pReader->tsdbDone) {
-    code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData, &pReader->nData);
+    code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData);
     if (code) {
       if (code == TSDB_CODE_VND_READ_END) {
         pReader->tsdbDone = 1;

From 4d0f46fb5d65222c841d827925be56fed43cb3b2 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 5 Jul 2022 03:05:04 +0000
Subject: [PATCH 10/50] more snapshot

---
 source/dnode/vnode/src/meta/metaSnapshot.c |  5 ++-
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 47 ++++++++++++++++++++--
 2 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 013ec19411..e3124096bd 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -75,7 +75,8 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
   for (;;) {
     code = tdbTbcGet(pReader->pTbc, &pKey, &nKey, &pData, &nData);
     if (code || ((STbDbKey*)pData)->version > pReader->ever) {
-      return TSDB_CODE_VND_READ_END;
+      code = TSDB_CODE_VND_READ_END;
+      goto _exit;
     }
 
     if (((STbDbKey*)pData)->version < pReader->sver) {
@@ -95,6 +96,8 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
   ((SSnapDataHdr*)(*ppData))->type = 0;  // TODO: use macro
   ((SSnapDataHdr*)(*ppData))->size = nData;
   memcpy(((SSnapDataHdr*)(*ppData))->data, pData, nData);
+
+_exit:
   return code;
 }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index e5500accab..c4bbca893d 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -26,6 +26,8 @@ struct STsdbSnapReader {
   SDataFReader* pDataFReader;
   int32_t       iBlockIdx;
   SArray*       aBlockIdx;  // SArray<SBlockIdx>
+  SMapData      mBlock;     // SMapData<SBlock>
+  SBlockData    blkData;
   // for del file
   int8_t       delDone;
   SDelFReader* pDelFReader;
@@ -57,7 +59,7 @@ _err:
 static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
   int32_t   code = 0;
   STsdb*    pTsdb = pReader->pTsdb;
-  SDelFile* pDelFile = pTsdb->fs->nState->pDelFile;
+  SDelFile* pDelFile = pTsdb->fs->cState->pDelFile;
 
   if (pReader->pDelFReader == NULL) {
     if (pDelFile == NULL) {
@@ -69,6 +71,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
     code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb, NULL);
     if (code) goto _err;
 
+    // read index
     code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx, NULL);
     if (code) goto _err;
 
@@ -121,6 +124,29 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe
   pReader->sver = sver;
   pReader->ever = ever;
 
+  pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+  if (pReader->aBlockIdx == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  pReader->mBlock = tMapDataInit();
+
+  code = tBlockDataInit(&pReader->blkData);
+  if (code) goto _err;
+
+  pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
+  if (pReader->aDelIdx == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  pReader->aDelData = taosArrayInit(0, sizeof(SDelData));
+  if (pReader->aDelData == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
   *ppReader = pReader;
   return code;
 
@@ -131,8 +157,23 @@ _err:
 }
 
 int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
-  int32_t code = 0;
-  taosMemoryFree(ppReader);
+  int32_t          code = 0;
+  STsdbSnapReader* pReader = *ppReader;
+
+  taosArrayDestroy(pReader->aDelData);
+  taosArrayDestroy(pReader->aDelIdx);
+  if (pReader->pDelFReader) {
+    tsdbDelFReaderClose(&pReader->pDelFReader);
+  }
+  tBlockDataClear(&pReader->blkData);
+  tMapDataClear(&pReader->mBlock);
+  taosArrayDestroy(&pReader->aBlockIdx);
+  if (pReader->pDataFReader) {
+    tsdbDataFReaderClose(&pReader->pDataFReader);
+  }
+  taosMemoryFree(pReader);
+  *ppReader = NULL;
+
   return code;
 }
 

From a031500dadac57e30261730d564fb4794961a2b0 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 5 Jul 2022 03:14:28 +0000
Subject: [PATCH 11/50] fix some compile error

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index c4bbca893d..f92445f51a 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -167,7 +167,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
   }
   tBlockDataClear(&pReader->blkData);
   tMapDataClear(&pReader->mBlock);
-  taosArrayDestroy(&pReader->aBlockIdx);
+  taosArrayDestroy(pReader->aBlockIdx);
   if (pReader->pDataFReader) {
     tsdbDataFReaderClose(&pReader->pDataFReader);
   }

From 6c8de26f73f363021a1cd962f73a47dc8e22ee9e Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 5 Jul 2022 03:47:06 +0000
Subject: [PATCH 12/50] more snapshot

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 23 +++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index f92445f51a..3578c4e192 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -80,7 +80,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
 
   while (pReader->iDelIdx < taosArrayGetSize(pReader->aDelIdx)) {
     SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pReader->aDelIdx, pReader->iDelIdx);
-    int8_t   overlap = 0;
+    int32_t  size = 0;
 
     code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData, NULL);
     if (code) goto _err;
@@ -89,13 +89,26 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
       SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData);
 
       if (pDelData->version >= pReader->sver && pDelData->version <= pReader->ever) {
-        // encode the data to sync (todo)
-        overlap = 1;
+        size += tPutDelData(NULL, pDelData);
       }
     }
 
-    if (overlap) {
-      // prepare the data
+    if (size > 0) {
+      code = tRealloc(ppData, sizeof(SSnapDataHdr) + size);  // TODO
+      if (code) goto _err;
+
+      // encode
+      ((SSnapDataHdr*)(*ppData))->type = 1;
+      ((SSnapDataHdr*)(*ppData))->size = size;
+
+      for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) {
+        SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData);
+
+        if (pDelData->version >= pReader->sver && pDelData->version <= pReader->ever) {
+          // size += tPutDelData(NULL, pDelData); (todo)
+        }
+      }
+
       goto _exit;
     }
   }

From 5c7189a4333eaec63946950f7d33422fd83558c1 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 5 Jul 2022 05:43:05 +0000
Subject: [PATCH 13/50] more snapshot read

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 53 +++++++++++++++++++---
 1 file changed, 47 insertions(+), 6 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 3578c4e192..dd17f71f46 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -26,7 +26,8 @@ struct STsdbSnapReader {
   SDataFReader* pDataFReader;
   int32_t       iBlockIdx;
   SArray*       aBlockIdx;  // SArray<SBlockIdx>
-  SMapData      mBlock;     // SMapData<SBlock>
+  int32_t       iBlock;
+  SMapData      mBlock;  // SMapData<SBlock>
   SBlockData    blkData;
   // for del file
   int8_t       delDone;
@@ -38,6 +39,8 @@ struct STsdbSnapReader {
 
 static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
   int32_t code = 0;
+  SBlock  block;
+  SBlock* pBlock = &block;
 
   if (pReader->pDataFReader == NULL) {
     code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, NULL);
@@ -49,6 +52,30 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
     pReader->iBlockIdx = 0;
   }
 
+  while (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) {
+    SBlockIdx* pBlockIdx = taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+
+    pReader->iBlockIdx++;
+
+    code = tsdbReadBlock(pReader->pDataFReader, pBlockIdx, &pReader->mBlock, NULL);
+    if (code) goto _err;
+
+    break;
+  }
+
+  while (pReader->iBlock < pReader->mBlock.nItem) {
+    tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, pBlock, tGetBlock);
+
+    pReader->iBlock++;
+
+    if ((pBlock->minVersion >= pReader->sver && pBlock->minVersion <= pReader->ever) &&
+        (pBlock->maxVersion >= pReader->sver && pBlock->maxVersion <= pReader->ever)) {
+      // block in range, encode and return the data (todo)
+      goto _exit;
+    }
+  }
+
+_exit:
   return code;
 
 _err:
@@ -82,6 +109,8 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
     SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pReader->aDelIdx, pReader->iDelIdx);
     int32_t  size = 0;
 
+    pReader->iDelIdx++;
+
     code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData, NULL);
     if (code) goto _err;
 
@@ -94,18 +123,30 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
     }
 
     if (size > 0) {
-      code = tRealloc(ppData, sizeof(SSnapDataHdr) + size);  // TODO
+      int64_t n = 0;
+
+      size = size + sizeof(SSnapDataHdr) + sizeof(TABLEID);
+      code = tRealloc(ppData, size);
       if (code) goto _err;
 
-      // encode
-      ((SSnapDataHdr*)(*ppData))->type = 1;
-      ((SSnapDataHdr*)(*ppData))->size = size;
+      // SSnapDataHdr
+      SSnapDataHdr* pSnapDataHdr = (SSnapDataHdr*)(*ppData + n);
+      pSnapDataHdr->type = 1;
+      pSnapDataHdr->size = size;  // TODO: size here may incorrect
+      n += sizeof(SSnapDataHdr);
 
+      // TABLEID
+      TABLEID* pId = (TABLEID*)(*ppData + n);
+      pId->suid = pDelIdx->suid;
+      pId->uid = pDelIdx->uid;
+      n += sizeof(*pId);
+
+      // DATA
       for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) {
         SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData);
 
         if (pDelData->version >= pReader->sver && pDelData->version <= pReader->ever) {
-          // size += tPutDelData(NULL, pDelData); (todo)
+          n += tPutDelData(*ppData + n, pDelData);
         }
       }
 

From 9512b51ec0e6b8ffcd26c304f1096c3a7c7c6663 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 5 Jul 2022 07:42:56 +0000
Subject: [PATCH 14/50] more snapshor reader

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 80 +++++++++++++++-------
 1 file changed, 55 insertions(+), 25 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index dd17f71f46..0f80ce080a 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -24,10 +24,11 @@ struct STsdbSnapReader {
   int8_t        dataDone;
   int32_t       fid;
   SDataFReader* pDataFReader;
-  int32_t       iBlockIdx;
   SArray*       aBlockIdx;  // SArray<SBlockIdx>
-  int32_t       iBlock;
+  int32_t       iBlockIdx;
+  SBlockIdx*    pBlockIdx;
   SMapData      mBlock;  // SMapData<SBlock>
+  int32_t       iBlock;
   SBlockData    blkData;
   // for del file
   int8_t       delDone;
@@ -39,39 +40,68 @@ struct STsdbSnapReader {
 
 static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
   int32_t code = 0;
-  SBlock  block;
-  SBlock* pBlock = &block;
 
-  if (pReader->pDataFReader == NULL) {
-    code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, NULL);
-    if (code) goto _err;
+  while (true) {
+    if (pReader->pDataFReader == NULL) {
+      SDFileSet* pSet = NULL;
 
-    code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx, NULL);
-    if (code) goto _err;
+      // search the next data file set to read (todo)
+      if (0 /* TODO */) {
+        code = TSDB_CODE_VND_READ_END;
+        goto _exit;
+      }
 
-    pReader->iBlockIdx = 0;
-  }
+      // open
+      code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet);
+      if (code) goto _err;
 
-  while (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) {
-    SBlockIdx* pBlockIdx = taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+      // SBlockIdx
+      code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx, NULL);
+      if (code) goto _err;
 
-    pReader->iBlockIdx++;
+      pReader->iBlockIdx = 0;
+      pReader->pBlockIdx = NULL;
+    }
 
-    code = tsdbReadBlock(pReader->pDataFReader, pBlockIdx, &pReader->mBlock, NULL);
-    if (code) goto _err;
+    while (true) {
+      if (pReader->pBlockIdx == NULL) {
+        if (pReader->iBlockIdx >= taosArrayGetSize(pReader->aBlockIdx)) {
+          tsdbDataFReaderClose(&pReader->pDataFReader);
+          break;
+        }
 
-    break;
-  }
+        pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+        pReader->iBlockIdx++;
 
-  while (pReader->iBlock < pReader->mBlock.nItem) {
-    tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, pBlock, tGetBlock);
+        // SBlock
+        code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock, NULL);
+        if (code) goto _err;
 
-    pReader->iBlock++;
+        pReader->iBlock = 0;
+      }
 
-    if ((pBlock->minVersion >= pReader->sver && pBlock->minVersion <= pReader->ever) &&
-        (pBlock->maxVersion >= pReader->sver && pBlock->maxVersion <= pReader->ever)) {
-      // block in range, encode and return the data (todo)
-      goto _exit;
+      while (true) {
+        SBlock  block;
+        SBlock* pBlock = &block;
+
+        if (pReader->iBlock >= pReader->mBlock.nItem) {
+          pReader->pBlockIdx = NULL;
+          break;
+        }
+
+        tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, pBlock, tGetBlock);
+        pReader->iBlock++;
+
+        if ((pBlock->minVersion >= pReader->sver && pBlock->minVersion <= pReader->ever) ||
+            (pBlock->maxVersion >= pReader->sver && pBlock->maxVersion <= pReader->ever)) {
+          // overlap (todo)
+
+          code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->blkData, NULL, NULL);
+          if (code) goto _err;
+
+          goto _exit;
+        }
+      }
     }
   }
 

From e9d157dd2c66a4609d149531b86c668f757e10c7 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 5 Jul 2022 11:19:37 +0000
Subject: [PATCH 15/50] more snapshot

---
 source/dnode/vnode/inc/vnode.h             | 6 +++---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 3 ++-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 6 +++---
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 1c29470da3..0c0e663738 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -161,9 +161,9 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapRe
 int32_t vnodeSnapReaderClose(SVSnapReader *pReader);
 int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData);
 // SVSnapWriter
-int32_t vnodeSnapshotWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter);
-int32_t vnodeSnapshotWriterClose(SVSnapWriter *pWriter, int8_t rollback);
-int32_t vnodeSnapshotWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
+int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter);
+int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback);
+int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
 
 // structs
 struct STsdbCfg {
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 0f80ce080a..219d5edb7b 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -185,6 +185,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
   }
 
   code = TSDB_CODE_VND_READ_END;
+  tsdbDelFReaderClose(&pReader->pDelFReader);
 
 _exit:
   return code;
@@ -302,7 +303,7 @@ _err:
   return code;
 }
 
-// STsdbSnapReader ========================================
+// STsdbSnapWriter ========================================
 struct STsdbSnapWriter {
   STsdb*  pTsdb;
   int64_t sver;
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 410e66e464..c5353949ef 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -118,7 +118,7 @@ struct SVSnapWriter {
   int64_t ever;
 };
 
-int32_t vnodeSnapshotWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) {
+int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) {
   int32_t       code = 0;
   SVSnapWriter *pWriter = NULL;
 
@@ -138,7 +138,7 @@ _err:
   return code;
 }
 
-int32_t vnodeSnapshotWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
+int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
   int32_t code = 0;
 
   if (!rollback) {
@@ -151,7 +151,7 @@ int32_t vnodeSnapshotWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
   return code;
 }
 
-int32_t vnodeSnapshotWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
+int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
   int32_t code = 0;
   // TODO
   return code;

From aa8f1604f08e5608265da20129760d00e0ebb5df Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 5 Jul 2022 11:55:07 +0000
Subject: [PATCH 16/50] more snapshot writer

---
 source/dnode/vnode/src/inc/vnodeInt.h      |  7 ++++++
 source/dnode/vnode/src/meta/metaSnapshot.c | 25 ++++++++++++++++++++-
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 20 ++++++++++++++++-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 26 ++++++++++++++++++++--
 4 files changed, 74 insertions(+), 4 deletions(-)

diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 70a59ca2d7..d648044ebb 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -180,11 +180,18 @@ int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapRe
 int32_t metaSnapReaderClose(SMetaSnapReader** ppReader);
 int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData);
 // SMetaSnapWriter ========================================
+int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter);
+int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
+int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback);
 // STsdbSnapReader ========================================
 int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapReader** ppReader);
 int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader);
 int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData);
 // STsdbSnapWriter ========================================
+int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter);
+int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
+int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback);
+
 typedef struct {
   int8_t  streamType;  // sma or other
   int8_t  dstType;
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index e3124096bd..59b9d566d5 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -101,4 +101,27 @@ _exit:
   return code;
 }
 
-// SMetaSnapWriter ========================================
\ No newline at end of file
+// SMetaSnapWriter ========================================
+struct SMetaSnapWriter {
+  SMeta*  pMeta;
+  int64_t sver;
+  int64_t ever;
+};
+
+int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 219d5edb7b..1bc2e45a41 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -313,4 +313,22 @@ struct STsdbSnapWriter {
   SDataFWriter* pDataFWriter;
   // for del file
   SDelFWriter* pDelFWriter;
-};
\ No newline at end of file
+};
+
+int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index c5353949ef..b1b30c79a2 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -116,6 +116,10 @@ struct SVSnapWriter {
   SVnode *pVnode;
   int64_t sver;
   int64_t ever;
+  // meta
+  SMetaSnapWriter *pMetaSnapWriter;
+  // tsdb
+  STsdbSnapWriter *pTsdbSnapWriter;
 };
 
 int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) {
@@ -152,7 +156,25 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
 }
 
 int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
-  int32_t code = 0;
-  // TODO
+  int32_t       code = 0;
+  SSnapDataHdr *pSnapDataHdr = (SSnapDataHdr *)pData;
+  SVnode       *pVnode = pWriter->pVnode;
+
+  ASSERT(pSnapDataHdr->size + sizeof(SSnapDataHdr) == nData);
+
+  if (pSnapDataHdr->type == 0) {
+    // meta
+    if (pWriter->pMetaSnapWriter == NULL) {
+      code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
+      if (code) goto _err;
+    }
+  } else {
+    // tsdb
+  }
+
+  return code;
+
+_err:
+  vError("vgId:%d vnode snapshot write failed since %s", TD_VID(pVnode), tstrerror(code));
   return code;
 }
\ No newline at end of file

From a2d5c964e4b617f787eb8144cd995f5c3a5060fc Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 5 Jul 2022 12:08:10 +0000
Subject: [PATCH 17/50] more snapshot writer

---
 source/dnode/vnode/src/meta/metaSnapshot.c |  4 ++--
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 10 ++++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 59b9d566d5..7a7f259f45 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -114,13 +114,13 @@ int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWr
   return code;
 }
 
-int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
   int32_t code = 0;
   // TODO
   return code;
 }
 
-int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
+int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t code = 0;
   // TODO
   return code;
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index b1b30c79a2..1bfce15528 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -168,8 +168,18 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
       code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
       if (code) goto _err;
     }
+
+    code = metaSnapWrite(pWriter->pMetaSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
+    if (code) goto _err;
   } else {
     // tsdb
+    if (pWriter->pTsdbSnapWriter == NULL) {
+      code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter);
+      if (code) goto _err;
+    }
+
+    code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
+    if (code) goto _err;
   }
 
   return code;

From 947359fd17b8dcef723c61b3fea968e5530189be Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 05:51:45 +0000
Subject: [PATCH 18/50] more work

---
 source/dnode/vnode/src/meta/metaSnapshot.c | 48 +++++++++++++++++-
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 59 +++++++++++++++++++---
 2 files changed, 98 insertions(+), 9 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 7a7f259f45..296d5fb3b6 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -108,18 +108,62 @@ struct SMetaSnapWriter {
   int64_t ever;
 };
 
-int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) {
+static int32_t metaSnapRollback(SMetaSnapWriter* pWriter) {
   int32_t code = 0;
   // TODO
   return code;
 }
 
-int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
+static int32_t metaSnapCommit(SMetaSnapWriter* pWriter) {
   int32_t code = 0;
   // TODO
   return code;
 }
 
+int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) {
+  int32_t          code = 0;
+  SMetaSnapWriter* pWriter;
+
+  // alloc
+  pWriter = (SMetaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->pMeta = pMeta;
+  pWriter->sver = sver;
+  pWriter->ever = ever;
+
+  *ppWriter = pWriter;
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot writer open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
+  *ppWriter = NULL;
+  return code;
+}
+
+int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
+  int32_t          code = 0;
+  SMetaSnapWriter* pWriter = *ppWriter;
+
+  if (rollback) {
+    code = metaSnapRollback(pWriter);
+    if (code) goto _err;
+  } else {
+    code = metaSnapCommit(pWriter);
+    if (code) goto _err;
+  }
+  taosMemoryFree(pWriter);
+  *ppWriter = NULL;
+
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot writer close failed since %s", TD_VID(pWriter->pMeta->pVnode), tstrerror(code));
+  return code;
+}
+
 int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t code = 0;
   // TODO
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 1bc2e45a41..dc2e006d05 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -315,20 +315,65 @@ struct STsdbSnapWriter {
   SDelFWriter* pDelFWriter;
 };
 
-int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) {
+static int32_t tsdbSnapRollback(STsdbSnapWriter* pWriter) {
   int32_t code = 0;
   // TODO
   return code;
 }
 
+static int32_t tsdbSnapCommit(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) {
+  int32_t          code = 0;
+  STsdbSnapWriter* pWriter = NULL;
+
+  // alloc
+  pWriter = (STsdbSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->pTsdb = pTsdb;
+  pWriter->sver = sver;
+  pWriter->ever = ever;
+
+  *ppWriter = pWriter;
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  *ppWriter = NULL;
+  return code;
+}
+
+int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
+  int32_t          code = 0;
+  STsdbSnapWriter* pWriter = *ppWriter;
+
+  if (rollback) {
+    code = tsdbSnapRollback(pWriter);
+    if (code) goto _err;
+  } else {
+    code = tsdbSnapCommit(pWriter);
+    if (code) goto _err;
+  }
+
+  taosMemoryFree(pWriter);
+  *ppWriter = NULL;
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot writer close failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
 int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t code = 0;
   // TODO
   return code;
 }
-
-int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}

From 1cce6e2d80146289fb22e9a64f524e5f1da591a1 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 06:07:53 +0000
Subject: [PATCH 19/50] more writer code

---
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 25 +++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 1bfce15528..27f30ec787 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -122,6 +122,18 @@ struct SVSnapWriter {
   STsdbSnapWriter *pTsdbSnapWriter;
 };
 
+static int32_t vnodeSnapRollback(SVSnapWriter *pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
+static int32_t vnodeSnapCommit(SVSnapWriter *pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
 int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) {
   int32_t       code = 0;
   SVSnapWriter *pWriter = NULL;
@@ -139,20 +151,27 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr
   return code;
 
 _err:
+  vError("vgId:%d vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
   return code;
 }
 
 int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
   int32_t code = 0;
 
-  if (!rollback) {
-    // apply the change
+  if (rollback) {
+    code = vnodeSnapRollback(pWriter);
+    if (code) goto _err;
   } else {
-    // rollback the change
+    code = vnodeSnapCommit(pWriter);
+    if (code) goto _err;
   }
 
   taosMemoryFree(pWriter);
   return code;
+
+_err:
+  vError("vgId:%d vnode snapshow writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code));
+  return code;
 }
 
 int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {

From 9720f0549193c887a88c1ef4e048aff71df7c766 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 06:59:53 +0000
Subject: [PATCH 20/50] more tsdb snapshow writer

---
 source/dnode/vnode/src/inc/tsdb.h          |  1 +
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 73 +++++++++++++++++++++-
 2 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 0f65a536e0..eec9c0f312 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -64,6 +64,7 @@ typedef struct SRowIter       SRowIter;
 typedef struct STsdbFS        STsdbFS;
 typedef struct SRowMerger     SRowMerger;
 typedef struct STsdbFSState   STsdbFSState;
+typedef struct STsdbSnapHdr   STsdbSnapHdr;
 
 #define TSDB_MAX_SUBBLOCKS 8
 #define TSDB_FHDR_SIZE     512
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index dc2e006d05..133a519aa2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -308,8 +308,11 @@ struct STsdbSnapWriter {
   STsdb*  pTsdb;
   int64_t sver;
   int64_t ever;
+  // config
+  int32_t minutes;
+  int8_t  precision;
   // for data file
-  int32_t       iDFileSet;
+  int32_t       fid;
   SDataFWriter* pDataFWriter;
   // for del file
   SDelFWriter* pDelFWriter;
@@ -327,6 +330,47 @@ static int32_t tsdbSnapCommit(STsdbSnapWriter* pWriter) {
   return code;
 }
 
+static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+  int64_t suid = 0;  // todo
+  int64_t uid = 0;   // todo
+  int64_t skey;      // todo
+  int64_t ekey;      // todo
+
+  int32_t fid = tsdbKeyFid(skey, pWriter->minutes, pWriter->precision);
+  ASSERT(fid == tsdbKeyFid(ekey, pWriter->minutes, pWriter->precision));
+  if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) {
+    if (pWriter->pDataFWriter) {
+      // finish current file and close the SDataFWriter
+    }
+
+    pWriter->fid = fid;
+    SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid);
+    SDFileSet  wSet = {0};
+    if (pSet == NULL) {
+      wSet = (SDFileSet){0};  // todo
+    } else {
+      wSet = (SDFileSet){0};  // todo
+    }
+
+    code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet);
+    if (code) goto _err;
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot write data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
 int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) {
   int32_t          code = 0;
   STsdbSnapWriter* pWriter = NULL;
@@ -374,6 +418,31 @@ _err:
 
 int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t code = 0;
-  // TODO
+  int8_t  type = pData[0];
+
+  // ts data
+  if (type == 0) {
+    code = tsdbSnapWriteData(pWriter, pData + 1, nData - 1);
+    if (code) goto _err;
+  } else {
+    if (pWriter->pDataFWriter) {
+      // commit the remain data of the FSet (todo)
+
+      // close and update the file
+      code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1);
+      if (code) goto _err;
+    }
+  }
+
+  // del data
+  if (type == 1) {
+    code = tsdbSnapWriteDel(pWriter, pData + 1, nData - 1);
+    if (code) goto _err;
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshow write failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }

From c70d2a578a148a09dc0be88c4335f8f44cfe3d37 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 07:47:16 +0000
Subject: [PATCH 21/50] more tsdb del snapshot writer

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 94 +++++++++++++++++++++-
 1 file changed, 93 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 133a519aa2..984002fb24 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -313,9 +313,19 @@ struct STsdbSnapWriter {
   int8_t  precision;
   // for data file
   int32_t       fid;
+  SDataFReader* pDataFReader;
+  SArray*       aBlockIdx;
+  SMapData      mBlock;
   SDataFWriter* pDataFWriter;
+  SArray*       aBlockIdxN;
+  SMapData      mBlockN;
   // for del file
+  SDelFReader* pDelFReader;
   SDelFWriter* pDelFWriter;
+  int32_t      iDelIdx;
+  SArray*      aDelIdx;
+  SArray*      aDelData;
+  SArray*      aDelIdxN;
 };
 
 static int32_t tsdbSnapRollback(STsdbSnapWriter* pWriter) {
@@ -367,7 +377,89 @@ _err:
 
 static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t code = 0;
-  // TODO
+  STsdb*  pTsdb = pWriter->pTsdb;
+
+  if (pWriter->pDelFWriter == NULL) {
+    SDelFile* pDelFile = tsdbFSStateGetDelFile(pTsdb->fs->nState);
+
+    // reader
+    if (pDelFile) {
+      code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL);
+      if (code) goto _err;
+
+      code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdx, NULL);
+      if (code) goto _err;
+    }
+
+    // writer
+    SDelFile delFile = {.commitID = pTsdb->pVnode->state.commitID, .offset = 0, .size = 0};
+    code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb);
+    if (code) goto _err;
+  }
+
+  // process the del data
+  TABLEID id = {0};  // todo
+
+  while (true) {
+    SDelIdx* pDelIdx = NULL;
+    int64_t  n = 0;
+    SDelData delData;
+    SDelIdx  delIdx;
+    int8_t   toBreak = 0;
+
+    if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx)) {
+      pDelIdx = taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx);
+    }
+
+    if (pDelIdx) {
+      int32_t c = tTABLEIDCmprFn(&id, pDelIdx);
+      if (c < 0) {
+        goto _new_del;
+      } else {
+        code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL);
+        if (code) goto _err;
+
+        pWriter->iDelIdx++;
+        if (c == 0) {
+          toBreak = 1;
+          goto _merge_del;
+        } else {
+          goto _write_del;
+        }
+      }
+    }
+
+  _new_del:
+    toBreak = 1;
+    taosArrayClear(pWriter->aDelData);
+
+  _merge_del:
+    while (n < nData) {
+      n += tGetDelData(pData + n, &delData);
+      if (taosArrayPush(pWriter->aDelData, &delData) == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto _err;
+      }
+    }
+
+  _write_del:
+    delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid};
+    code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx);
+    if (code) goto _err;
+
+    if (taosArrayPush(pWriter->aDelIdxN, &delIdx) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+
+    if (toBreak) break;
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot write del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 

From 8d92366ed1eddec2af6e82f9f99575a5f7e3a1df Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 08:07:02 +0000
Subject: [PATCH 22/50] more work

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 984002fb24..963288566a 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -463,6 +463,12 @@ _err:
   return code;
 }
 
+static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  // TODO
+  return code;
+}
+
 int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) {
   int32_t          code = 0;
   STsdbSnapWriter* pWriter = NULL;

From d4a5d5118a14032246c45b2ff9f329f76efb15ab Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 08:24:46 +0000
Subject: [PATCH 23/50] finish del data writer

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 43 +++++++++++++++++++++-
 1 file changed, 41 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 963288566a..9dc69d6b01 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -422,8 +422,10 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
         pWriter->iDelIdx++;
         if (c == 0) {
           toBreak = 1;
+          delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid};
           goto _merge_del;
         } else {
+          delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid};
           goto _write_del;
         }
       }
@@ -431,6 +433,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
 
   _new_del:
     toBreak = 1;
+    delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid};
     taosArrayClear(pWriter->aDelData);
 
   _merge_del:
@@ -443,7 +446,6 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
     }
 
   _write_del:
-    delIdx = (SDelIdx){.suid = id.suid, .uid = id.uid};
     code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx);
     if (code) goto _err;
 
@@ -465,7 +467,44 @@ _err:
 
 static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
   int32_t code = 0;
-  // TODO
+  STsdb*  pTsdb = pWriter->pTsdb;
+
+  if (pWriter->pDelFWriter == NULL) goto _exit;
+  for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx); pWriter->iDelIdx++) {
+    SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx);
+
+    code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL);
+    if (code) goto _err;
+
+    SDelIdx delIdx = (SDelIdx){.suid = pDelIdx->suid, .uid = pDelIdx->uid};
+    code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx);
+    if (code) goto _err;
+
+    if (taosArrayPush(pWriter->aDelIdx, &delIdx) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+  }
+
+  code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter);
+  if (code) goto _err;
+
+  code = tsdbFSStateUpsertDelFile(pTsdb->fs->nState, &pWriter->pDelFWriter->fDel);
+  if (code) goto _err;
+
+  code = tsdbDelFWriterClose(&pWriter->pDelFWriter, 1);
+  if (code) goto _err;
+
+  if (pWriter->pDelFReader) {
+    code = tsdbDelFReaderClose(&pWriter->pDelFReader);
+    if (code) goto _err;
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshow write del end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 

From e6b0271819d0079fefa789e831bfef62a28ce489 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 08:28:59 +0000
Subject: [PATCH 24/50] more code

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 26 ++++++++++++++++------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 9dc69d6b01..59344f7e63 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -465,6 +465,17 @@ _err:
   return code;
 }
 
+static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+
+_exit:
+  return code;
+
+_err:
+  return code;
+}
+
 static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
   int32_t code = 0;
   STsdb*  pTsdb = pWriter->pTsdb;
@@ -539,6 +550,12 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
     code = tsdbSnapRollback(pWriter);
     if (code) goto _err;
   } else {
+    code = tsdbSnapWriteDataEnd(pWriter);
+    if (code) goto _err;
+
+    code = tsdbSnapWriteDelEnd(pWriter);
+    if (code) goto _err;
+
     code = tsdbSnapCommit(pWriter);
     if (code) goto _err;
   }
@@ -562,13 +579,8 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
     code = tsdbSnapWriteData(pWriter, pData + 1, nData - 1);
     if (code) goto _err;
   } else {
-    if (pWriter->pDataFWriter) {
-      // commit the remain data of the FSet (todo)
-
-      // close and update the file
-      code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1);
-      if (code) goto _err;
-    }
+    code = tsdbSnapWriteDataEnd(pWriter);
+    if (code) goto _err;
   }
 
   // del data

From 409de824a269d9bf6d8861da05574ac0edaca1f8 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 09:36:14 +0000
Subject: [PATCH 25/50] more code

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 46 +++++++++++++++-------
 1 file changed, 31 insertions(+), 15 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 59344f7e63..f22cefd4f1 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -340,6 +340,30 @@ static int32_t tsdbSnapCommit(STsdbSnapWriter* pWriter) {
   return code;
 }
 
+static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+
+  if (pWriter->pDataFWriter == NULL) goto _exit;
+
+  // TODO
+
+  code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 0);
+  if (code) goto _err;
+
+  if (pWriter->pDataFReader) {
+    code = tsdbDataFReaderClose(&pWriter->pDataFReader);
+    if (code) goto _err;
+  }
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot writer data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
 static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t code = 0;
   STsdb*  pTsdb = pWriter->pTsdb;
@@ -350,14 +374,17 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
 
   int32_t fid = tsdbKeyFid(skey, pWriter->minutes, pWriter->precision);
   ASSERT(fid == tsdbKeyFid(ekey, pWriter->minutes, pWriter->precision));
+
   if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) {
-    if (pWriter->pDataFWriter) {
-      // finish current file and close the SDataFWriter
-    }
+    code = tsdbSnapWriteDataEnd(pWriter);
+    if (code) goto _err;
 
     pWriter->fid = fid;
     SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid);
-    SDFileSet  wSet = {0};
+    // reader
+
+    // writer
+    SDFileSet wSet = {0};
     if (pSet == NULL) {
       wSet = (SDFileSet){0};  // todo
     } else {
@@ -465,17 +492,6 @@ _err:
   return code;
 }
 
-static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
-  int32_t code = 0;
-  STsdb*  pTsdb = pWriter->pTsdb;
-
-_exit:
-  return code;
-
-_err:
-  return code;
-}
-
 static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
   int32_t code = 0;
   STsdb*  pTsdb = pWriter->pTsdb;

From 0901640ba23f1ee3cd30cd9b172b4dcbca825fdd Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 09:46:14 +0000
Subject: [PATCH 26/50] make snapshot compile

---
 include/libs/transport/trpc.h            | 2 +-
 source/dnode/vnode/src/inc/vnodeInt.h    | 2 ++
 source/dnode/vnode/src/vnd/vnodeCommit.c | 6 +++++-
 source/dnode/vnode/src/vnd/vnodeOpen.c   | 9 ++++++++-
 source/dnode/vnode/src/vnd/vnodeSvr.c    | 6 ++++--
 source/dnode/vnode/src/vnd/vnodeSync.c   | 7 +++----
 6 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h
index d59a0a64b3..d8dea8a1be 100644
--- a/include/libs/transport/trpc.h
+++ b/include/libs/transport/trpc.h
@@ -27,7 +27,7 @@ extern "C" {
 
 #define TAOS_CONN_SERVER 0
 #define TAOS_CONN_CLIENT 1
-#define IsReq(pMsg) (pMsg->msgType & 1U)
+#define IsReq(pMsg)      (pMsg->msgType & 1U)
 
 extern int32_t tsRpcHeadSize;
 
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index c5e9b794da..f0fb6c2d61 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -212,7 +212,9 @@ typedef struct {
 struct SVState {
   int64_t committed;
   int64_t applied;
+  int64_t applyTerm;
   int64_t commitID;
+  int64_t commitTerm;
 };
 
 struct SVnodeInfo {
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index 21db14f0df..ed829666cd 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -223,6 +223,7 @@ int vnodeCommit(SVnode *pVnode) {
   // save info
   info.config = pVnode->config;
   info.state.committed = pVnode->state.applied;
+  info.state.commitTerm = pVnode->state.applyTerm;
   info.state.commitID = pVnode->state.commitID;
   snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
   if (vnodeSaveInfo(dir, &info) < 0) {
@@ -270,7 +271,7 @@ int vnodeCommit(SVnode *pVnode) {
     ASSERT(0);
     return -1;
   }
-  
+
   pVnode->state.committed = info.state.committed;
 
   // postCommit
@@ -316,6 +317,7 @@ static int vnodeEncodeState(const void *pObj, SJson *pJson) {
 
   if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "commit ID", pState->commitID) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "commit term", pState->commitTerm) < 0) return -1;
 
   return 0;
 }
@@ -328,6 +330,8 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
   if (code < 0) return -1;
   tjsonGetNumberValue(pJson, "commit ID", pState->commitID, code);
   if (code < 0) return -1;
+  tjsonGetNumberValue(pJson, "commit term", pState->commitTerm, code);
+  if (code < 0) return -1;
 
   return 0;
 }
diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index 4267dd9b1f..e59f8ae558 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -79,8 +79,10 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   strcpy(pVnode->path, path);
   pVnode->config = info.config;
   pVnode->state.committed = info.state.committed;
+  pVnode->state.commitTerm = info.state.commitTerm;
   pVnode->state.applied = info.state.committed;
   pVnode->state.commitID = info.state.commitID;
+  pVnode->state.commitTerm = info.state.commitTerm;
   pVnode->pTfs = pTfs;
   pVnode->msgCb = msgCb;
   pVnode->blockCount = 0;
@@ -194,4 +196,9 @@ void vnodeStop(SVnode *pVnode) {}
 
 int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; }
 
-void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; }
+void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) {
+  pSnapshot->data = NULL;
+  pSnapshot->lastApplyIndex = pVnode->state.committed;
+  pSnapshot->lastApplyTerm = pVnode->state.commitTerm;
+  pSnapshot->lastConfigIndex = -1;
+}
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index 4530eee4a5..f7f3128b7d 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -143,6 +143,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
          version);
 
   pVnode->state.applied = version;
+  pVnode->state.applyTerm = pMsg->info.conn.applyTerm;
 
   // skip header
   pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
@@ -169,7 +170,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
       if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_DROP_TTL_TABLE:
-      //if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
+      // if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
       break;
     case TDMT_VND_CREATE_SMA: {
       if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
@@ -799,7 +800,8 @@ _exit:
   taosArrayDestroy(submitRsp.pArray);
 
   // TODO: the partial success scenario and the error case
-  // => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level 1/level 2.
+  // => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level
+  // 1/level 2.
   // TODO: refactor
   if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
     tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);
diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index 399a5b6299..07d0380151 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -432,20 +432,19 @@ static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta
 static int32_t vnodeSnapshotStartRead(struct SSyncFSM *pFsm, void *pParam, void **ppReader) {
   SVnode         *pVnode = pFsm->data;
   SSnapshotParam *pSnapshotParam = pParam;
-  int32_t         code =
-      vnodeSnapshotReaderOpen(pVnode, (SVSnapshotReader **)ppReader, pSnapshotParam->start, pSnapshotParam->end);
+  int32_t code = vnodeSnapReaderOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapReader **)ppReader);
   return code;
 }
 
 static int32_t vnodeSnapshotStopRead(struct SSyncFSM *pFsm, void *pReader) {
   SVnode *pVnode = pFsm->data;
-  int32_t code = vnodeSnapshotReaderClose(pReader);
+  int32_t code = vnodeSnapReaderClose(pReader);
   return code;
 }
 
 static int32_t vnodeSnapshotDoRead(struct SSyncFSM *pFsm, void *pReader, void **ppBuf, int32_t *len) {
   SVnode *pVnode = pFsm->data;
-  int32_t code = vnodeSnapshotRead(pReader, (const void **)ppBuf, len);
+  int32_t code = vnodeSnapRead(pReader, (uint8_t **)ppBuf, len);
   return code;
 }
 

From aa27f51deabb21d14268248476a48882fe37bf80 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 10:37:26 +0000
Subject: [PATCH 27/50] some retention code

---
 source/dnode/vnode/CMakeLists.txt           |  1 +
 source/dnode/vnode/src/tsdb/tsdbRetention.c | 42 +++++++++++++++++++++
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c  | 16 ++++++++
 3 files changed, 59 insertions(+)
 create mode 100644 source/dnode/vnode/src/tsdb/tsdbRetention.c

diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index a15f6fe4ee..35483397a0 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -50,6 +50,7 @@ target_sources(
     "src/tsdb/tsdbUtil.c"
     "src/tsdb/tsdbSnapshot.c"
     "src/tsdb/tsdbCacheRead.c"
+    "src/tsdb/tsdbRetention.c"
 
     # tq
     "src/tq/tq.c"
diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c
new file mode 100644
index 0000000000..e73f3f947c
--- /dev/null
+++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdb.h"
+
+int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
+  int32_t code = 0;
+
+  // begin
+  code = tsdbFSBegin(pTsdb->fs);
+  if (code) goto _err;
+
+  // do retention
+  for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs->nState->aDFileSet); iSet++) {
+    SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pTsdb->fs->nState->aDFileSet, iSet);
+
+    // TODO
+  }
+
+  // commit
+  code = tsdbFSCommit(pTsdb->fs);
+  if (code) goto _err;
+
+_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb do retention failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index f22cefd4f1..050891c9da 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -315,7 +315,11 @@ struct STsdbSnapWriter {
   int32_t       fid;
   SDataFReader* pDataFReader;
   SArray*       aBlockIdx;
+  int32_t       iBlockIdx;
   SMapData      mBlock;
+  int32_t       iBlock;
+  SBlockData    blockData;
+  int32_t       iRow;
   SDataFWriter* pDataFWriter;
   SArray*       aBlockIdxN;
   SMapData      mBlockN;
@@ -382,6 +386,16 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     pWriter->fid = fid;
     SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid);
     // reader
+    if (pSet) {
+      code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
+      if (code) goto _err;
+
+      code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx, NULL);
+      if (code) goto _err;
+    } else {
+      taosArrayClear(pWriter->aBlockIdx);
+    }
+    pWriter->iBlockIdx = 0;
 
     // writer
     SDFileSet wSet = {0};
@@ -393,6 +407,8 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
 
     code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet);
     if (code) goto _err;
+
+    taosArrayClear(pWriter->aBlockIdxN);
   }
 
   return code;

From 9a90714ecc26e1427c203365ccb51ef5530b7dbf Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 10:44:28 +0000
Subject: [PATCH 28/50] more work

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 24 ++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 050891c9da..502e08a381 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -379,6 +379,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
   int32_t fid = tsdbKeyFid(skey, pWriter->minutes, pWriter->precision);
   ASSERT(fid == tsdbKeyFid(ekey, pWriter->minutes, pWriter->precision));
 
+  // begin
   if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) {
     code = tsdbSnapWriteDataEnd(pWriter);
     if (code) goto _err;
@@ -411,6 +412,29 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     taosArrayClear(pWriter->aBlockIdxN);
   }
 
+  // process
+  TABLEID id = {0};  // TODO
+
+  while (true) {
+    SBlockIdx* pBlockIdx = NULL;
+
+    if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
+      pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
+    }
+
+    if (pBlockIdx) {
+      int32_t c = tTABLEIDCmprFn(&id, pBlockIdx);
+
+      if (c == 0) {
+        // merge
+      } else if (c < 0) {
+        // write the block
+      } else {
+        // commit the block
+      }
+    }
+  }
+
   return code;
 
 _err:

From be85e27234312927431355aac188c7e54809ea23 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 12:03:08 +0000
Subject: [PATCH 29/50] more tsdb snapshot writer

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 33 +++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 502e08a381..b4547a6cc9 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -323,6 +323,7 @@ struct STsdbSnapWriter {
   SDataFWriter* pDataFWriter;
   SArray*       aBlockIdxN;
   SMapData      mBlockN;
+  SBlockData    nBlockData;
   // for del file
   SDelFReader* pDelFReader;
   SDelFWriter* pDelFWriter;
@@ -368,6 +369,24 @@ _err:
   return code;
 }
 
+static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
+  int32_t     code = 0;
+  int32_t     iRow = 0;           // todo
+  int32_t     nRow = 0;           // todo
+  SBlockData* pBlockData = NULL;  // todo
+
+  while (iRow < nRow) {
+    code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
+    if (code) goto _err;
+  }
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb snapshot write append data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
 static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t code = 0;
   STsdb*  pTsdb = pWriter->pTsdb;
@@ -428,9 +447,21 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
       if (c == 0) {
         // merge
       } else if (c < 0) {
-        // write the block
+        code = tsdbSnapWriteAppendData(pWriter, pData, nData);
+        if (code) goto _err;
       } else {
         // commit the block
+        code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL);
+        if (code) goto _err;
+
+        SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
+        code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlock, NULL, &blockIdx);
+        if (code) goto _err;
+
+        if (taosArrayPush(pWriter->aBlockIdxN, &blockIdx) == NULL) {
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto _err;
+        }
       }
     }
   }

From b3da0e2a159bca6d97dc146e12d452215dd57dfd Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Jul 2022 12:21:59 +0000
Subject: [PATCH 30/50] meta snapshot write

---
 source/dnode/vnode/src/inc/meta.h          |   3 +
 source/dnode/vnode/src/meta/metaSnapshot.c |  17 +++-
 source/dnode/vnode/src/meta/metaTable.c    | 101 ++++++++++-----------
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c |   2 +
 4 files changed, 69 insertions(+), 54 deletions(-)

diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h
index 66d1689d57..e08925acc3 100644
--- a/source/dnode/vnode/src/inc/meta.h
+++ b/source/dnode/vnode/src/inc/meta.h
@@ -57,6 +57,9 @@ int  metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
 // metaCommit ==================
 static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); }
 
+// metaTable ==================
+int metaHandleEntry(SMeta* pMeta, const SMetaEntry* pME);
+
 struct SMeta {
   TdThreadRwlock lock;
 
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 296d5fb3b6..ac84842e85 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -165,7 +165,20 @@ _err:
 }
 
 int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
-  int32_t code = 0;
-  // TODO
+  int32_t    code = 0;
+  SMeta*     pMeta = pWriter->pMeta;
+  SMetaEntry metaEntry = {0};
+  SDecoder*  pDecoder = &(SDecoder){0};
+
+  tDecoderInit(pDecoder, pData, nData);
+  metaDecodeEntry(pDecoder, &metaEntry);
+
+  code = metaHandleEntry(pMeta, &metaEntry);
+  if (code) goto _err;
+
+  return code;
+
+_err:
+  metaError("vgId:%d meta snapshot write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
   return code;
 }
\ No newline at end of file
diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index a621b4ddb0..2b4106182c 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -16,7 +16,6 @@
 #include "meta.h"
 
 static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
-static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME);
@@ -50,7 +49,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
   if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
     return -1;
   }
-  void *      data = pCtbEntry->ctbEntry.pTags;
+  void       *data = pCtbEntry->ctbEntry.pTags;
   const char *tagName = pSchema->name;
 
   tb_uid_t    suid = pCtbEntry->ctbEntry.suid;
@@ -69,7 +68,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
     STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
     char     type = pTagVal->type;
 
-    char *  key = pTagVal->pKey;
+    char   *key = pTagVal->pKey;
     int32_t nKey = strlen(key);
 
     SIndexTerm *term = NULL;
@@ -77,13 +76,13 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
       // handle null value
     } else if (type == TSDB_DATA_TYPE_NCHAR) {
       if (pTagVal->nData > 0) {
-        char *  val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
+        char   *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
         int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
         memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
         type = TSDB_DATA_TYPE_VARCHAR;
         term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, val, len);
       } else if (pTagVal->nData == 0) {
-        char *  val = NULL;
+        char   *val = NULL;
         int32_t len = 0;
         // handle NULL key
       }
@@ -112,9 +111,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   int         vLen = 0;
   const void *pKey = NULL;
   const void *pVal = NULL;
-  void *      pBuf = NULL;
+  void       *pBuf = NULL;
   int32_t     szBuf = 0;
-  void *      p = NULL;
+  void       *p = NULL;
   SMetaReader mr = {0};
 
   // validate req
@@ -172,7 +171,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq) {
   }
 
   // drop all child tables
-  TBC *   pCtbIdxc = NULL;
+  TBC    *pCtbIdxc = NULL;
   SArray *pArray = taosArrayInit(8, sizeof(tb_uid_t));
 
   tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn);
@@ -228,8 +227,8 @@ _exit:
 int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   SMetaEntry  oStbEntry = {0};
   SMetaEntry  nStbEntry = {0};
-  TBC *       pUidIdxc = NULL;
-  TBC *       pTbDbc = NULL;
+  TBC        *pUidIdxc = NULL;
+  TBC        *pTbDbc = NULL;
   const void *pData;
   int         nData;
   int64_t     oversion;
@@ -352,7 +351,7 @@ _err:
 }
 
 int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) {
-  void *   pData = NULL;
+  void    *pData = NULL;
   int      nData = 0;
   int      rc = 0;
   tb_uid_t uid;
@@ -380,22 +379,22 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
 int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) {
   metaWLock(pMeta);
   int ret = metaTtlSmaller(pMeta, ttl, tbUids);
-  if(ret != 0){
+  if (ret != 0) {
     metaULock(pMeta);
     return ret;
   }
   for (int i = 0; i < taosArrayGetSize(tbUids); ++i) {
     tb_uid_t *uid = (tb_uid_t *)taosArrayGet(tbUids, i);
     metaDropTableByUid(pMeta, *uid, NULL);
-    metaDebug("ttl drop table:%"PRId64, *uid);
+    metaDebug("ttl drop table:%" PRId64, *uid);
   }
   metaULock(pMeta);
   return 0;
 }
 
-static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){
-  int64_t    ttlDays;
-  int64_t    ctime;
+static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME) {
+  int64_t ttlDays;
+  int64_t ctime;
   if (pME->type == TSDB_CHILD_TABLE) {
     ctime = pME->ctbEntry.ctime;
     ttlDays = pME->ctbEntry.ttlDays;
@@ -415,13 +414,12 @@ static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){
 static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
   STtlIdxKey ttlKey = {0};
   metaBuildTtlIdxKey(&ttlKey, pME);
-  if(ttlKey.dtime == 0) return 0;
+  if (ttlKey.dtime == 0) return 0;
   return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), &pMeta->txn);
 }
 
-
 static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
-  void *     pData = NULL;
+  void      *pData = NULL;
   int        nData = 0;
   int        rc = 0;
   SMetaEntry e = {0};
@@ -440,8 +438,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
   tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pMeta->txn);
   tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, &pMeta->txn);
   tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn);
-  if(e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e);
-
+  if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e);
 
   if (e.type == TSDB_CHILD_TABLE) {
     tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), &pMeta->txn);
@@ -459,14 +456,14 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) {
 }
 
 static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) {
-  void *          pVal = NULL;
+  void           *pVal = NULL;
   int             nVal = 0;
-  const void *    pData = NULL;
+  const void     *pData = NULL;
   int             nData = 0;
   int             ret = 0;
   tb_uid_t        uid;
   int64_t         oversion;
-  SSchema *       pColumn = NULL;
+  SSchema        *pColumn = NULL;
   SMetaEntry      entry = {0};
   SSchemaWrapper *pSchema;
   int             c;
@@ -620,7 +617,7 @@ _err:
 static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
   SMetaEntry  ctbEntry = {0};
   SMetaEntry  stbEntry = {0};
-  void *      pVal = NULL;
+  void       *pVal = NULL;
   int         nVal = 0;
   int         ret;
   int         c;
@@ -651,7 +648,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   oversion = *(int64_t *)pData;
 
   // search table.db
-  TBC *    pTbDbc = NULL;
+  TBC     *pTbDbc = NULL;
   SDecoder dc1 = {0};
   SDecoder dc2 = {0};
 
@@ -675,7 +672,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
   metaDecodeEntry(&dc2, &stbEntry);
 
   SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
-  SSchema *       pColumn = NULL;
+  SSchema        *pColumn = NULL;
   int32_t         iCol = 0;
   for (;;) {
     pColumn = NULL;
@@ -705,8 +702,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
     memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
   } else {
     const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags;
-    STag *      pNewTag = NULL;
-    SArray *    pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal));
+    STag       *pNewTag = NULL;
+    SArray     *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal));
     if (!pTagArray) {
       terrno = TSDB_CODE_OUT_OF_MEMORY;
       goto _err;
@@ -765,15 +762,15 @@ _err:
 }
 
 static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
-  void *          pVal = NULL;
-  int             nVal = 0;
-  const void *    pData = NULL;
-  int             nData = 0;
-  int             ret = 0;
-  tb_uid_t        uid;
-  int64_t         oversion;
-  SMetaEntry      entry = {0};
-  int             c = 0;
+  void       *pVal = NULL;
+  int         nVal = 0;
+  const void *pData = NULL;
+  int         nData = 0;
+  int         ret = 0;
+  tb_uid_t    uid;
+  int64_t     oversion;
+  SMetaEntry  entry = {0};
+  int         c = 0;
 
   // search name index
   ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal);
@@ -816,22 +813,22 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
   metaWLock(pMeta);
   // build SMetaEntry
   if (entry.type == TSDB_CHILD_TABLE) {
-    if(pAlterTbReq->updateTTL) {
+    if (pAlterTbReq->updateTTL) {
       metaDeleteTtlIdx(pMeta, &entry);
       entry.ctbEntry.ttlDays = pAlterTbReq->newTTL;
       metaUpdateTtlIdx(pMeta, &entry);
     }
-    if(pAlterTbReq->newCommentLen >= 0) {
+    if (pAlterTbReq->newCommentLen >= 0) {
       entry.ctbEntry.commentLen = pAlterTbReq->newCommentLen;
       entry.ctbEntry.comment = pAlterTbReq->newComment;
     }
   } else {
-    if(pAlterTbReq->updateTTL) {
+    if (pAlterTbReq->updateTTL) {
       metaDeleteTtlIdx(pMeta, &entry);
       entry.ntbEntry.ttlDays = pAlterTbReq->newTTL;
       metaUpdateTtlIdx(pMeta, &entry);
     }
-    if(pAlterTbReq->newCommentLen >= 0) {
+    if (pAlterTbReq->newCommentLen >= 0) {
       entry.ntbEntry.commentLen = pAlterTbReq->newCommentLen;
       entry.ntbEntry.comment = pAlterTbReq->newComment;
     }
@@ -869,8 +866,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta
 
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
   STbDbKey tbDbKey;
-  void *   pKey = NULL;
-  void *   pVal = NULL;
+  void    *pKey = NULL;
+  void    *pVal = NULL;
   int      kLen = 0;
   int      vLen = 0;
   SEncoder coder = {0};
@@ -930,7 +927,7 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) {
 static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
   STtlIdxKey ttlKey = {0};
   metaBuildTtlIdxKey(&ttlKey, pME);
-  if(ttlKey.dtime == 0) return 0;
+  if (ttlKey.dtime == 0) return 0;
   return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn);
 }
 
@@ -976,19 +973,19 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) {
 }
 
 static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
-  void *         pData = NULL;
+  void          *pData = NULL;
   int            nData = 0;
   STbDbKey       tbDbKey = {0};
   SMetaEntry     stbEntry = {0};
-  STagIdxKey *   pTagIdxKey = NULL;
+  STagIdxKey    *pTagIdxKey = NULL;
   int32_t        nTagIdxKey;
   const SSchema *pTagColumn;       // = &stbEntry.stbEntry.schema.pSchema[0];
-  const void *   pTagData = NULL;  //
+  const void    *pTagData = NULL;  //
   int32_t        nTagData = 0;
   SDecoder       dc = {0};
 
   // get super table
-  if(tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0){
+  if (tdbTbGet(pMeta->pUidIdx, &pCtbEntry->ctbEntry.suid, sizeof(tb_uid_t), &pData, &nData) != 0) {
     return -1;
   }
   tbDbKey.uid = pCtbEntry->ctbEntry.suid;
@@ -1030,7 +1027,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
 
 static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
   SEncoder              coder = {0};
-  void *                pVal = NULL;
+  void                 *pVal = NULL;
   int                   vLen = 0;
   int                   rcode = 0;
   SSkmDbKey             skmDbKey = {0};
@@ -1072,7 +1069,7 @@ _exit:
   return rcode;
 }
 
-static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
+int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
   metaWLock(pMeta);
 
   // save to table.db
@@ -1096,7 +1093,7 @@ static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
 
     if (pME->type == TSDB_SUPER_TABLE) {
       if (metaUpdateSuidIdx(pMeta, pME) < 0) goto _err;
-    }    
+    }
   }
 
   if (pME->type != TSDB_SUPER_TABLE) {
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index b4547a6cc9..76867e2a3e 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -449,6 +449,8 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
       } else if (c < 0) {
         code = tsdbSnapWriteAppendData(pWriter, pData, nData);
         if (code) goto _err;
+
+        break;
       } else {
         // commit the block
         code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL);

From aae8adeb5b2d82d70c5e6fe82a321ad02c032d32 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Wed, 6 Jul 2022 20:57:42 +0800
Subject: [PATCH 31/50] fix:timestamp precision in josn protocol

---
 source/client/src/clientSml.c  |  25 ++--
 source/client/test/smlTest.cpp | 208 ++++++++++++++++++++++++++++++++-
 2 files changed, 220 insertions(+), 13 deletions(-)

diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c
index 5039f8bbaf..b6972e4670 100644
--- a/source/client/src/clientSml.c
+++ b/source/client/src/clientSml.c
@@ -1570,41 +1570,40 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV
     return TSDB_CODE_TSC_INVALID_TIME_STAMP;
   }
 
+  *tsVal = timeDouble;
   size_t typeLen = strlen(type->valuestring);
   if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) {
     // seconds
-    timeDouble = timeDouble * 1e9;
+    *tsVal = *tsVal * NANOSECOND_PER_SEC;
+    timeDouble = timeDouble * NANOSECOND_PER_SEC;
     if (smlDoubleToInt64OverFlow(timeDouble)) {
       smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
       return TSDB_CODE_TSC_INVALID_TIME_STAMP;
     }
-    *tsVal = timeDouble;
   } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) {
     switch (type->valuestring[0]) {
       case 'm':
       case 'M':
         // milliseconds
-        timeDouble = timeDouble * 1e6;
+        *tsVal = *tsVal * NANOSECOND_PER_MSEC;
+        timeDouble = timeDouble * NANOSECOND_PER_MSEC;
         if (smlDoubleToInt64OverFlow(timeDouble)) {
           smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
           return TSDB_CODE_TSC_INVALID_TIME_STAMP;
         }
-        *tsVal = timeDouble;
         break;
       case 'u':
       case 'U':
         // microseconds
-        timeDouble = timeDouble * 1e3;
+        *tsVal = *tsVal * NANOSECOND_PER_USEC;
+        timeDouble = timeDouble * NANOSECOND_PER_USEC;
         if (smlDoubleToInt64OverFlow(timeDouble)) {
           smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
           return TSDB_CODE_TSC_INVALID_TIME_STAMP;
         }
-        *tsVal = timeDouble;
         break;
       case 'n':
       case 'N':
-        // nanoseconds
-        *tsVal = timeDouble;
         break;
       default:
         return TSDB_CODE_TSC_INVALID_JSON;
@@ -1641,21 +1640,23 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) {
     if (timeDouble < 0) {
       return TSDB_CODE_TSC_INVALID_TIME_STAMP;
     }
+
     uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble);
+    tsVal = (int64_t)timeDouble;
     if (tsLen == TSDB_TIME_PRECISION_SEC_DIGITS) {
-      timeDouble = timeDouble * 1e9;
+      tsVal = tsVal * NANOSECOND_PER_SEC;
+      timeDouble = timeDouble * NANOSECOND_PER_SEC;
       if (smlDoubleToInt64OverFlow(timeDouble)) {
         smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
         return TSDB_CODE_TSC_INVALID_TIME_STAMP;
       }
-      tsVal = timeDouble;
     } else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) {
-      timeDouble = timeDouble * 1e6;
+      tsVal = tsVal * NANOSECOND_PER_MSEC;
+      timeDouble = timeDouble * NANOSECOND_PER_MSEC;
       if (smlDoubleToInt64OverFlow(timeDouble)) {
         smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL);
         return TSDB_CODE_TSC_INVALID_TIME_STAMP;
       }
-      tsVal = timeDouble;
     } else if (timeDouble == 0) {
       tsVal = taosGetTimestampNs();
     } else {
diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp
index 832564e0db..a6062efb98 100644
--- a/source/client/test/smlTest.cpp
+++ b/source/client/test/smlTest.cpp
@@ -1284,4 +1284,210 @@ TEST(testCase, sml_dup_time_Test) {
   ASSERT_EQ(taos_errno(pRes), 0);
   taos_free_result(pRes);
 }
-*/
+
+
+TEST(testCase, sml_16960_Test) {
+TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
+ASSERT_NE(taos, nullptr);
+
+TAOS_RES* pRes = taos_query(taos, "create database if not exists d16368 schemaless 1");
+taos_free_result(pRes);
+
+pRes = taos_query(taos, "use d16368");
+taos_free_result(pRes);
+
+const char *sql[] = {
+    "[\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800000, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": 830525384, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800001, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -588348364, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800002, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -370310823, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800003, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -811250191, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "},\n"
+    "{\n"
+    "\"timestamp\":\n"
+    "\n"
+    "{ \"value\": 1349020800004, \"type\": \"ms\" }\n"
+    ",\n"
+    "\"value\":\n"
+    "\n"
+    "{ \"value\": -330340558, \"type\": \"int\" }\n"
+    ",\n"
+    "\"tags\": {\n"
+    "\"id\": \"stb00_0\",\n"
+    "\"t0\":\n"
+    "\n"
+    "{ \"value\": 83972721, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t1\":\n"
+    "\n"
+    "{ \"value\": 539147525, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t2\":\n"
+    "\n"
+    "{ \"value\": 618258572, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t3\":\n"
+    "\n"
+    "{ \"value\": -10536201, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t4\":\n"
+    "\n"
+    "{ \"value\": 349227409, \"type\": \"int\" }\n"
+    ",\n"
+    "\"t5\":\n"
+    "\n"
+    "{ \"value\": 249347042, \"type\": \"int\" }\n"
+    "},\n"
+    "\"metric\": \"stb0\"\n"
+    "}\n"
+    "]"
+};
+
+pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS);
+ASSERT_EQ(taos_errno(pRes), 0);
+taos_free_result(pRes);
+}
+*/
\ No newline at end of file

From 4edcd7011e1983bc8684f4c443025524f5231536 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 7 Jul 2022 05:40:49 +0000
Subject: [PATCH 32/50] more work

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 41 ++++++++++------------
 1 file changed, 19 insertions(+), 22 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 76867e2a3e..b2f4bf10e2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -316,6 +316,7 @@ struct STsdbSnapWriter {
   SDataFReader* pDataFReader;
   SArray*       aBlockIdx;
   int32_t       iBlockIdx;
+  SBlockIdx*    pBlockIdx;
   SMapData      mBlock;
   int32_t       iBlock;
   SBlockData    blockData;
@@ -432,37 +433,33 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
   }
 
   // process
-  TABLEID id = {0};  // TODO
+  TABLEID id = {0};    // TODO
+  TSKEY   minKey = 0;  // TODO
+  TSKEY   maxKey = 0;  // TODO
 
   while (true) {
-    SBlockIdx* pBlockIdx = NULL;
-
-    if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
-      pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
-    }
-
-    if (pBlockIdx) {
-      int32_t c = tTABLEIDCmprFn(&id, pBlockIdx);
+    if (pWriter->pBlockIdx == NULL) {
+      // TODO
+    } else {
+      int32_t c = tTABLEIDCmprFn(&id, pWriter->pBlockIdx);
 
       if (c == 0) {
-        // merge
       } else if (c < 0) {
-        code = tsdbSnapWriteAppendData(pWriter, pData, nData);
-        if (code) goto _err;
-
-        break;
+        // keep merge
       } else {
-        // commit the block
-        code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL);
+        code = tsdbSnapWriteTableDataEnd(pWriter);
         if (code) goto _err;
 
-        SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
-        code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlock, NULL, &blockIdx);
-        if (code) goto _err;
+        pWriter->iBlockIdx++;
+        if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
+          pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
+        } else {
+          pWriter->pBlockIdx = NULL;
+        }
 
-        if (taosArrayPush(pWriter->aBlockIdxN, &blockIdx) == NULL) {
-          code = TSDB_CODE_OUT_OF_MEMORY;
-          goto _err;
+        if (pWriter->pBlockIdx) {
+          code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock, NULL);
+          if (code) goto _err;
         }
       }
     }

From 77f6c41d3159aab5eacec65d6fd8b90d01bc9017 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 7 Jul 2022 09:02:07 +0000
Subject: [PATCH 33/50] more work

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 25 ++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index b2f4bf10e2..54087a7871 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -308,9 +308,11 @@ struct STsdbSnapWriter {
   STsdb*  pTsdb;
   int64_t sver;
   int64_t ever;
+
   // config
   int32_t minutes;
   int8_t  precision;
+
   // for data file
   int32_t       fid;
   SDataFReader* pDataFReader;
@@ -321,10 +323,14 @@ struct STsdbSnapWriter {
   int32_t       iBlock;
   SBlockData    blockData;
   int32_t       iRow;
+
   SDataFWriter* pDataFWriter;
   SArray*       aBlockIdxN;
+  SBlockIdx     blockIdx;
   SMapData      mBlockN;
+  SBlock        block;
   SBlockData    nBlockData;
+
   // for del file
   SDelFReader* pDelFReader;
   SDelFWriter* pDelFWriter;
@@ -408,9 +414,11 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid);
     // reader
     if (pSet) {
+      // open
       code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
       if (code) goto _err;
 
+      // SBlockIdx
       code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx, NULL);
       if (code) goto _err;
     } else {
@@ -438,16 +446,14 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
   TSKEY   maxKey = 0;  // TODO
 
   while (true) {
-    if (pWriter->pBlockIdx == NULL) {
-      // TODO
-    } else {
+    if (pWriter->pBlockIdx) {
       int32_t c = tTABLEIDCmprFn(&id, pWriter->pBlockIdx);
 
       if (c == 0) {
       } else if (c < 0) {
         // keep merge
       } else {
-        code = tsdbSnapWriteTableDataEnd(pWriter);
+        // code = tsdbSnapWriteTableDataEnd(pWriter);
         if (code) goto _err;
 
         pWriter->iBlockIdx++;
@@ -462,6 +468,17 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
           if (code) goto _err;
         }
       }
+    } else {
+      int32_t c = tTABLEIDCmprFn(&id, &pWriter->blockIdx);
+
+      if (c == 0) {
+        // merge commit the block data
+      } else if (c > 0) {
+        // code = tsdbSnapWriteTableDataEnd(pWriter);
+        if (code) goto _err;
+      } else {
+        ASSERT(0);
+      }
     }
   }
 

From 57cc3598586e28cc8b9018341c1d2b8ff709b0cf Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 8 Jul 2022 05:31:34 +0000
Subject: [PATCH 34/50] more work

---
 source/dnode/vnode/inc/vnode.h         | 26 +++++++++++++-------------
 source/dnode/vnode/src/tsdb/tsdbUtil.c |  7 ++++++-
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index 9080358a26..c07884a67e 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -61,8 +61,8 @@ void    vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId);
 int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
 int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list);
 int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list);
-void *  vnodeGetIdx(SVnode *pVnode);
-void *  vnodeGetIvtIdx(SVnode *pVnode);
+void   *vnodeGetIdx(SVnode *pVnode);
+void   *vnodeGetIvtIdx(SVnode *pVnode);
 
 int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
 int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName);
@@ -92,7 +92,7 @@ typedef struct SMetaFltParam {
   tb_uid_t suid;
   int16_t  cid;
   int16_t  type;
-  char *   val;
+  char    *val;
   bool     reverse;
   int (*filterFunc)(void *a, void *b, int16_t type);
 
@@ -133,8 +133,8 @@ SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdLis
 int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond, int32_t tWinIdx);
 int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
 int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle);
-void *  tsdbGetIdx(SMeta *pMeta);
-void *  tsdbGetIvtIdx(SMeta *pMeta);
+void   *tsdbGetIdx(SMeta *pMeta);
+void   *tsdbGetIvtIdx(SMeta *pMeta);
 
 int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t *colId, int32_t numOfCols,
                               void **pReader);
@@ -218,7 +218,7 @@ struct SMetaEntry {
   int8_t   type;
   int8_t   flags;  // TODO: need refactor?
   tb_uid_t uid;
-  char *   name;
+  char    *name;
   union {
     struct {
       SSchemaWrapper schemaRow;
@@ -229,7 +229,7 @@ struct SMetaEntry {
       int64_t  ctime;
       int32_t  ttlDays;
       int32_t  commentLen;
-      char *   comment;
+      char    *comment;
       tb_uid_t suid;
       uint8_t *pTags;
     } ctbEntry;
@@ -237,7 +237,7 @@ struct SMetaEntry {
       int64_t        ctime;
       int32_t        ttlDays;
       int32_t        commentLen;
-      char *         comment;
+      char          *comment;
       int32_t        ncid;  // next column id
       SSchemaWrapper schemaRow;
     } ntbEntry;
@@ -251,17 +251,17 @@ struct SMetaEntry {
 
 struct SMetaReader {
   int32_t    flags;
-  SMeta *    pMeta;
+  SMeta     *pMeta;
   SDecoder   coder;
   SMetaEntry me;
-  void *     pBuf;
+  void      *pBuf;
   int32_t    szBuf;
 };
 
 struct SMTbCursor {
-  TBC *       pDbc;
-  void *      pKey;
-  void *      pVal;
+  TBC        *pDbc;
+  void       *pKey;
+  void       *pVal;
   int32_t     kLen;
   int32_t     vLen;
   SMetaReader mr;
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index f906ef1b54..79095b3672 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -87,8 +87,10 @@ int32_t tPutMapData(uint8_t *p, SMapData *pMapData) {
 
   n += tPutI32v(p ? p + n : p, pMapData->nItem);
   if (pMapData->nItem) {
+    int32_t lOffset = 0;
     for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
-      n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem]);
+      n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem] - lOffset);
+      lOffset = pMapData->aOffset[iItem];
     }
 
     n += tPutI32v(p ? p + n : p, pMapData->nData);
@@ -111,8 +113,11 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData) {
   if (pMapData->nItem) {
     if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) return -1;
 
+    int32_t lOffset = 0;
     for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
       n += tGetI32v(p + n, &pMapData->aOffset[iItem]);
+      pMapData->aOffset[iItem] += lOffset;
+      lOffset = pMapData->aOffset[iItem];
     }
 
     n += tGetI32v(p + n, &pMapData->nData);

From 6b46423a97aaecf4857860cfde4bcb141640e73c Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 8 Jul 2022 18:37:47 +0800
Subject: [PATCH 35/50] feat:write meta from tmq to taosd

---
 examples/c/tmq.c                        |  18 +-
 include/client/taos.h                   |  17 +-
 include/common/tmsg.h                   |  12 +-
 include/common/tname.h                  |   2 +
 include/libs/parser/parser.h            |   3 +
 include/util/tlog.h                     |   2 +-
 source/client/src/tmq.c                 | 527 +++++++++++++++++++++++-
 source/common/src/tmsg.c                |  30 ++
 source/common/src/tname.c               |   8 +
 source/dnode/mnode/impl/src/mndStb.c    |  11 +-
 source/libs/function/src/builtinsimpl.c |   7 +
 source/libs/parser/src/parTranslater.c  |  38 +-
 12 files changed, 630 insertions(+), 45 deletions(-)

diff --git a/examples/c/tmq.c b/examples/c/tmq.c
index 89be847f8e..9aa301b020 100644
--- a/examples/c/tmq.c
+++ b/examples/c/tmq.c
@@ -28,15 +28,23 @@ static void msg_process(TAOS_RES* msg) {
   printf("db: %s\n", tmq_get_db_name(msg));
   printf("vg: %d\n", tmq_get_vgroup_id(msg));
   if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) {
-    void*   meta;
-    int32_t metaLen;
-    tmq_get_raw_meta(msg, &meta, &metaLen);
+    tmq_raw_data *raw = tmq_get_raw_meta(msg);
+    if(raw){
+      TAOS* pConn = taos_connect("localhost", "root", "taosdata", "abc1", 0);
+      if (pConn == NULL) {
+        return;
+      }
+      int32_t ret = taos_write_raw_meta(pConn, raw);
+      printf("write raw data: %s\n", tmq_err2str(ret));
+      free(raw);
+      taos_close(pConn);
+    }
     char* result = tmq_get_json_meta(msg);
     if(result){
       printf("meta result: %s\n", result);
       free(result);
     }
-    printf("meta, len is %d\n", metaLen);
+    printf("meta:%p\n", raw);
     return;
   }
   while (1) {
@@ -60,7 +68,7 @@ int32_t init_env() {
     return -1;
   }
 
-  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 3");
   if (taos_errno(pRes) != 0) {
     printf("error in create db, reason:%s\n", taos_errstr(pRes));
     return -1;
diff --git a/include/client/taos.h b/include/client/taos.h
index 216a5832b0..362782b420 100644
--- a/include/client/taos.h
+++ b/include/client/taos.h
@@ -260,15 +260,16 @@ enum tmq_res_t {
 };
 
 typedef enum tmq_res_t tmq_res_t;
+typedef struct tmq_raw_data tmq_raw_data;
 
-DLL_EXPORT tmq_res_t   tmq_get_res_type(TAOS_RES *res);
-DLL_EXPORT int32_t     tmq_get_raw_meta(TAOS_RES *res, void **raw_meta, int32_t *raw_meta_len);
-DLL_EXPORT int32_t     taos_write_raw_meta(TAOS *res, void *raw_meta, int32_t raw_meta_len);
-DLL_EXPORT char       *tmq_get_json_meta(TAOS_RES *res);   // Returning null means error. Returned result need to be freed.
-DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
-DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
-DLL_EXPORT int32_t     tmq_get_vgroup_id(TAOS_RES *res);
-DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res);
+DLL_EXPORT tmq_res_t     tmq_get_res_type(TAOS_RES *res);
+DLL_EXPORT tmq_raw_data *tmq_get_raw_meta(TAOS_RES *res);
+DLL_EXPORT int32_t       taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta);
+DLL_EXPORT char         *tmq_get_json_meta(TAOS_RES *res);   // Returning null means error. Returned result need to be freed.
+DLL_EXPORT const char   *tmq_get_topic_name(TAOS_RES *res);
+DLL_EXPORT const char   *tmq_get_db_name(TAOS_RES *res);
+DLL_EXPORT int32_t       tmq_get_vgroup_id(TAOS_RES *res);
+DLL_EXPORT const char   *tmq_get_table_name(TAOS_RES *res);
 
 /* ------------------------------ TMQ END -------------------------------- */
 
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index dedc06a2b9..896ea27598 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -446,11 +446,16 @@ typedef struct {
   int32_t ast1Len;
   int32_t ast2Len;
   SArray* pColumns;  // array of SField
+  int32_t cVersion;
   SArray* pTags;     // array of SField
+  int32_t tVersion;
   SArray* pFuncs;
   char*   pComment;
   char*   pAst1;
   char*   pAst2;
+  tb_uid_t  suid;
+  int8_t    source; // 1-taosX or 0-taosClient
+  int8_t    reserved[8];
 } SMCreateStbReq;
 
 int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq);
@@ -458,8 +463,11 @@ int32_t tDeserializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pR
 void    tFreeSMCreateStbReq(SMCreateStbReq* pReq);
 
 typedef struct {
-  char   name[TSDB_TABLE_FNAME_LEN];
-  int8_t igNotExists;
+  char      name[TSDB_TABLE_FNAME_LEN];
+  int8_t    igNotExists;
+  tb_uid_t  suid;
+  int8_t    source; // 1-taosX or 0-taosClient
+  int8_t    reserved[8];
 } SMDropStbReq;
 
 int32_t tSerializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
diff --git a/include/common/tname.h b/include/common/tname.h
index 3bf1cee870..77965947ad 100644
--- a/include/common/tname.h
+++ b/include/common/tname.h
@@ -37,6 +37,8 @@ typedef struct SName {
   char    tname[TSDB_TABLE_NAME_LEN];
 } SName;
 
+SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName);
+
 int32_t tNameExtractFullName(const SName* name, char* dst);
 
 int32_t tNameLen(const SName* name);
diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h
index c3007306ae..a4aec72d4f 100644
--- a/include/libs/parser/parser.h
+++ b/include/libs/parser/parser.h
@@ -96,6 +96,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols
                     char* tableName, char* msgBuf, int16_t msgBufLen);
 int32_t smlBuildOutput(void* handle, SHashObj* pVgHash);
 
+int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
+SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);
+SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap);
 #ifdef __cplusplus
 }
 #endif
diff --git a/include/util/tlog.h b/include/util/tlog.h
index 988d9c6890..8ae0df6f71 100644
--- a/include/util/tlog.h
+++ b/include/util/tlog.h
@@ -94,7 +94,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
 #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
 #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
 // clang-format on
-
+#define BUF_PAGE_DEBUG
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index 1475663a05..a2de9f88da 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -107,6 +107,12 @@ struct tmq_t {
   tsem_t rspSem;
 };
 
+struct tmq_raw_data{
+  void    *raw_meta;
+  int32_t raw_meta_len;
+  int16_t raw_meta_type;
+};
+
 enum {
   TMQ_VG_STATUS__IDLE = 0,
   TMQ_VG_STATUS__WAIT,
@@ -1840,14 +1846,16 @@ const char* tmq_get_table_name(TAOS_RES* res) {
   return NULL;
 }
 
-int32_t tmq_get_raw_meta(TAOS_RES* res, void** raw_meta, int32_t* raw_meta_len) {
+tmq_raw_data *tmq_get_raw_meta(TAOS_RES* res) {
   if (TD_RES_TMQ_META(res)) {
+    tmq_raw_data *raw = taosMemoryCalloc(1, sizeof(tmq_raw_data));
     SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
-    *raw_meta = pMetaRspObj->metaRsp.metaRsp;
-    *raw_meta_len = pMetaRspObj->metaRsp.metaRspLen;
-    return 0;
+    raw->raw_meta = pMetaRspObj->metaRsp.metaRsp;
+    raw->raw_meta_len = pMetaRspObj->metaRsp.metaRspLen;
+    raw->raw_meta_type = pMetaRspObj->metaRsp.resMsgType;
+    return raw;
   }
-  return -1;
+  return NULL;
 }
 
 static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t){
@@ -2250,6 +2258,515 @@ char *tmq_get_json_meta(TAOS_RES *res){
   return NULL;
 }
 
+static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){
+  SVCreateStbReq req = {0};
+  SDecoder       coder;
+  SMCreateStbReq pReq = {0};
+  int32_t code = TSDB_CODE_SUCCESS;
+  SRequestObj* pRequest = NULL;
+
+  STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
+  if (NULL == pTscObj) {
+    code = TSDB_CODE_TSC_DISCONNECTED;
+    goto end;
+  }
+
+  code = buildRequest(pTscObj, "", 0, &pRequest);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  if(!pRequest->pDb){
+    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
+    goto end;
+  }
+  // decode and process req
+  void* data = POINTER_SHIFT(meta, sizeof(SMsgHead));
+  int32_t len = metaLen - sizeof(SMsgHead);
+  tDecoderInit(&coder, data, len);
+  if (tDecodeSVCreateStbReq(&coder, &req) < 0) {
+    code = TSDB_CODE_INVALID_PARA;
+    goto end;
+  }
+  // build create stable
+  pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SField));
+  for(int32_t i = 0; i < req.schemaRow.nCols; i++){
+    SSchema* pSchema = req.schemaRow.pSchema + i;
+    SField   field   = {.type = pSchema->type, .bytes = pSchema->bytes};
+    strcpy(field.name, pSchema->name);
+    taosArrayPush(pReq.pColumns, &field);
+  }
+  pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField));
+  for(int32_t i = 0; i < req.schemaTag.nCols; i++){
+    SSchema* pSchema = req.schemaTag.pSchema + i;
+    SField   field   = {.type = pSchema->type, .bytes = pSchema->bytes};
+    strcpy(field.name, pSchema->name);
+    taosArrayPush(pReq.pTags, &field);
+  }
+  pReq.cVersion = req.schemaRow.version;
+  pReq.tVersion = req.schemaTag.version;
+  pReq.numOfColumns = req.schemaRow.nCols;
+  pReq.numOfTags = req.schemaTag.nCols;
+  pReq.commentLen = -1;
+  pReq.suid = req.suid;
+  pReq.source = 1;
+
+  SName tableName;
+  tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, "rname", &tableName), pReq.name);
+
+  SCmdMsgInfo pCmdMsg = {0};
+  pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
+  pCmdMsg.msgType = TDMT_MND_CREATE_STB;
+  pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq);
+  pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen);
+  if (NULL == pCmdMsg.pMsg) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq);
+
+  SQuery      pQuery = {0};
+  pQuery.execMode = QUERY_EXEC_MODE_RPC;
+  pQuery.pCmdMsg = &pCmdMsg;
+  pQuery.msgType = pQuery.pCmdMsg->msgType;
+  pQuery.stableQuery = true;
+
+  launchQueryImpl(pRequest, &pQuery, true, NULL);
+  code = pRequest->code;
+  taosMemoryFree(pCmdMsg.pMsg);
+
+  end:
+  destroyRequest(pRequest);
+  tFreeSMCreateStbReq(&pReq);
+  tDecoderClear(&coder);
+  return code;
+}
+
+static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){
+  SVDropStbReq req = {0};
+  SDecoder     coder;
+  SMDropStbReq pReq = {0};
+  int32_t code = TSDB_CODE_SUCCESS;
+  SRequestObj* pRequest = NULL;
+
+  STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
+  if (NULL == pTscObj) {
+    code = TSDB_CODE_TSC_DISCONNECTED;
+    goto end;
+  }
+
+  code = buildRequest(pTscObj, "", 0, &pRequest);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  if(!pRequest->pDb){
+    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
+    goto end;
+  }
+  // decode and process req
+  void* data = POINTER_SHIFT(meta, sizeof(SMsgHead));
+  int32_t len = metaLen - sizeof(SMsgHead);
+  tDecoderInit(&coder, data, len);
+  if (tDecodeSVDropStbReq(&coder, &req) < 0) {
+    code = TSDB_CODE_INVALID_PARA;
+    goto end;
+  }
+
+  // build drop stable
+  pReq.igNotExists = true;
+  pReq.source = 1;
+  pReq.suid = req.suid;
+  SName tableName;
+  tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, "rname", &tableName), pReq.name);
+
+  SCmdMsgInfo pCmdMsg = {0};
+  pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
+  pCmdMsg.msgType = TDMT_MND_DROP_STB;
+  pCmdMsg.msgLen = tSerializeSMDropStbReq(NULL, 0, &pReq);
+  pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen);
+  if (NULL == pCmdMsg.pMsg) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq);
+
+  SQuery      pQuery = {0};
+  pQuery.execMode = QUERY_EXEC_MODE_RPC;
+  pQuery.pCmdMsg = &pCmdMsg;
+  pQuery.msgType = pQuery.pCmdMsg->msgType;
+  pQuery.stableQuery = true;
+
+  launchQueryImpl(pRequest, &pQuery, true, NULL);
+  code = pRequest->code;
+  taosMemoryFree(pCmdMsg.pMsg);
+
+  end:
+  destroyRequest(pRequest);
+  tDecoderClear(&coder);
+  return code;
+}
+
+typedef struct SVgroupCreateTableBatch {
+  SVCreateTbBatchReq req;
+  SVgroupInfo        info;
+  char               dbName[TSDB_DB_NAME_LEN];
+} SVgroupCreateTableBatch;
+
+static void destroyCreateTbReqBatch(void* data) {
+  SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*) data;
+  taosArrayDestroy(pTbBatch->req.pArray);
+}
+
+static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){
+  SVCreateTbBatchReq  req             = {0};
+  SDecoder            coder           = {0};
+  int32_t             code            = TSDB_CODE_SUCCESS;
+  SRequestObj        *pRequest        = NULL;
+  SQuery             *pQuery          = NULL;
+  SHashObj           *pVgroupHashmap  = NULL;
+  STscObj            *pTscObj         = acquireTscObj(*(int64_t *)taos);
+
+  if (NULL == pTscObj) {
+    code = TSDB_CODE_TSC_DISCONNECTED;
+    goto end;
+  }
+
+  code = buildRequest(pTscObj, "", 0, &pRequest);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  if(!pRequest->pDb){
+    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
+    goto end;
+  }
+  // decode and process req
+  void* data = POINTER_SHIFT(meta, sizeof(SMsgHead));
+  int32_t len = metaLen - sizeof(SMsgHead);
+  tDecoderInit(&coder, data, len);
+  if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) {
+    code = TSDB_CODE_INVALID_PARA;
+    goto end;
+  }
+
+  SVCreateTbReq     *pCreateReq = NULL;
+  SCatalog* pCatalog = NULL;
+  code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+  if (NULL == pVgroupHashmap) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch);
+
+  SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
+      .requestId = pRequest->requestId,
+      .requestObjRefId = pRequest->self,
+      .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
+  // loop to create table
+  for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
+    pCreateReq = req.pReqs + iReq;
+
+    SVgroupInfo pInfo = {0};
+    SName pName;
+    toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName);
+    code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto end;
+    }
+
+    SVgroupCreateTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId));
+    if (pTableBatch == NULL) {
+      SVgroupCreateTableBatch tBatch = {0};
+      tBatch.info = pInfo;
+      strcpy(tBatch.dbName, pRequest->pDb);
+
+      tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq));
+      taosArrayPush(tBatch.req.pArray, pCreateReq);
+
+      taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch));
+    } else {  // add to the correct vgroup
+      taosArrayPush(pTableBatch->req.pArray, pCreateReq);
+    }
+  }
+
+  SArray* pBufArray = serializeVgroupsCreateTableBatch(pVgroupHashmap);
+  if (NULL == pBufArray) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+
+  pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
+  pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
+  pQuery->msgType = TDMT_VND_CREATE_TABLE;
+  pQuery->stableQuery = false;
+  pQuery->pRoot   = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT);
+
+  code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  launchQueryImpl(pRequest, pQuery, false, NULL);
+  pQuery  = NULL;          // no need to free in the end
+  code    = pRequest->code;
+
+  end:
+  taosHashCleanup(pVgroupHashmap);
+  destroyRequest(pRequest);
+  tDecoderClear(&coder);
+  qDestroyQuery(pQuery);
+  return code;
+}
+
+typedef struct SVgroupDropTableBatch {
+  SVDropTbBatchReq req;
+  SVgroupInfo      info;
+  char             dbName[TSDB_DB_NAME_LEN];
+} SVgroupDropTableBatch;
+
+static void destroyDropTbReqBatch(void* data) {
+  SVgroupDropTableBatch* pTbBatch = (SVgroupDropTableBatch*)data;
+  taosArrayDestroy(pTbBatch->req.pArray);
+}
+
+static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){
+  SVDropTbBatchReq    req             = {0};
+  SDecoder            coder           = {0};
+  int32_t             code            = TSDB_CODE_SUCCESS;
+  SRequestObj        *pRequest        = NULL;
+  SQuery             *pQuery          = NULL;
+  SHashObj           *pVgroupHashmap  = NULL;
+  STscObj            *pTscObj         = acquireTscObj(*(int64_t *)taos);
+
+  if (NULL == pTscObj) {
+    code = TSDB_CODE_TSC_DISCONNECTED;
+    goto end;
+  }
+
+  code = buildRequest(pTscObj, "", 0, &pRequest);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  if(!pRequest->pDb){
+    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
+    goto end;
+  }
+  // decode and process req
+  void* data = POINTER_SHIFT(meta, sizeof(SMsgHead));
+  int32_t len = metaLen - sizeof(SMsgHead);
+  tDecoderInit(&coder, data, len);
+  if (tDecodeSVDropTbBatchReq(&coder, &req) < 0) {
+    code = TSDB_CODE_INVALID_PARA;
+    goto end;
+  }
+
+  SVDropTbReq     *pDropReq = NULL;
+  SCatalog        *pCatalog = NULL;
+  code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
+  if (NULL == pVgroupHashmap) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch);
+
+  SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
+      .requestId = pRequest->requestId,
+      .requestObjRefId = pRequest->self,
+      .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
+  // loop to create table
+  for (int32_t iReq = 0; iReq < req.nReqs; iReq++) {
+    pDropReq = req.pReqs + iReq;
+
+    SVgroupInfo pInfo = {0};
+    SName pName;
+    toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName);
+    code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto end;
+    }
+
+    SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId));
+    if (pTableBatch == NULL) {
+      SVgroupDropTableBatch tBatch = {0};
+      tBatch.info = pInfo;
+      tBatch.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq));
+      taosArrayPush(tBatch.req.pArray, pDropReq);
+
+      taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch));
+    } else {  // add to the correct vgroup
+      taosArrayPush(pTableBatch->req.pArray, pDropReq);
+    }
+  }
+
+  SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap);
+  if (NULL == pBufArray) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+
+  pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
+  pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
+  pQuery->msgType = TDMT_VND_DROP_TABLE;
+  pQuery->stableQuery = false;
+  pQuery->pRoot   = nodesMakeNode(QUERY_NODE_DROP_TABLE_STMT);
+
+  code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  launchQueryImpl(pRequest, pQuery, false, NULL);
+  pQuery  = NULL;          // no need to free in the end
+  code    = pRequest->code;
+
+  end:
+  taosHashCleanup(pVgroupHashmap);
+  destroyRequest(pRequest);
+  tDecoderClear(&coder);
+  qDestroyQuery(pQuery);
+  return code;
+}
+
+static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){
+  SVAlterTbReq        req             = {0};
+  SDecoder            coder           = {0};
+  int32_t             code            = TSDB_CODE_SUCCESS;
+  SRequestObj        *pRequest        = NULL;
+  SQuery             *pQuery          = NULL;
+  SArray             *pArray          = NULL;
+  SVgDataBlocks      *pVgData         = NULL;
+  STscObj            *pTscObj         = acquireTscObj(*(int64_t *)taos);
+
+  if (NULL == pTscObj) {
+    code = TSDB_CODE_TSC_DISCONNECTED;
+    goto end;
+  }
+
+  code = buildRequest(pTscObj, "", 0, &pRequest);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  if(!pRequest->pDb){
+    code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
+    goto end;
+  }
+  // decode and process req
+  void* data = POINTER_SHIFT(meta, sizeof(SMsgHead));
+  int32_t len = metaLen - sizeof(SMsgHead);
+  tDecoderInit(&coder, data, len);
+  if (tDecodeSVAlterTbReq(&coder, &req) < 0) {
+    code = TSDB_CODE_INVALID_PARA;
+    goto end;
+  }
+
+  // do not deal TSDB_ALTER_TABLE_UPDATE_OPTIONS
+  if(req.action == TSDB_ALTER_TABLE_UPDATE_OPTIONS){
+    goto end;
+  }
+
+  SCatalog        *pCatalog = NULL;
+  code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter,
+      .requestId = pRequest->requestId,
+      .requestObjRefId = pRequest->self,
+      .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)};
+
+  SVgroupInfo pInfo = {0};
+  SName pName = {0};
+  toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName);
+  code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  pArray = taosArrayInit(1, sizeof(void*));
+  if (NULL == pArray) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+
+  pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks));
+  if (NULL == pVgData) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  pVgData->vg = pInfo;
+  pVgData->pData = taosMemoryMalloc(metaLen);
+  if (NULL == pVgData->pData) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto end;
+  }
+  memcpy(pVgData->pData, meta, metaLen);
+  ((SMsgHead*)pVgData->pData)->vgId = htonl(pInfo.vgId);
+  pVgData->size = metaLen;
+  pVgData->numOfTables = 1;
+  taosArrayPush(pArray, &pVgData);
+
+  pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
+  pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
+  pQuery->msgType = TDMT_VND_ALTER_TABLE;
+  pQuery->stableQuery = false;
+  pQuery->pRoot   = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT);
+
+  code = rewriteToVnodeModifyOpStmt(pQuery, pArray);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  launchQueryImpl(pRequest, pQuery, false, NULL);
+  pQuery  = NULL;          // no need to free in the end
+  pVgData = NULL;
+  pArray  = NULL;
+  code    = pRequest->code;
+
+end:
+  taosArrayDestroy(pArray);
+  if(pVgData) taosMemoryFreeClear(pVgData->pData);
+  taosMemoryFreeClear(pVgData);
+  destroyRequest(pRequest);
+  tDecoderClear(&coder);
+  qDestroyQuery(pQuery);
+  return code;
+}
+
+int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta){
+  if (!taos || !raw_meta) {
+    return TSDB_CODE_INVALID_PARA;
+  }
+
+  if(raw_meta->raw_meta_type == TDMT_VND_CREATE_STB) {
+    return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+  }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_STB){
+    return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+  }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_STB){
+    return taosDropStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+  }else if(raw_meta->raw_meta_type == TDMT_VND_CREATE_TABLE){
+    return taosCreateTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+  }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_TABLE){
+    return taosAlterTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+  }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_TABLE){
+    return taosDropTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len);
+  }
+  return TSDB_CODE_INVALID_PARA;
+}
+
 void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) {
   tmqCommitInner2(tmq, msg, 0, 1, cb, param);
 }
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 8157ba4d92..165e8631c4 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -516,6 +516,8 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
     if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
   }
 
+  if (tEncodeI32(&encoder, pReq->cVersion) < 0) return -1;
+
   for (int32_t i = 0; i < pReq->numOfTags; ++i) {
     SField *pField = taosArrayGet(pReq->pTags, i);
     if (tEncodeI8(&encoder, pField->type) < 0) return -1;
@@ -524,6 +526,8 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
     if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
   }
 
+  if (tEncodeI32(&encoder, pReq->tVersion) < 0) return -1;
+
   for (int32_t i = 0; i < pReq->numOfFuncs; ++i) {
     const char *pFunc = taosArrayGet(pReq->pFuncs, i);
     if (tEncodeCStr(&encoder, pFunc) < 0) return -1;
@@ -538,6 +542,11 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
   if (pReq->ast2Len > 0) {
     if (tEncodeBinary(&encoder, pReq->pAst2, pReq->ast2Len) < 0) return -1;
   }
+  if (tEncodeI64(&encoder, pReq->suid) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) {
+    if (tEncodeI8(&encoder, pReq->reserved[i]) < 0) return -1;
+  }
 
   tEndEncode(&encoder);
 
@@ -585,6 +594,8 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     }
   }
 
+  if (tDecodeI32(&decoder, &pReq->cVersion) < 0) return -1;
+
   for (int32_t i = 0; i < pReq->numOfTags; ++i) {
     SField field = {0};
     if (tDecodeI8(&decoder, &field.type) < 0) return -1;
@@ -597,6 +608,8 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     }
   }
 
+  if (tDecodeI32(&decoder, &pReq->tVersion) < 0) return -1;
+
   for (int32_t i = 0; i < pReq->numOfFuncs; ++i) {
     char pFunc[TSDB_FUNC_NAME_LEN] = {0};
     if (tDecodeCStrTo(&decoder, pFunc) < 0) return -1;
@@ -624,6 +637,12 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     if (tDecodeCStrTo(&decoder, pReq->pAst2) < 0) return -1;
   }
 
+  if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) {
+    if (tDecodeI8(&decoder, &pReq->reserved[i]) < 0) return -1;
+  }
+
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
   return 0;
@@ -645,6 +664,11 @@ int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) {
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->suid) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) {
+    if (tEncodeI8(&encoder, pReq->reserved[i]) < 0) return -1;
+  }
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -659,6 +683,12 @@ int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq)
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) {
+    if (tDecodeI8(&decoder, &pReq->reserved[i]) < 0) return -1;
+  }
+
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
diff --git a/source/common/src/tname.c b/source/common/src/tname.c
index fb3a2d7464..7183153824 100644
--- a/source/common/src/tname.c
+++ b/source/common/src/tname.c
@@ -115,6 +115,14 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in
 
 #endif
 
+SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) {
+  pName->type = TSDB_TABLE_NAME_T;
+  pName->acctId = acctId;
+  strcpy(pName->dbname, pDbName);
+  strcpy(pName->tname, pTableName);
+  return pName;
+}
+
 int32_t tNameExtractFullName(const SName* name, char* dst) {
   assert(name != NULL && dst != NULL);
 
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index c7a89945f8..6d1cc05ace 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -706,10 +706,10 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
   memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN);
   pDst->createdTime = taosGetTimestampMs();
   pDst->updateTime = pDst->createdTime;
-  pDst->uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
+  pDst->uid = (pCreate->source == 1) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
   pDst->dbUid = pDb->uid;
-  pDst->tagVer = 1;
-  pDst->colVer = 1;
+  pDst->tagVer = (pCreate->source == 1) ? pCreate->tVersion : 1;
+  pDst->colVer = (pCreate->source == 1) ? pCreate->cVersion : 1;
   pDst->smaVer = 1;
   pDst->nextColId = 1;
   pDst->maxdelay[0] = pCreate->delay1;
@@ -1752,6 +1752,11 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
     }
   }
 
+  if (dropReq.source == 1 && pStb->uid != dropReq.suid){
+    terrno = TSDB_CODE_MND_STB_NOT_EXIST;
+    goto _OVER;
+  }
+
   pDb = mndAcquireDbByStb(pMnode, dropReq.name);
   if (pDb == NULL) {
     terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 7c40f373a7..adec14701a 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -3451,6 +3451,10 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
 
   setBufPageDirty(pPage, true);
   releaseBufPage(pCtx->pBuf, pPage);
+#ifdef BUF_PAGE_DEBUG
+  qDebug("page_saveTuple pos:%p,pageId:%d, offset:%d\n", pPos, pPos->pageId,
+           pPos->offset);
+#endif
 }
 
 void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) {
@@ -3485,6 +3489,9 @@ void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
 
   setBufPageDirty(pPage, true);
   releaseBufPage(pCtx->pBuf, pPage);
+#ifdef BUF_PAGE_DEBUG
+  qDebug("page_copyTuple pos:%p, pageId:%d, offset:%d", pPos, pPos->pageId, pPos->offset);
+#endif
 }
 
 int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 1f4294279b..ed6d3c1c93 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -77,14 +77,6 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) {
   return TSDB_CODE_SUCCESS;
 }
 
-static SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) {
-  pName->type = TSDB_TABLE_NAME_T;
-  pName->acctId = acctId;
-  strcpy(pName->dbname, pDbName);
-  strcpy(pName->tname, pTableName);
-  return pName;
-}
-
 static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) {
   SFullDatabaseName name = {0};
   strcpy(name.fullDbName, pFullDbName);
@@ -5294,7 +5286,8 @@ static int32_t serializeVgroupCreateTableBatch(SVgroupCreateTableBatch* pTbBatch
   return TSDB_CODE_SUCCESS;
 }
 
-static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) {
+static void destroyCreateTbReqBatch(void* data) {
+  SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*) data;
   size_t size = taosArrayGetSize(pTbBatch->req.pArray);
   for (int32_t i = 0; i < size; ++i) {
     SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i);
@@ -5311,7 +5304,7 @@ static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) {
   taosArrayDestroy(pTbBatch->req.pArray);
 }
 
-static int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray) {
+int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray) {
   SVnodeModifOpStmt* pNewStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT);
   if (pNewStmt == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -5377,10 +5370,10 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) {
 
 static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt,
                                      const STag* pTag, uint64_t suid, SVgroupInfo* pVgInfo) {
-  char  dbFName[TSDB_DB_FNAME_LEN] = {0};
-  SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId};
-  strcpy(name.dbname, pStmt->dbName);
-  tNameGetFullDbName(&name, dbFName);
+//  char  dbFName[TSDB_DB_FNAME_LEN] = {0};
+//  SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId};
+//  strcpy(name.dbname, pStmt->dbName);
+//  tNameGetFullDbName(&name, dbFName);
 
   struct SVCreateTbReq req = {0};
   req.type = TD_CHILD_TABLE;
@@ -5641,7 +5634,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla
   return code;
 }
 
-static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgroupHashmap) {
+SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap) {
   SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*));
   if (NULL == pBufArray) {
     return NULL;
@@ -5656,7 +5649,6 @@ static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgrou
     }
 
     serializeVgroupCreateTableBatch(pTbBatch, pBufArray);
-    destroyCreateTbReqBatch(pTbBatch);
   } while (true);
 
   return pBufArray;
@@ -5670,6 +5662,7 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery)
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch);
   int32_t code = TSDB_CODE_SUCCESS;
   SNode*  pNode;
   FOREACH(pNode, pStmt->pSubTables) {
@@ -5681,7 +5674,7 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery)
     }
   }
 
-  SArray* pBufArray = serializeVgroupsCreateTableBatch(pCxt->pParseCxt->acctId, pVgroupHashmap);
+  SArray* pBufArray = serializeVgroupsCreateTableBatch(pVgroupHashmap);
   taosHashCleanup(pVgroupHashmap);
   if (NULL == pBufArray) {
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -5741,7 +5734,10 @@ over:
   return code;
 }
 
-static void destroyDropTbReqBatch(SVgroupDropTableBatch* pTbBatch) { taosArrayDestroy(pTbBatch->req.pArray); }
+static void destroyDropTbReqBatch(void* data) {
+  SVgroupDropTableBatch* pTbBatch = (SVgroupDropTableBatch*)data;
+  taosArrayDestroy(pTbBatch->req.pArray);
+}
 
 static int32_t serializeVgroupDropTableBatch(SVgroupDropTableBatch* pTbBatch, SArray* pBufArray) {
   int      tlen;
@@ -5775,7 +5771,7 @@ static int32_t serializeVgroupDropTableBatch(SVgroupDropTableBatch* pTbBatch, SA
   return TSDB_CODE_SUCCESS;
 }
 
-static SArray* serializeVgroupsDropTableBatch(int32_t acctId, SHashObj* pVgroupHashmap) {
+SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap) {
   SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*));
   if (NULL == pBufArray) {
     return NULL;
@@ -5790,7 +5786,6 @@ static SArray* serializeVgroupsDropTableBatch(int32_t acctId, SHashObj* pVgroupH
     }
 
     serializeVgroupDropTableBatch(pTbBatch, pBufArray);
-    destroyDropTbReqBatch(pTbBatch);
   } while (true);
 
   return pBufArray;
@@ -5804,6 +5799,7 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch);
   bool   isSuperTable = false;
   SNode* pNode;
   FOREACH(pNode, pStmt->pTables) {
@@ -5822,7 +5818,7 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) {
     return TSDB_CODE_SUCCESS;
   }
 
-  SArray* pBufArray = serializeVgroupsDropTableBatch(pCxt->pParseCxt->acctId, pVgroupHashmap);
+  SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap);
   taosHashCleanup(pVgroupHashmap);
   if (NULL == pBufArray) {
     return TSDB_CODE_OUT_OF_MEMORY;

From cefd65f69ebda7c21e8776bc0d3187a02b27ce8d Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 8 Jul 2022 18:51:51 +0800
Subject: [PATCH 36/50] feat:write meta from tmq to taosd

---
 examples/c/tmq.c        | 4 ++--
 source/client/src/tmq.c | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/examples/c/tmq.c b/examples/c/tmq.c
index 9aa301b020..07af30ab72 100644
--- a/examples/c/tmq.c
+++ b/examples/c/tmq.c
@@ -30,7 +30,7 @@ static void msg_process(TAOS_RES* msg) {
   if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) {
     tmq_raw_data *raw = tmq_get_raw_meta(msg);
     if(raw){
-      TAOS* pConn = taos_connect("localhost", "root", "taosdata", "abc1", 0);
+      TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", "abc1", 0);
       if (pConn == NULL) {
         return;
       }
@@ -68,7 +68,7 @@ int32_t init_env() {
     return -1;
   }
 
-  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 3");
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
   if (taos_errno(pRes) != 0) {
     printf("error in create db, reason:%s\n", taos_errstr(pRes));
     return -1;
diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index a2de9f88da..6fcb9c1573 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -2312,7 +2312,7 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){
   pReq.source = 1;
 
   SName tableName;
-  tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, "rname", &tableName), pReq.name);
+  tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
 
   SCmdMsgInfo pCmdMsg = {0};
   pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);

From 054b16c2db5754eaf24d9c8d78f6ab35c65a49e3 Mon Sep 17 00:00:00 2001
From: wangmm0220 <wangmm0220@gmail.com>
Date: Fri, 8 Jul 2022 18:56:36 +0800
Subject: [PATCH 37/50] feat:write meta from tmq to taosd

---
 source/client/src/tmq.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index 6fcb9c1573..907691698c 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -2378,7 +2378,7 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){
   pReq.source = 1;
   pReq.suid = req.suid;
   SName tableName;
-  tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, "rname", &tableName), pReq.name);
+  tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name);
 
   SCmdMsgInfo pCmdMsg = {0};
   pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);

From 017452f4ca276634e81a647bb2550bcf64bbbd7b Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Fri, 8 Jul 2022 19:01:22 +0800
Subject: [PATCH 38/50] enh: query redirect for merge task

---
 source/dnode/qnode/src/qnode.c        | 5 +++++
 source/libs/qworker/src/qwDbg.c       | 2 +-
 source/libs/scheduler/inc/schInt.h    | 8 ++++----
 source/libs/scheduler/src/schRemote.c | 2 +-
 4 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c
index b65189153e..03d14d7ab4 100644
--- a/source/dnode/qnode/src/qnode.c
+++ b/source/dnode/qnode/src/qnode.c
@@ -64,11 +64,16 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) {
   return 0; 
 }
 
+int32_t qnodeCrash = 1;
 int32_t qndPreprocessQueryMsg(SQnode *pQnode, SRpcMsg * pMsg) {
   if (TDMT_SCH_QUERY != pMsg->msgType && TDMT_SCH_MERGE_QUERY != pMsg->msgType) {
     return 0;
   }
 
+  if (qnodeCrash) {
+    ASSERT(0);
+  }
+
   return qWorkerPreprocessQueryMsg(pQnode->pQuery, pMsg);
 }
 
diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c
index dfe5a04d19..869eedf8f6 100644
--- a/source/libs/qworker/src/qwDbg.c
+++ b/source/libs/qworker/src/qwDbg.c
@@ -9,7 +9,7 @@
 #include "tmsg.h"
 #include "tname.h"
 
-SQWDebug     gQWDebug = {.statusEnable = true, .dumpEnable = false, .tmp = false};
+SQWDebug     gQWDebug = {.statusEnable = true, .dumpEnable = false, .tmp = true};
 
 int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) {
   if (!gQWDebug.statusEnable) {
diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h
index 9018deaf13..591301ef44 100644
--- a/source/libs/scheduler/inc/schInt.h
+++ b/source/libs/scheduler/inc/schInt.h
@@ -328,10 +328,10 @@ extern SSchedulerMgmt schMgmt;
 #define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job))
 #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode)
 #define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
-#define SCH_SUB_TASK_NETWORK_ERR(_code, _len) (SCH_NETWORK_ERR(_code) && ((_len) > 0))
-#define SCH_NEED_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH)
-#define SCH_NEED_REDIRECT(_msgType, _code, _rspLen) (SCH_NEED_REDIRECT_MSGTYPE(_msgType) && (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_SUB_TASK_NETWORK_ERR(_code, _rspLen)))
-#define SCH_NEED_RETRY(_msgType, _code) ((SCH_NETWORK_ERR(_code) && SCH_NEED_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
+#define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) (SCH_NETWORK_ERR(_code) && ((_len) > 0 || SCH_IS_DATA_BIND_TASK(_task)))
+#define SCH_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH)
+#define SCH_TASK_NEED_REDIRECT(_task, _msgType, _code, _rspLen) (SCH_REDIRECT_MSGTYPE(_msgType) && (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_MERGE_TASK_NETWORK_ERR((_task), (_code), (_rspLen))))
+#define SCH_NEED_RETRY(_msgType, _code) ((SCH_NETWORK_ERR(_code) && SCH_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
 
 #define SCH_IS_LEVEL_UNFINISHED(_level) ((_level)->taskLaunchedNum < (_level)->taskNum)
 #define SCH_GET_CUR_EP(_addr) (&(_addr)->epSet.eps[(_addr)->epSet.inUse])
diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c
index f9c3d80e46..2257ba8328 100644
--- a/source/libs/scheduler/src/schRemote.c
+++ b/source/libs/scheduler/src/schRemote.c
@@ -85,7 +85,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa
   SCH_ERR_JRET(schValidateRspMsgType(pJob, pTask, msgType));
 
   int32_t reqType = IsReq(pMsg) ? pMsg->msgType : (pMsg->msgType - 1);
-  if (SCH_NEED_REDIRECT(reqType, rspCode, pMsg->len)) {
+  if (SCH_TASK_NEED_REDIRECT(pTask, reqType, rspCode, pMsg->len)) {
     SCH_RET(schHandleRedirect(pJob, pTask, (SDataBuf *)pMsg, rspCode));
   }
 

From 189f81e94f0fd3ffbd9b4ea0e5ff7b3fd4a38abb Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Sun, 10 Jul 2022 10:28:57 +0800
Subject: [PATCH 39/50] test: add tmq_sim into pakage

---
 packaging/tools/install.sh | 2 ++
 packaging/tools/makepkg.sh | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh
index 2d17ef4812..230172cdc6 100755
--- a/packaging/tools/install.sh
+++ b/packaging/tools/install.sh
@@ -30,6 +30,7 @@ configDir="/etc/taos"
 installDir="/usr/local/taos"
 adapterName="taosadapter"
 benchmarkName="taosBenchmark"
+tmqName="tmq_sim"
 dumpName="taosdump"
 demoName="taosdemo"
 
@@ -205,6 +206,7 @@ function install_bin() {
   [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || :
   [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || :
   [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || :
+  [ -x ${install_main_dir}/bin/${tmqName} ] && ${csudo}ln -s ${install_main_dir}/bin/${tmqName} ${bin_link_dir}/${tmqName} || :
   [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || :
   [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
   [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh
index cd656fe6f2..e098acdb40 100755
--- a/packaging/tools/makepkg.sh
+++ b/packaging/tools/makepkg.sh
@@ -60,7 +60,7 @@ if [ "$pagMode" == "lite" ]; then
   strip ${build_dir}/bin/${serverName}
   strip ${build_dir}/bin/${clientName}
   # lite version doesn't include taosadapter,  which will lead to no restful interface
-  bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark"
+  bin_files="${build_dir}/bin/${serverName} ${build_dir}/bin/${clientName} ${script_dir}/remove.sh ${script_dir}/startPre.sh ${build_dir}/bin/taosBenchmark  ${build_dir}/bin/tmq_sim"
   taostools_bin_files=""
 else
 
@@ -78,6 +78,7 @@ else
 
   taostools_bin_files=" ${build_dir}/bin/taosdump \
       ${build_dir}/bin/taosBenchmark \
+      ${build_dir}/bin/tmq_sim \
       ${build_dir}/bin/TDinsight.sh \
       $tdinsight_caches"
 

From e1db3eee3613cb4b52c4c0ca8a97fdd59f9868a7 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Sun, 10 Jul 2022 10:55:46 +0800
Subject: [PATCH 40/50] refactor(sync): merge tsdb_snapshot code

---
 source/dnode/vnode/src/meta/metaTable.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c
index 72c39634f5..daf7ccb26a 100644
--- a/source/dnode/vnode/src/meta/metaTable.c
+++ b/source/dnode/vnode/src/meta/metaTable.c
@@ -17,7 +17,6 @@
 
 static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
 static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
-static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME);
 static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME);

From f2f45a1ec71f1a19a6b2122ae65ec84fe116997e Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 10 Jul 2022 10:43:01 +0800
Subject: [PATCH 41/50] feat:write meta from tmq to taosd

---
 include/common/tmsg.h                | 61 +++++++++++++++-------------
 include/util/taoserror.h             |  3 +-
 source/client/src/tmq.c              |  4 +-
 source/common/src/tmsg.c             | 41 ++++++++-----------
 source/dnode/mnode/impl/src/mndStb.c | 48 ++++++++++++++++++----
 source/util/src/terror.c             |  1 +
 6 files changed, 94 insertions(+), 64 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 896ea27598..f21d0fa4f6 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -169,6 +169,9 @@ typedef enum _mgmt_table {
 #define TD_CHILD_TABLE  TSDB_CHILD_TABLE
 #define TD_NORMAL_TABLE TSDB_NORMAL_TABLE
 
+#define TD_REQ_FROM_APP  0
+#define TD_REQ_FROM_TAOX 1
+
 typedef struct {
   int32_t vgId;
   char*   dbFName;
@@ -432,30 +435,30 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW
 STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols);
 
 typedef struct {
-  char    name[TSDB_TABLE_FNAME_LEN];
-  int8_t  igExists;
-  int64_t delay1;
-  int64_t delay2;
-  int64_t watermark1;
-  int64_t watermark2;
-  int32_t ttl;
-  int32_t numOfColumns;
-  int32_t numOfTags;
-  int32_t numOfFuncs;
-  int32_t commentLen;
-  int32_t ast1Len;
-  int32_t ast2Len;
-  SArray* pColumns;  // array of SField
-  int32_t cVersion;
-  SArray* pTags;     // array of SField
-  int32_t tVersion;
-  SArray* pFuncs;
-  char*   pComment;
-  char*   pAst1;
-  char*   pAst2;
-  tb_uid_t  suid;
-  int8_t    source; // 1-taosX or 0-taosClient
-  int8_t    reserved[8];
+  char     name[TSDB_TABLE_FNAME_LEN];
+  int8_t   igExists;
+  int8_t   source;  // 1-taosX or 0-taosClient
+  int8_t   reserved[6];
+  tb_uid_t suid;
+  int64_t  delay1;
+  int64_t  delay2;
+  int64_t  watermark1;
+  int64_t  watermark2;
+  int32_t  ttl;
+  int32_t  colVer;
+  int32_t  tagVer;
+  int32_t  numOfColumns;
+  int32_t  numOfTags;
+  int32_t  numOfFuncs;
+  int32_t  commentLen;
+  int32_t  ast1Len;
+  int32_t  ast2Len;
+  SArray*  pColumns;  // array of SField
+  SArray*  pTags;     // array of SField
+  SArray*  pFuncs;
+  char*    pComment;
+  char*    pAst1;
+  char*    pAst2;
 } SMCreateStbReq;
 
 int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq);
@@ -463,11 +466,11 @@ int32_t tDeserializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pR
 void    tFreeSMCreateStbReq(SMCreateStbReq* pReq);
 
 typedef struct {
-  char      name[TSDB_TABLE_FNAME_LEN];
-  int8_t    igNotExists;
-  tb_uid_t  suid;
-  int8_t    source; // 1-taosX or 0-taosClient
-  int8_t    reserved[8];
+  char     name[TSDB_TABLE_FNAME_LEN];
+  int8_t   igNotExists;
+  int8_t   source;  // 1-taosX or 0-taosClient
+  int8_t   reserved[6];
+  tb_uid_t suid;
 } SMDropStbReq;
 
 int32_t tSerializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 7a4bcd85b8..9866da39da 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -246,9 +246,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_COLUMN_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x03AE)
 #define TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03AF)
 #define TSDB_CODE_MND_SINGLE_STB_MODE_DB        TAOS_DEF_ERROR_CODE(0, 0x03B0)
+#define TSDB_CODE_MND_INVALID_SCHEMA_VER        TAOS_DEF_ERROR_CODE(0, 0x03B1)
 
 // mnode-infoSchema
-#define TSDB_CODE_MND_INVALID_SYS_TABLENAME         TAOS_DEF_ERROR_CODE(0, 0x03BA)
+#define TSDB_CODE_MND_INVALID_SYS_TABLENAME     TAOS_DEF_ERROR_CODE(0, 0x03BA)
 
 // mnode-func
 #define TSDB_CODE_MND_FUNC_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x03C0)
diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index 907691698c..a576dff950 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -2303,8 +2303,8 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){
     strcpy(field.name, pSchema->name);
     taosArrayPush(pReq.pTags, &field);
   }
-  pReq.cVersion = req.schemaRow.version;
-  pReq.tVersion = req.schemaTag.version;
+  pReq.colVer = req.schemaRow.version;
+  pReq.tagVer = req.schemaTag.version;
   pReq.numOfColumns = req.schemaRow.nCols;
   pReq.numOfTags = req.schemaTag.nCols;
   pReq.commentLen = -1;
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 165e8631c4..aa43a4d2c3 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -496,11 +496,18 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
+    if (tEncodeI8(&encoder, pReq->reserved[i]) < 0) return -1;
+  }
+  if (tEncodeI64(&encoder, pReq->suid) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->delay1) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->delay2) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->watermark1) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->watermark2) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->colVer) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->tagVer) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfFuncs) < 0) return -1;
@@ -516,8 +523,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
     if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
   }
 
-  if (tEncodeI32(&encoder, pReq->cVersion) < 0) return -1;
-
   for (int32_t i = 0; i < pReq->numOfTags; ++i) {
     SField *pField = taosArrayGet(pReq->pTags, i);
     if (tEncodeI8(&encoder, pField->type) < 0) return -1;
@@ -526,8 +531,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
     if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
   }
 
-  if (tEncodeI32(&encoder, pReq->tVersion) < 0) return -1;
-
   for (int32_t i = 0; i < pReq->numOfFuncs; ++i) {
     const char *pFunc = taosArrayGet(pReq->pFuncs, i);
     if (tEncodeCStr(&encoder, pFunc) < 0) return -1;
@@ -542,11 +545,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
   if (pReq->ast2Len > 0) {
     if (tEncodeBinary(&encoder, pReq->pAst2, pReq->ast2Len) < 0) return -1;
   }
-  if (tEncodeI64(&encoder, pReq->suid) < 0) return -1;
-  if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
-  for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) {
-    if (tEncodeI8(&encoder, pReq->reserved[i]) < 0) return -1;
-  }
 
   tEndEncode(&encoder);
 
@@ -562,11 +560,18 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
+    if (tDecodeI8(&decoder, &pReq->reserved[i]) < 0) return -1;
+  }
+  if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->delay1) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->delay2) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->watermark1) < 0) return -1;
   if (tDecodeI64(&decoder, &pReq->watermark2) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->colVer) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->tagVer) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfFuncs) < 0) return -1;
@@ -594,8 +599,6 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     }
   }
 
-  if (tDecodeI32(&decoder, &pReq->cVersion) < 0) return -1;
-
   for (int32_t i = 0; i < pReq->numOfTags; ++i) {
     SField field = {0};
     if (tDecodeI8(&decoder, &field.type) < 0) return -1;
@@ -608,8 +611,6 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     }
   }
 
-  if (tDecodeI32(&decoder, &pReq->tVersion) < 0) return -1;
-
   for (int32_t i = 0; i < pReq->numOfFuncs; ++i) {
     char pFunc[TSDB_FUNC_NAME_LEN] = {0};
     if (tDecodeCStrTo(&decoder, pFunc) < 0) return -1;
@@ -637,12 +638,6 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
     if (tDecodeCStrTo(&decoder, pReq->pAst2) < 0) return -1;
   }
 
-  if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1;
-  if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
-  for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) {
-    if (tDecodeI8(&decoder, &pReq->reserved[i]) < 0) return -1;
-  }
-
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
   return 0;
@@ -664,11 +659,11 @@ int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) {
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
-  if (tEncodeI64(&encoder, pReq->suid) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->source) < 0) return -1;
-  for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) {
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
     if (tEncodeI8(&encoder, pReq->reserved[i]) < 0) return -1;
   }
+  if (tEncodeI64(&encoder, pReq->suid) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -683,11 +678,11 @@ int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq)
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
-  if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->source) < 0) return -1;
-  for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) {
+  for (int32_t i = 0; i < sizeof(pReq->reserved) / sizeof(int8_t); ++i) {
     if (tDecodeI8(&decoder, &pReq->reserved[i]) < 0) return -1;
   }
+  if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1;
 
   tEndDecode(&decoder);
 
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 6d1cc05ace..dc15c59fc4 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -708,8 +708,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
   pDst->updateTime = pDst->createdTime;
   pDst->uid = (pCreate->source == 1) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
   pDst->dbUid = pDb->uid;
-  pDst->tagVer = (pCreate->source == 1) ? pCreate->tVersion : 1;
-  pDst->colVer = (pCreate->source == 1) ? pCreate->cVersion : 1;
+  pDst->tagVer = (pCreate->source != TD_REQ_FROM_APP) ? pCreate->tagVer : 1;
+  pDst->colVer = (pCreate->source != TD_REQ_FROM_APP) ? pCreate->colVer : 1;
   pDst->smaVer = 1;
   pDst->nextColId = 1;
   pDst->maxdelay[0] = pCreate->delay1;
@@ -900,6 +900,25 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
+  if (createReq.tagVer > 0 || createReq.colVer > 0) {
+    int32_t tagDelta = pStb->tagVer - createReq.tagVer;
+    int32_t colDelta = pStb->colVer - createReq.colVer;
+    int32_t verDelta = tagDelta + verDelta;
+    mInfo("stb:%s, already exist while create, input tagVer:%d colVer:%d, mnode tagVer:%d colVer:%d", createReq.name,
+          createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
+    if (tagDelta <= 0 && colDelta <= 0) {
+      mInfo("stb:%s, schema version is not incremented and nothing needs to be done", createReq.name);
+      code = 0;
+      goto _OVER;
+    } else if ((tagDelta == 1 || colDelta == 1) && (verDelta == 1)) {
+      mInfo("stb:%s, schema version is only increased by 1 digit, do alter operation", createReq.name);
+    } else {
+      mInfo("stb:%s, schema version increase more than 1 digit, error is returned", createReq.name);
+      terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
+      goto _OVER;
+    }
+  }
+
   code = mndCreateStb(pMnode, pReq, &createReq, pDb);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -1614,12 +1633,23 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if ((alterReq.tagVer > 0 && alterReq.colVer > 0) &&
-      (alterReq.tagVer <= pStb->tagVer || alterReq.colVer <= pStb->colVer)) {
-    mDebug("stb:%s, already exist, tagVer:%d colVer:%d smaller than in mnode, tagVer:%d colVer:%d, alter success",
-           alterReq.name, alterReq.tagVer, alterReq.colVer, pStb->tagVer, pStb->colVer);
-    code = 0;
-    goto _OVER;
+  if (alterReq.tagVer > 0 || alterReq.colVer > 0) {
+    int32_t tagDelta = pStb->tagVer - alterReq.tagVer;
+    int32_t colDelta = pStb->colVer - alterReq.colVer;
+    int32_t verDelta = tagDelta + verDelta;
+    mInfo("stb:%s, already exist while alter, input tagVer:%d colVer:%d, mnode tagVer:%d colVer:%d", alterReq.name,
+          alterReq.tagVer, alterReq.colVer, pStb->tagVer, pStb->colVer);
+    if (tagDelta <= 0 && colDelta <= 0) {
+      mInfo("stb:%s, schema version is not incremented and nothing needs to be done", alterReq.name);
+      code = 0;
+      goto _OVER;
+    } else if ((tagDelta == 1 || colDelta == 1) && (verDelta == 1)) {
+      mInfo("stb:%s, schema version is only increased by 1 digit, do alter operation", alterReq.name);
+    } else {
+      mInfo("stb:%s, schema version increase more than 1 digit, error is returned", alterReq.name);
+      terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
+      goto _OVER;
+    }
   }
 
   if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
@@ -1752,7 +1782,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
     }
   }
 
-  if (dropReq.source == 1 && pStb->uid != dropReq.suid){
+  if (dropReq.source != TD_REQ_FROM_APP && pStb->uid != dropReq.suid) {
     terrno = TSDB_CODE_MND_STB_NOT_EXIST;
     goto _OVER;
   }
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index e05b634222..6642d76c19 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -250,6 +250,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST,     "Column already exists
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST,         "Column does not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC,"Field used by topic")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_SINGLE_STB_MODE_DB,       "Database is single stable mode")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SCHEMA_VER,       "Invalid schema version while alter stb")
 
 // mnode-infoSchema
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME,    "Invalid system table name")

From 998de54728e0f7fbcc3c91c7fc09886e27320613 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 10 Jul 2022 11:17:51 +0800
Subject: [PATCH 42/50] feat:write meta from tmq to taosd

---
 include/common/tmsg.h                |  2 -
 include/util/taoserror.h             |  1 +
 source/common/src/tmsg.c             |  4 --
 source/dnode/mnode/impl/src/mndStb.c | 73 ++++++++++++----------------
 source/util/src/terror.c             |  1 +
 5 files changed, 34 insertions(+), 47 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index 4e42437b09..03b456b212 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -479,8 +479,6 @@ int32_t tDeserializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
   int8_t  alterType;
-  int32_t tagVer;
-  int32_t colVer;
   int32_t numOfFields;
   SArray* pFields;
   int32_t ttl;
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index a4ee38b144..fcc00d0d80 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -248,6 +248,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03AF)
 #define TSDB_CODE_MND_SINGLE_STB_MODE_DB        TAOS_DEF_ERROR_CODE(0, 0x03B0)
 #define TSDB_CODE_MND_INVALID_SCHEMA_VER        TAOS_DEF_ERROR_CODE(0, 0x03B1)
+#define TSDB_CODE_MND_STABLE_UID_NOT_MATCH      TAOS_DEF_ERROR_CODE(0, 0x03B2)
 
 // mnode-infoSchema
 #define TSDB_CODE_MND_INVALID_SYS_TABLENAME     TAOS_DEF_ERROR_CODE(0, 0x03BA)
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 6f02bd91c1..42c18313d9 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -697,8 +697,6 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq)
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
   if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->tagVer) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->colVer) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->numOfFields) < 0) return -1;
   for (int32_t i = 0; i < pReq->numOfFields; ++i) {
     SField *pField = taosArrayGet(pReq->pFields, i);
@@ -725,8 +723,6 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
   if (tDecodeI8(&decoder, &pReq->alterType) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->tagVer) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->colVer) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->numOfFields) < 0) return -1;
   pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SField));
   if (pReq->pFields == NULL) {
diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index 8a575de854..224c1dbb5d 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -869,9 +869,38 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
   pStb = mndAcquireStb(pMnode, createReq.name);
   if (pStb != NULL) {
     if (createReq.igExists) {
-      mDebug("stb:%s, already exist, ignore exist is set", createReq.name);
-      code = 0;
-      goto _OVER;
+      if (createReq.source == TD_REQ_FROM_APP) {
+        mDebug("stb:%s, already exist, ignore exist is set", createReq.name);
+        code = 0;
+        goto _OVER;
+      } else if (pStb->uid != createReq.suid) {
+        mError("stb:%s, already exist while create, input suid:%" PRId64 " not match with exist suid:%" PRId64,
+               createReq.name, createReq.suid, pStb->uid);
+        terrno = TSDB_CODE_MND_STABLE_UID_NOT_MATCH;
+        goto _OVER;
+      } else if (createReq.tagVer > 0 || createReq.colVer > 0) {
+        int32_t tagDelta = pStb->tagVer - createReq.tagVer;
+        int32_t colDelta = pStb->colVer - createReq.colVer;
+        int32_t verDelta = tagDelta + verDelta;
+        mInfo("stb:%s, already exist while create, input tagVer:%d colVer:%d, exist tagVer:%d colVer:%d",
+              createReq.name, createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
+        if (tagDelta <= 0 && colDelta <= 0) {
+          mInfo("stb:%s, schema version is not incremented and nothing needs to be done", createReq.name);
+          code = 0;
+          goto _OVER;
+        } else if ((tagDelta == 1 || colDelta == 1) && (verDelta == 1)) {
+          mInfo("stb:%s, schema version is only increased by 1 number, do alter operation", createReq.name);
+        } else {
+          mError("stb:%s, schema version increase more than 1 number, error is returned", createReq.name);
+          terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
+          goto _OVER;
+        }
+      } else {
+        mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid", createReq.name,
+               createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
+        terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
+        goto _OVER;
+      }
     } else {
       terrno = TSDB_CODE_MND_STB_ALREADY_EXIST;
       goto _OVER;
@@ -900,25 +929,6 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (createReq.tagVer > 0 || createReq.colVer > 0) {
-    int32_t tagDelta = pStb->tagVer - createReq.tagVer;
-    int32_t colDelta = pStb->colVer - createReq.colVer;
-    int32_t verDelta = tagDelta + verDelta;
-    mInfo("stb:%s, already exist while create, input tagVer:%d colVer:%d, mnode tagVer:%d colVer:%d", createReq.name,
-          createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
-    if (tagDelta <= 0 && colDelta <= 0) {
-      mInfo("stb:%s, schema version is not incremented and nothing needs to be done", createReq.name);
-      code = 0;
-      goto _OVER;
-    } else if ((tagDelta == 1 || colDelta == 1) && (verDelta == 1)) {
-      mInfo("stb:%s, schema version is only increased by 1 digit, do alter operation", createReq.name);
-    } else {
-      mInfo("stb:%s, schema version increase more than 1 digit, error is returned", createReq.name);
-      terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
-      goto _OVER;
-    }
-  }
-
   code = mndCreateStb(pMnode, pReq, &createReq, pDb);
   if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
 
@@ -1633,25 +1643,6 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  if (alterReq.tagVer > 0 || alterReq.colVer > 0) {
-    int32_t tagDelta = pStb->tagVer - alterReq.tagVer;
-    int32_t colDelta = pStb->colVer - alterReq.colVer;
-    int32_t verDelta = tagDelta + verDelta;
-    mInfo("stb:%s, already exist while alter, input tagVer:%d colVer:%d, mnode tagVer:%d colVer:%d", alterReq.name,
-          alterReq.tagVer, alterReq.colVer, pStb->tagVer, pStb->colVer);
-    if (tagDelta <= 0 && colDelta <= 0) {
-      mInfo("stb:%s, schema version is not incremented and nothing needs to be done", alterReq.name);
-      code = 0;
-      goto _OVER;
-    } else if ((tagDelta == 1 || colDelta == 1) && (verDelta == 1)) {
-      mInfo("stb:%s, schema version is only increased by 1 digit, do alter operation", alterReq.name);
-    } else {
-      mInfo("stb:%s, schema version increase more than 1 digit, error is returned", alterReq.name);
-      terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
-      goto _OVER;
-    }
-  }
-
   if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) {
     goto _OVER;
   }
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 288bc63363..ef6697b3b5 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -252,6 +252,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST,         "Column does not exist
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC,"Field used by topic")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_SINGLE_STB_MODE_DB,       "Database is single stable mode")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SCHEMA_VER,       "Invalid schema version while alter stb")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_STABLE_UID_NOT_MATCH,     "Invalid stable uid while alter stb")
 
 // mnode-infoSchema
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME,    "Invalid system table name")

From a2107992e257028b1ddc2b607d2ec003f2f489d2 Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Sun, 10 Jul 2022 11:18:15 +0800
Subject: [PATCH 43/50] refactor(stream)

---
 include/libs/executor/executor.h        |  2 +-
 include/libs/wal/wal.h                  |  1 +
 source/dnode/vnode/src/tq/tq.c          | 34 +++++-----------
 source/dnode/vnode/src/tq/tqExec.c      |  6 +--
 source/dnode/vnode/src/tq/tqRead.c      | 22 +++++-----
 source/libs/executor/src/executorMain.c | 53 +++++++++++++++++++++++--
 source/libs/executor/src/scanoperator.c | 17 +++++---
 source/libs/wal/src/walRead.c           | 38 ++++++++++--------
 8 files changed, 108 insertions(+), 65 deletions(-)

diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h
index 1edf41b6f7..cff4b0234c 100644
--- a/include/libs/executor/executor.h
+++ b/include/libs/executor/executor.h
@@ -176,7 +176,7 @@ int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);
 
 int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts);
 
-int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset);
+int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset);
 
 int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset);
 
diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h
index e4189afb44..45f7034bfe 100644
--- a/include/libs/wal/wal.h
+++ b/include/libs/wal/wal.h
@@ -161,6 +161,7 @@ typedef struct {
   int64_t        curFileFirstVer;
   int64_t        curVersion;
   int64_t        capacity;
+  int8_t         curInvalid;
   TdThreadMutex  mutex;
   SWalFilterCond cond;
   SWalCkHead    *pHead;
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 4fcc47b0ad..fb51cca07a 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -270,6 +270,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   tqDebug("tmq poll: consumer %ld (epoch %d), subkey %s, recv poll req in vg %d, req offset %s", consumerId,
           pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf);
 
+  SMqDataRsp dataRsp = {0};
+  tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
+
   // 2.reset offset if needed
   if (reqOffset.type > 0) {
     fetchOffsetNew = reqOffset;
@@ -293,40 +296,24 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
           tqOffsetResetToLog(&fetchOffsetNew, walGetFirstVer(pTq->pVnode->pWal));
         }
       } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) {
-        tqOffsetResetToLog(&fetchOffsetNew, walGetLastVer(pTq->pVnode->pWal));
+        tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal));
         tqDebug("tmq poll: consumer %ld, subkey %s, offset reset to %ld", consumerId, pHandle->subKey,
-                fetchOffsetNew.version);
-        SMqDataRsp dataRsp = {0};
-        tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
-        dataRsp.rspOffset = fetchOffsetNew;
-        code = 0;
+                dataRsp.rspOffset.version);
         if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
           code = -1;
         }
-        taosArrayDestroy(dataRsp.blockDataLen);
-        taosArrayDestroyP(dataRsp.blockData, (FDelete)taosMemoryFree);
-
-        if (dataRsp.withSchema) {
-          taosArrayDestroyP(dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
-        }
-
-        if (dataRsp.withTbName) {
-          taosArrayDestroyP(dataRsp.blockTbName, (FDelete)taosMemoryFree);
-        }
-        return code;
+        goto OVER;
       } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) {
         tqError("tmq poll: subkey %s, no offset committed for consumer %ld in vg %d, subkey %s, reset none failed",
                 pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey);
         terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
-        return -1;
+        code = -1;
+        goto OVER;
       }
     }
   }
 
   // 3.query
-  SMqDataRsp dataRsp = {0};
-  tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
-
   if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) {
     fetchOffsetNew.version++;
     if (tqScanLog(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) {
@@ -335,7 +322,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       goto OVER;
     }
     if (dataRsp.blockNum == 0) {
-      // TODO add to async task
+      // TODO add to async task pool
       /*dataRsp.rspOffset.version--;*/
     }
     if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) {
@@ -348,7 +335,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     int64_t     fetchVer = fetchOffsetNew.version + 1;
     SWalCkHead* pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
     if (pCkHead == NULL) {
-      return -1;
+      code = -1;
+      goto OVER;
     }
 
     walSetReaderCapacity(pHandle->pWalReader, 2048);
diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c
index d381cfcdc7..4ecd88b1fd 100644
--- a/source/dnode/vnode/src/tq/tqExec.c
+++ b/source/dnode/vnode/src/tq/tqExec.c
@@ -62,7 +62,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) {
 int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
   qTaskInfo_t task = pExec->execCol.task[0];
 
-  if (qStreamPrepareScan1(task, pOffset) < 0) {
+  if (qStreamPrepareScan(task, pOffset) < 0) {
     pRsp->rspOffset = *pOffset;
     pRsp->rspOffset.version--;
     return 0;
@@ -110,10 +110,6 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, S
   ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
   qTaskInfo_t task = pExec->execCol.task[workerId];
 
-  /*if (qStreamScanSnapshot(task) < 0) {*/
-  /*ASSERT(0);*/
-  /*}*/
-
   if (qStreamPrepareTsdbScan(task, offset.uid, offset.ts) < 0) {
     ASSERT(0);
   }
diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 4115479e25..2d39cd1346 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -104,8 +104,13 @@ void tqCloseReader(STqReader* pReader) {
 }
 
 int32_t tqSeekVer(STqReader* pReader, int64_t ver) {
-  //
-  return walReadSeekVer(pReader->pWalReader, ver);
+  if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
+    ASSERT(pReader->pWalReader->curInvalid);
+    ASSERT(pReader->pWalReader->curVersion == ver);
+    return -1;
+  }
+  ASSERT(pReader->pWalReader->curVersion == ver);
+  return 0;
 }
 
 int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
@@ -114,9 +119,11 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
   while (1) {
     if (!fromProcessedMsg) {
       if (walNextValidMsg(pReader->pWalReader) < 0) {
+        pReader->ver = pReader->pWalReader->curVersion;
         ret->offset.type = TMQ_OFFSET__LOG;
         ret->offset.version = pReader->ver;
         ret->fetchType = FETCH_TYPE__NONE;
+        ASSERT(ret->offset.version >= 0);
         return -1;
       }
       void* body = pReader->pWalReader->pHead->head.body;
@@ -131,19 +138,12 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
     }
 
     while (tqNextDataBlock(pReader)) {
+      // TODO mem free
       memset(&ret->data, 0, sizeof(SSDataBlock));
       int32_t code = tqRetrieveDataBlock(&ret->data, pReader);
       if (code != 0 || ret->data.info.rows == 0) {
         ASSERT(0);
         continue;
-#if 0
-        if (fromProcessedMsg) {
-          ret->fetchType = FETCH_TYPE__NONE;
-          return 0;
-        } else {
-          break;
-        }
-#endif
       }
       ret->fetchType = FETCH_TYPE__DATA;
       return 0;
@@ -152,7 +152,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
     if (fromProcessedMsg) {
       ret->offset.type = TMQ_OFFSET__LOG;
       ret->offset.version = pReader->ver;
-      ASSERT(pReader->ver != -1);
+      ASSERT(pReader->ver >= 0);
       ret->fetchType = FETCH_TYPE__NONE;
       return 0;
     }
diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c
index ad33c0ae55..ca1d8b2b10 100644
--- a/source/libs/executor/src/executorMain.c
+++ b/source/libs/executor/src/executorMain.c
@@ -280,7 +280,7 @@ int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) {
   return 0;
 }
 
-int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
+int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
   SOperatorInfo* pOperator = pTaskInfo->pRoot;
   ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM);
@@ -293,8 +293,55 @@ int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) {
     pOperator->status = OP_OPENED;
     if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
       SStreamScanInfo* pInfo = pOperator->info;
-      if (tqSeekVer(pInfo->tqReader, pOffset->version) < 0) {
-        return -1;
+      if (pOffset->type == TMQ_OFFSET__LOG) {
+        if (tqSeekVer(pInfo->tqReader, pOffset->version) < 0) {
+          return -1;
+        }
+        ASSERT(pInfo->tqReader->pWalReader->curVersion == pOffset->version);
+      } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) {
+        pInfo->blockType = STREAM_INPUT__TABLE_SCAN;
+        int64_t uid = pOffset->uid;
+        int64_t ts = pOffset->ts;
+
+        if (uid == 0) {
+          if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) {
+            STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0);
+            uid = pTableInfo->uid;
+            ts = INT64_MIN;
+          }
+        }
+        if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||
+            pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {
+          STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
+          int32_t         tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
+          bool            found = false;
+          for (int32_t i = 0; i < tableSz; i++) {
+            STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i);
+            if (pTableInfo->uid == uid) {
+              found = true;
+              pTableScanInfo->currentTable = i;
+            }
+          }
+
+          // TODO after dropping table, table may be not found
+          ASSERT(found);
+
+          tsdbSetTableId(pTableScanInfo->dataReader, uid);
+          int64_t oldSkey = pTableScanInfo->cond.twindows[0].skey;
+          pTableScanInfo->cond.twindows[0].skey = ts + 1;
+          tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
+          pTableScanInfo->cond.twindows[0].skey = oldSkey;
+          pTableScanInfo->scanTimes = 0;
+          pTableScanInfo->curTWinIdx = 0;
+
+          qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts,
+                 pTableScanInfo->currentTable, tableSz);
+        } else {
+          // switch to log
+        }
+
+      } else {
+        ASSERT(0);
       }
       return 0;
     } else {
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 65680de347..07df6339db 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -1234,17 +1234,21 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
         pTaskInfo->streamInfo.metaBlk = ret.meta;
         return NULL;
       } else if (ret.fetchType == FETCH_TYPE__NONE) {
-        if (ret.offset.version == -1) {
-          pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG;
-          pTaskInfo->streamInfo.lastStatus.version = pTaskInfo->streamInfo.prepareStatus.version - 1;
-        } else {
-          pTaskInfo->streamInfo.lastStatus = ret.offset;
-        }
+        /*if (ret.offset.version == -1) {*/
+        /*pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG;*/
+        /*pTaskInfo->streamInfo.lastStatus.version = pTaskInfo->streamInfo.prepareStatus.version - 1;*/
+        /*} else {*/
+        pTaskInfo->streamInfo.lastStatus = ret.offset;
+        ASSERT(pTaskInfo->streamInfo.lastStatus.version >= pTaskInfo->streamInfo.prepareStatus.version);
+        /*}*/
         return NULL;
       } else {
         ASSERT(0);
       }
     }
+  } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) {
+    SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
+    return pResult && pResult->info.rows > 0 ? pResult : NULL;
   }
 
   size_t total = taosArrayGetSize(pInfo->pBlockLists);
@@ -1448,6 +1452,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
     return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
 
   } else if (pInfo->blockType == STREAM_INPUT__TABLE_SCAN) {
+    ASSERT(0);
     // check reader last status
     // if not match, reset status
     SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index 74fa7b0d52..b104e07e55 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -30,8 +30,9 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) {
   pRead->pWal = pWal;
   pRead->pIdxFile = NULL;
   pRead->pLogFile = NULL;
-  pRead->curVersion = -1;
+  pRead->curVersion = -5;
   pRead->curFileFirstVer = -1;
+  pRead->curInvalid = 1;
   pRead->capacity = 0;
   if (cond)
     pRead->cond = *cond;
@@ -149,10 +150,14 @@ static int32_t walReadChangeFile(SWalReader *pRead, int64_t fileFirstVer) {
 
 int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) {
   SWal *pWal = pRead->pWal;
-  if (ver == pRead->curVersion) {
+  if (!pRead->curInvalid && ver == pRead->curVersion) {
     wDebug("wal version %ld match, no need to reset", ver);
     return 0;
   }
+
+  pRead->curInvalid = 1;
+  pRead->curVersion = ver;
+
   if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
     wError("invalid version: % " PRId64 ", first ver %ld, last ver %ld", ver, pWal->vers.firstVer, pWal->vers.lastVer);
     terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
@@ -189,9 +194,11 @@ void walSetReaderCapacity(SWalReader *pRead, int32_t capacity) { pRead->capacity
 
 static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) {
   int64_t contLen;
-  if (pRead->curVersion != fetchVer) {
+  if (pRead->curInvalid || pRead->curVersion != fetchVer) {
     if (walReadSeekVer(pRead, fetchVer) < 0) {
       ASSERT(0);
+      pRead->curVersion = fetchVer;
+      pRead->curInvalid = 1;
       return -1;
     }
   }
@@ -203,7 +210,7 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) {
       terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     }
     ASSERT(0);
-    pRead->curVersion = -1;
+    pRead->curInvalid = 1;
     return -1;
   }
   return 0;
@@ -234,14 +241,14 @@ static int32_t walFetchBodyNew(SWalReader *pRead) {
              pRead->pHead->head.version, ver);
       terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     }
-    pRead->curVersion = -1;
+    pRead->curInvalid = 1;
     ASSERT(0);
     return -1;
   }
 
   if (pReadHead->version != ver) {
     wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, ver);
-    pRead->curVersion = -1;
+    pRead->curInvalid = 1;
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     ASSERT(0);
     return -1;
@@ -249,7 +256,7 @@ static int32_t walFetchBodyNew(SWalReader *pRead) {
 
   if (walValidBodyCksum(pRead->pHead) != 0) {
     wError("wal fetch body error: % " PRId64 ", since body checksum not passed", ver);
-    pRead->curVersion = -1;
+    pRead->curInvalid = 1;
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     ASSERT(0);
     return -1;
@@ -268,7 +275,7 @@ static int32_t walSkipFetchBodyNew(SWalReader *pRead) {
   code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR);
   if (code < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    pRead->curVersion = -1;
+    pRead->curInvalid = 1;
     ASSERT(0);
     return -1;
   }
@@ -287,7 +294,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
     return -1;
   }
 
-  if (pRead->curVersion != ver) {
+  if (pRead->curInvalid || pRead->curVersion != ver) {
     code = walReadSeekVer(pRead, ver);
     if (code < 0) return -1;
   }
@@ -318,7 +325,7 @@ int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) {
   code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR);
   if (code < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
-    pRead->curVersion = -1;
+    pRead->curInvalid = 1;
     return -1;
   }
 
@@ -349,14 +356,14 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
 
   if (pReadHead->version != ver) {
     wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, ver);
-    pRead->curVersion = -1;
+    pRead->curInvalid = 1;
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;
   }
 
   if (walValidBodyCksum(*ppHead) != 0) {
     wError("wal fetch body error: % " PRId64 ", since body checksum not passed", ver);
-    pRead->curVersion = -1;
+    pRead->curInvalid = 1;
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;
   }
@@ -373,8 +380,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) {
     return -1;
   }
 
-  // TODO: check wal life
-  if (pRead->curVersion != ver) {
+  if (pRead->curInvalid || pRead->curVersion != ver) {
     if (walReadSeekVer(pRead, ver) < 0) {
       wError("unexpected wal log version: % " PRId64 ", since %s", ver, terrstr());
       return -1;
@@ -432,7 +438,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) {
   if (pRead->pHead->head.version != ver) {
     wError("unexpected wal log version: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version,
            ver);
-    pRead->curVersion = -1;
+    pRead->curInvalid = 1;
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;
   }
@@ -440,7 +446,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) {
   code = walValidBodyCksum(pRead->pHead);
   if (code != 0) {
     wError("unexpected wal log version: % " PRId64 ", since body checksum not passed", ver);
-    pRead->curVersion = -1;
+    pRead->curInvalid = 1;
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;
   }

From e733f8067d8fa56b417884473744372cf230e9ab Mon Sep 17 00:00:00 2001
From: dapan1121 <wpan@taosdata.com>
Date: Sun, 10 Jul 2022 11:21:13 +0800
Subject: [PATCH 44/50] enh: enhance query redirect

---
 source/client/src/clientImpl.c      |  2 +
 source/common/src/tmsg.c            |  8 ++--
 source/dnode/qnode/src/qnode.c      |  5 ---
 source/libs/scheduler/inc/schInt.h  | 70 +++++++++++++++--------------
 source/libs/scheduler/src/schJob.c  |  2 +-
 source/libs/scheduler/src/schTask.c | 42 ++++++++++++-----
 6 files changed, 76 insertions(+), 53 deletions(-)

diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c
index bff65d3527..6fb3d98027 100644
--- a/source/client/src/clientImpl.c
+++ b/source/client/src/clientImpl.c
@@ -325,11 +325,13 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) {
   if (pInfo->pQnodeList) {
     taosArrayDestroy(pInfo->pQnodeList);
     pInfo->pQnodeList = NULL;
+    tscDebug("QnodeList cleared in cluster 0x%" PRIx64, pInfo->clusterId);
   }
 
   if (pNodeList) {
     pInfo->pQnodeList = taosArrayDup(pNodeList);
     taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad);
+    tscDebug("QnodeList updated in cluster 0x%" PRIx64 ", num:%d", pInfo->clusterId, taosArrayGetSize(pInfo->pQnodeList));
   }
   taosThreadMutexUnlock(&pInfo->qnodeMutex);
 
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index 192a41a70e..6c202c6202 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -381,9 +381,11 @@ static int32_t tDeserializeSClientHbRsp(SDecoder *pDecoder, SClientHbRsp *pRsp)
     if (pQnodeNum > 0) {
       pRsp->query->pQnodeList = taosArrayInit(pQnodeNum, sizeof(SQueryNodeLoad));
       if (NULL == pRsp->query->pQnodeList) return -1;
-      SQueryNodeLoad load = {0};
-      if (tDecodeSQueryNodeLoad(pDecoder, &load) < 0) return -1;
-      taosArrayPush(pRsp->query->pQnodeList, &load);
+      for (int32_t i = 0; i < pQnodeNum; ++i) {
+        SQueryNodeLoad load = {0};
+        if (tDecodeSQueryNodeLoad(pDecoder, &load) < 0) return -1;
+        taosArrayPush(pRsp->query->pQnodeList, &load);
+      }
     }
   }
 
diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c
index 03d14d7ab4..b65189153e 100644
--- a/source/dnode/qnode/src/qnode.c
+++ b/source/dnode/qnode/src/qnode.c
@@ -64,16 +64,11 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) {
   return 0; 
 }
 
-int32_t qnodeCrash = 1;
 int32_t qndPreprocessQueryMsg(SQnode *pQnode, SRpcMsg * pMsg) {
   if (TDMT_SCH_QUERY != pMsg->msgType && TDMT_SCH_MERGE_QUERY != pMsg->msgType) {
     return 0;
   }
 
-  if (qnodeCrash) {
-    ASSERT(0);
-  }
-
   return qWorkerPreprocessQueryMsg(pQnode->pQuery, pMsg);
 }
 
diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h
index 591301ef44..052fdefa61 100644
--- a/source/libs/scheduler/inc/schInt.h
+++ b/source/libs/scheduler/inc/schInt.h
@@ -35,7 +35,6 @@ extern "C" {
 #define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000
 #define SCH_MAX_TASK_TIMEOUT_USEC 60000000
 
-#define SCH_TASK_MAX_EXEC_TIMES 8
 #define SCH_MAX_CANDIDATE_EP_NUM TSDB_MAX_REPLICA
 
 enum {
@@ -179,10 +178,10 @@ typedef struct SSchLevel {
 } SSchLevel;
 
 typedef struct SSchTaskProfile {
-  int64_t startTs;
-  int64_t execUseTime[SCH_TASK_MAX_EXEC_TIMES];
-  int64_t waitTime;
-  int64_t endTs;
+  int64_t  startTs;
+  int64_t* execTime;
+  int64_t  waitTime;
+  int64_t  endTs;
 } SSchTaskProfile;
 
 typedef struct SSchTask {
@@ -260,33 +259,7 @@ typedef struct SSchJob {
 
 extern SSchedulerMgmt schMgmt;
 
-#define SCH_LOG_TASK_START_TS(_task)                          \
-  do {                                                        \
-    int64_t us = taosGetTimestampUs();                        \
-    int32_t idx = (_task)->execId % SCH_TASK_MAX_EXEC_TIMES; \
-    (_task)->profile.execUseTime[idx] = us;                    \
-    if (0 == (_task)->execId) {                              \
-      (_task)->profile.startTs = us;                          \
-    }                                                         \
-  } while (0)  
-
-#define SCH_LOG_TASK_WAIT_TS(_task)                        \
-  do {                                                    \
-    int64_t us = taosGetTimestampUs();                    \
-    int32_t idx = (_task)->execId % SCH_TASK_MAX_EXEC_TIMES; \
-    (_task)->profile.waitTime += us - (_task)->profile.execUseTime[idx];    \
-  } while (0)  
-
-
-#define SCH_LOG_TASK_END_TS(_task)                        \
-  do {                                                    \
-    int64_t us = taosGetTimestampUs();                    \
-    int32_t idx = (_task)->execId % SCH_TASK_MAX_EXEC_TIMES; \
-    (_task)->profile.execUseTime[idx] = us - (_task)->profile.execUseTime[idx];    \
-    (_task)->profile.endTs = us;                          \
-  } while (0)  
-
-#define SCH_TASK_TIMEOUT(_task) ((taosGetTimestampUs() - (_task)->profile.execUseTime[(_task)->execId % SCH_TASK_MAX_EXEC_TIMES]) > (_task)->timeoutUsec)
+#define SCH_TASK_TIMEOUT(_task) ((taosGetTimestampUs() - (_task)->profile.execTime[(_task)->execId % (_task)->maxExecTimes]) > (_task)->timeoutUsec)
 
 #define SCH_TASK_READY_FOR_LAUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children))
 
@@ -320,6 +293,7 @@ extern SSchedulerMgmt schMgmt;
 #define SCH_TASK_NEED_FLOW_CTRL(_job, _task) (SCH_IS_DATA_BIND_QRY_TASK(_task) && SCH_JOB_NEED_FLOW_CTRL(_job) && SCH_IS_LEVEL_UNFINISHED((_task)->level))
 #define SCH_FETCH_TYPE(_pSrcTask) (SCH_IS_DATA_BIND_QRY_TASK(_pSrcTask) ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH)
 #define SCH_TASK_NEED_FETCH(_task) ((_task)->plan->subplanType != SUBPLAN_TYPE_MODIFY)
+#define SCH_TASK_MAX_EXEC_TIMES(_levelIdx, _levelNum) (SCH_MAX_CANDIDATE_EP_NUM * ((_levelNum) - (_levelIdx)))
 
 #define SCH_SET_JOB_TYPE(_job, type) do { if ((type) != SUBPLAN_TYPE_MODIFY) { (_job)->attr.queryJob = true; } } while (0)
 #define SCH_IS_QUERY_JOB(_job) ((_job)->attr.queryJob) 
@@ -328,7 +302,7 @@ extern SSchedulerMgmt schMgmt;
 #define SCH_JOB_NEED_DROP(_job) (SCH_IS_QUERY_JOB(_job))
 #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode)
 #define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
-#define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) (SCH_NETWORK_ERR(_code) && ((_len) > 0 || SCH_IS_DATA_BIND_TASK(_task)))
+#define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task))))
 #define SCH_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH)
 #define SCH_TASK_NEED_REDIRECT(_task, _msgType, _code, _rspLen) (SCH_REDIRECT_MSGTYPE(_msgType) && (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_MERGE_TASK_NETWORK_ERR((_task), (_code), (_rspLen))))
 #define SCH_NEED_RETRY(_msgType, _code) ((SCH_NETWORK_ERR(_code) && SCH_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR)
@@ -338,6 +312,33 @@ extern SSchedulerMgmt schMgmt;
 #define SCH_SWITCH_EPSET(_addr) ((_addr)->epSet.inUse = ((_addr)->epSet.inUse + 1) % (_addr)->epSet.numOfEps)
 #define SCH_TASK_NUM_OF_EPS(_addr) ((_addr)->epSet.numOfEps)
 
+#define SCH_LOG_TASK_START_TS(_task)                          \
+  do {                                                        \
+    int64_t us = taosGetTimestampUs();                        \
+    int32_t idx = (_task)->execId % (_task)->maxExecTimes; \
+    (_task)->profile.execTime[idx] = us;                    \
+    if (0 == (_task)->execId) {                              \
+      (_task)->profile.startTs = us;                          \
+    }                                                         \
+  } while (0)  
+
+#define SCH_LOG_TASK_WAIT_TS(_task)                        \
+  do {                                                    \
+    int64_t us = taosGetTimestampUs();                    \
+    int32_t idx = (_task)->execId % (_task)->maxExecTimes; \
+    (_task)->profile.waitTime += us - (_task)->profile.execTime[idx];    \
+  } while (0)  
+
+
+#define SCH_LOG_TASK_END_TS(_task)                        \
+  do {                                                    \
+    int64_t us = taosGetTimestampUs();                    \
+    int32_t idx = (_task)->execId % (_task)->maxExecTimes; \
+    (_task)->profile.execTime[idx] = us - (_task)->profile.execTime[idx];    \
+    (_task)->profile.endTs = us;                          \
+  } while (0)  
+
+
 #define SCH_JOB_ELOG(param, ...) qError("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__)
 #define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__)
 
@@ -431,7 +432,8 @@ void    schFreeTask(SSchJob *pJob, SSchTask *pTask);
 void    schDropTaskInHashList(SSchJob *pJob, SHashObj *list);
 int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level);
 int32_t schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask);
-int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel);
+int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel, int32_t levelNum);
+int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask);
 
 
 #ifdef __cplusplus
diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c
index 13100afb8b..d2f9624eee 100644
--- a/source/libs/scheduler/src/schJob.c
+++ b/source/libs/scheduler/src/schJob.c
@@ -337,7 +337,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
       SCH_SET_JOB_TYPE(pJob, plan->subplanType);
 
       SSchTask  task = {0};
-      SCH_ERR_JRET(schInitTask(pJob, &task, plan, pLevel));
+      SCH_ERR_JRET(schInitTask(pJob, &task, plan, pLevel, levelNum));
 
       SSchTask *pTask = taosArrayPush(pLevel->subTasks, &task);
       if (NULL == pTask) {
diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c
index d56397283c..e1e4ed8769 100644
--- a/source/libs/scheduler/src/schTask.c
+++ b/source/libs/scheduler/src/schTask.c
@@ -46,21 +46,31 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask) {
 }
 
 
-int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel) {
+int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel, int32_t levelNum) {
+  int32_t code = 0;
+  
   pTask->plan = pPlan;
   pTask->level = pLevel;
   pTask->execId = -1;
-  pTask->maxExecTimes = SCH_TASK_MAX_EXEC_TIMES;
+  pTask->maxExecTimes = SCH_TASK_MAX_EXEC_TIMES(pLevel->level, levelNum);
   pTask->timeoutUsec = SCH_DEFAULT_TASK_TIMEOUT_USEC;
-  SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT);
   pTask->taskId = schGenTaskId();
   pTask->execNodes = taosHashInit(SCH_MAX_CANDIDATE_EP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
-  if (NULL == pTask->execNodes) {
-    SCH_TASK_ELOG("taosHashInit %d execNodes failed", SCH_MAX_CANDIDATE_EP_NUM);
-    SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
+  pTask->profile.execTime = taosMemoryCalloc(pTask->maxExecTimes, sizeof(int64_t));
+  if (NULL == pTask->execNodes || NULL == pTask->profile.execTime) {
+    SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
 
+  SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT);
+
   return TSDB_CODE_SUCCESS;
+
+_return:
+
+  taosMemoryFreeClear(pTask->profile.execTime);
+  taosHashCleanup(pTask->execNodes);
+
+  SCH_RET(code);
 }
 
 int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) {
@@ -338,6 +348,11 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf* pData, int32
   
   qClearSubplanExecutionNode(pTask->plan);
 
+  // Note: current error task and upper level merge task
+  if ((pData && 0 == pData->len) || NULL == pData) {
+    SCH_ERR_JRET(schSwitchTaskCandidateAddr(pJob, pTask));
+  }
+
   SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT);
   
   int32_t childrenNum = taosArrayGetSize(pTask->children);
@@ -531,10 +546,7 @@ int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) {
   if (SCH_IS_DATA_BIND_TASK(pTask)) {
     SCH_SWITCH_EPSET(&pTask->plan->execNode);
   } else {
-    int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs);  
-    if (++pTask->candidateIdx >= candidateNum) {
-      pTask->candidateIdx = 0;
-    }
+    SCH_ERR_RET(schSwitchTaskCandidateAddr(pJob, pTask));
   }
 
   SCH_ERR_RET(schLaunchTask(pJob, pTask));
@@ -633,6 +645,16 @@ int32_t schUpdateTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask, SEpSet* pEpSe
   return TSDB_CODE_SUCCESS;
 }
 
+int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) {
+  int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs);  
+  if (++pTask->candidateIdx >= candidateNum) {
+    pTask->candidateIdx = 0;
+  }
+  SCH_TASK_DLOG("switch task candiateIdx to %d", pTask->candidateIdx);
+  return TSDB_CODE_SUCCESS;
+}
+
+
 
 int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask) {
   int32_t code = taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId));

From 833c1ec487041299da35c9f25731ec91afb3d31c Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Sun, 10 Jul 2022 11:29:55 +0800
Subject: [PATCH 45/50] enh: rsma optimization for info item and tmr handle

---
 source/dnode/vnode/src/inc/sma.h       |   5 +-
 source/dnode/vnode/src/sma/smaEnv.c    |  33 ++-
 source/dnode/vnode/src/sma/smaRollup.c | 286 +++++++++++++------------
 3 files changed, 168 insertions(+), 156 deletions(-)

diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h
index 7183c423fe..d5b719dfb9 100644
--- a/source/dnode/vnode/src/inc/sma.h
+++ b/source/dnode/vnode/src/inc/sma.h
@@ -49,6 +49,7 @@ struct SSmaEnv {
 typedef struct {
   int8_t  inited;
   int32_t rsetId;
+  void   *tmrHandle;  // shared by all fetch tasks
 } SSmaMgmt;
 
 #define SMA_ENV_LOCK(env) ((env)->lock)
@@ -65,7 +66,6 @@ struct SRSmaStat {
   SSma     *pSma;
   int64_t   submitVer;
   int64_t   refId;         // shared by fetch tasks
-  void     *tmrHandle;     // shared by fetch tasks
   int8_t    triggerStat;   // shared by fetch tasks
   int8_t    runningStat;   // for persistence task 
   SHashObj *rsmaInfoHash;  // key: stbUid, value: SRSmaInfo;
@@ -82,7 +82,6 @@ struct SSmaStat {
 #define SMA_TSMA_STAT(s)     (&(s)->tsmaStat)
 #define SMA_RSMA_STAT(s)     (&(s)->rsmaStat)
 #define RSMA_INFO_HASH(r)    ((r)->rsmaInfoHash)
-#define RSMA_TMR_HANDLE(r)   ((r)->tmrHandle)
 #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
 #define RSMA_RUNNING_STAT(r) (&(r)->runningStat)
 #define RSMA_REF_ID(r)       ((r)->refId)
@@ -189,7 +188,7 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) {
 
 static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
 void          *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
-void          *tdFreeRSmaInfo(SRSmaInfo *pInfo);
+void          *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo);
 int32_t        tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat);
 
 int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c
index bb8dd48236..5eb9665326 100644
--- a/source/dnode/vnode/src/sma/smaEnv.c
+++ b/source/dnode/vnode/src/sma/smaEnv.c
@@ -49,16 +49,26 @@ int32_t smaInit() {
   }
 
   if (old == 0) {
+    // init tref rset
     smaMgmt.rsetId = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat);
 
     if (smaMgmt.rsetId < 0) {
-      smaError("failed to init sma rset since %s", terrstr());
       atomic_store_8(&smaMgmt.inited, 0);
+      smaError("failed to init sma rset since %s", terrstr());
+      return TSDB_CODE_FAILED;
+    }
+
+    // init fetch timer handle
+    smaMgmt.tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA");
+    if (!smaMgmt.tmrHandle) {
+      taosCloseRef(smaMgmt.rsetId);
+      atomic_store_8(&smaMgmt.inited, 0);
+      smaError("failed to init sma tmr hanle since %s", terrstr());
       return TSDB_CODE_FAILED;
     }
 
-    smaInfo("sma rset is initialized, rsetId:%d", smaMgmt.rsetId);
     atomic_store_8(&smaMgmt.inited, 1);
+    smaInfo("sma mgmt env is initialized, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -81,8 +91,9 @@ void smaCleanUp() {
   }
 
   if (old == 1) {
-    smaInfo("sma rset is cleaned up, resetId:%d", smaMgmt.rsetId);
     taosCloseRef(smaMgmt.rsetId);
+    taosTmrCleanUp(smaMgmt.tmrHandle);
+    smaInfo("sma mgmt env is cleaned up, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle);
     atomic_store_8(&smaMgmt.inited, 0);
   }
 }
@@ -203,20 +214,11 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
       }
       pRSmaStat->refId = refId;
 
-      // init timer
-      RSMA_TMR_HANDLE(pRSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA");
-      if (!RSMA_TMR_HANDLE(pRSmaStat)) {
-        taosMemoryFreeClear(*pSmaStat);
-        return TSDB_CODE_FAILED;
-      }
 
       // init hash
       RSMA_INFO_HASH(pRSmaStat) = taosHashInit(
           RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
       if (!RSMA_INFO_HASH(pRSmaStat)) {
-        if (RSMA_TMR_HANDLE(pRSmaStat)) {
-          taosTmrCleanUp(RSMA_TMR_HANDLE(pRSmaStat));
-        }
         taosMemoryFreeClear(*pSmaStat);
         return TSDB_CODE_FAILED;
       }
@@ -277,7 +279,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
       void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
       while (infoHash) {
         SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
-        tdFreeRSmaInfo(pSmaInfo);
+        tdFreeRSmaInfo(pSma, pSmaInfo);
         infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
       }
     }
@@ -298,11 +300,6 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
         nLoops = 0;
       }
     }
-
-    // step 6: cleanup the timer handle
-    if (RSMA_TMR_HANDLE(pStat)) {
-      taosTmrCleanUp(RSMA_TMR_HANDLE(pStat));
-    }
   }
 }
 
diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c
index c44a46ac5a..efa2886e48 100644
--- a/source/dnode/vnode/src/sma/smaRollup.c
+++ b/source/dnode/vnode/src/sma/smaRollup.c
@@ -27,16 +27,19 @@ SSmaMgmt smaMgmt = {
 typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
 typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
 
-static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
-static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
-static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle,
-                                       int8_t idx);
-static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem,
-                                 tb_uid_t suid, int8_t level);
-static void    tdRSmaFetchTrigger(void *param, void *tmrId);
-static void    tdRSmaPersistTrigger(void *param, void *tmrId);
-static void   *tdRSmaPersistExec(void *param);
-static void    tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);
+static int32_t    tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
+static int32_t    tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids);
+static int32_t    tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
+                                          SReadHandle *handle, int8_t idx);
+static int32_t    tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem,
+                                    STSchema *pTSchema, tb_uid_t suid, int8_t level);
+static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid);
+static int32_t    tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat,
+                                             int8_t blkType);
+static void       tdRSmaFetchTrigger(void *param, void *tmrId);
+static void       tdRSmaPersistTrigger(void *param, void *tmrId);
+static void      *tdRSmaPersistExec(void *param);
+static void       tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);
 
 static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile);
 static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish);
@@ -48,25 +51,26 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed);
 static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed);
 
 struct SRSmaInfoItem {
-  SRSmaInfo *pRsmaInfo;
-  int64_t    refId;
-  void      *taskInfo;  // qTaskInfo_t
-  tmr_h      tmrId;
-  int8_t     level;
-  int8_t     tmrInitFlag;
-  int8_t     triggerStat;
-  int32_t    maxDelay;
+  void   *taskInfo;  // qTaskInfo_t
+  int64_t refId;
+  tmr_h   tmrId;
+  int32_t maxDelay;
+  int8_t  level;
+  int8_t  triggerStat;
 };
 
 struct SRSmaInfo {
   STSchema     *pTSchema;
-  SRSmaStat    *pStat;
   int64_t       suid;
   SRSmaInfoItem items[TSDB_RETENTION_L2];
 };
 
-#define RSMA_INFO_SMA(r)  ((r)->pStat->pSma)
-#define RSMA_INFO_STAT(r) ((r)->pStat)
+static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) {
+  // adapt accordingly if definition of SRSmaInfo update
+  int32_t rsmaInfoHeadLen = sizeof(int64_t) + sizeof(STSchema *);
+  ASSERT(pItem->level == 1 || pItem->level == 2);
+  return (SRSmaInfo *)POINTER_SHIFT(pItem, -sizeof(SRSmaInfoItem) * (pItem->level - 1) - rsmaInfoHeadLen);
+}
 
 struct SRSmaQTaskInfoItem {
   int32_t len;
@@ -108,9 +112,8 @@ static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId,
   }
 }
 
-void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
+void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) {
   if (pInfo) {
-    SSma *pSma = RSMA_INFO_SMA(pInfo);
     for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
       if (pItem->taskInfo) {
@@ -143,8 +146,6 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) {
 }
 
 static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) {
-  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
   SRSmaInfo *pRSmaInfo = NULL;
 
   if (!suid || !tbUids) {
@@ -153,8 +154,9 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids)
     return TSDB_CODE_FAILED;
   }
 
-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t));
-  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+  pRSmaInfo = tdGetRSmaInfoBySuid(pSma, *suid);
+
+  if (!pRSmaInfo) {
     smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid);
     terrno = TSDB_CODE_RSMA_INVALID_STAT;
     return TSDB_CODE_FAILED;
@@ -252,15 +254,14 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *pReadHandle,
-                                       int8_t idx) {
+static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo,
+                                       SReadHandle *pReadHandle, int8_t idx) {
   SRetention *pRetention = SMA_RETENTION(pSma);
   STsdbCfg   *pTsdbCfg = SMA_TSDB_CFG(pSma);
 
   if (param->qmsg[idx]) {
     SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
-    pItem->refId = RSMA_REF_ID(pRSmaInfo->pStat);
-    pItem->pRsmaInfo = pRSmaInfo;
+    pItem->refId = RSMA_REF_ID(pStat);
     pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle);
     if (!pItem->taskInfo) {
       terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
@@ -348,14 +349,13 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
     goto _err;
   }
   pRSmaInfo->pTSchema = pTSchema;
-  pRSmaInfo->pStat = pStat;
   pRSmaInfo->suid = suid;
 
-  if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) {
+  if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, &handle, 0) < 0) {
     goto _err;
   }
 
-  if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 1) < 0) {
+  if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, &handle, 1) < 0) {
     goto _err;
   }
 
@@ -367,7 +367,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
 
   return TSDB_CODE_SUCCESS;
 _err:
-  tdFreeRSmaInfo(pRSmaInfo);
+  tdFreeRSmaInfo(pSma, pRSmaInfo);
   taosMemoryFree(pReader);
   return TSDB_CODE_FAILED;
 }
@@ -538,10 +538,10 @@ int64_t tdRSmaGetMaxSubmitVer(SSma *pSma, int8_t level) {
   return atomic_load_64(&pRSmaStat->submitVer);
 }
 
-static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) {
-  SArray    *pResult = NULL;
-  SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo;
-  SSma      *pSma = RSMA_INFO_SMA(pRSmaInfo);
+static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid, SRSmaStat *pStat,
+                                          int8_t blkType) {
+  SArray *pResult = NULL;
+  SSma   *pSma = pStat->pSma;
 
   while (1) {
     SSDataBlock *output = NULL;
@@ -573,16 +573,16 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType)
     STsdb      *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2);
     SSubmitReq *pReq = NULL;
     // TODO: the schema update should be handled
-    if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) {
+    if (buildSubmitReqFromDataBlock(&pReq, pResult, pTSchema, SMA_VID(pSma), suid) < 0) {
       smaError("vgId:%d, build submit req for rsma table %" PRIi64 "l evel %" PRIi8 " failed since %s", SMA_VID(pSma),
-               pRSmaInfo->suid, pItem->level, terrstr());
+               suid, pItem->level, terrstr());
       goto _err;
     }
 
-    if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pRSmaInfo->pStat->submitVer, 1), pReq) < 0) {
+    if (pReq && tdProcessSubmitReq(sinkTsdb, atomic_add_fetch_64(&pStat->submitVer, 1), pReq) < 0) {
       taosMemoryFreeClear(pReq);
       smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma),
-               pRSmaInfo->suid, pItem->level, terrstr());
+               suid, pItem->level, terrstr());
       goto _err;
     }
 
@@ -600,84 +600,16 @@ _err:
   return TSDB_CODE_FAILED;
 }
 
-/**
- * @brief trigger to get rsma result
- *
- * @param param
- * @param tmrId
- */
-static void tdRSmaFetchTrigger(void *param, void *tmrId) {
-  SRSmaInfoItem *pItem = param;
-  SSma          *pSma = NULL;
-  SRSmaStat     *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
-
-  if (!pStat) {
-    smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId,
-             pItem->refId);
-    return;
-  }
-
-  pSma = RSMA_INFO_SMA(pItem->pRsmaInfo);
-
-  // if rsma trigger stat in paused, cancelled or finished, not start fetch task
-  int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
-  switch (rsmaTriggerStat) {
-    case TASK_TRIGGER_STAT_PAUSED:
-    case TASK_TRIGGER_STAT_CANCELLED:
-    case TASK_TRIGGER_STAT_FINISHED: {
-      tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is %" PRIi8
-               ", rsetId rsetId:%" PRIi64 " refId:%d",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId);
-      return;
-    }
-    default:
-      break;
-  }
-
-  int8_t fetchTriggerStat =
-      atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
-  switch (fetchTriggerStat) {
-    case TASK_TRIGGER_STAT_ACTIVE: {
-      smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
-               pItem->level, pItem->pRsmaInfo->suid);
-
-      tdRefSmaStat(pSma, (SSmaStat *)pStat);
-
-      SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
-      qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
-      tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
-
-      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
-    } break;
-    case TASK_TRIGGER_STAT_PAUSED: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    case TASK_TRIGGER_STAT_INACTIVE: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
-               SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    case TASK_TRIGGER_STAT_INIT: {
-      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
-               pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-    default: {
-      smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
-              SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
-    } break;
-  }
-
-_end:
-  tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
-}
-
-static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid,
-                                 int8_t level) {
+static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem,
+                                 STSchema *pTSchema, tb_uid_t suid, int8_t level) {
   if (!pItem || !pItem->taskInfo) {
     smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
     return TSDB_CODE_SUCCESS;
   }
+  if (!pTSchema) {
+    smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, suid);
+    return TSDB_CODE_FAILED;
+  }
 
   smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level,
            pItem->taskInfo, suid);
@@ -687,14 +619,14 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
     return TSDB_CODE_FAILED;
   }
 
-  tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_SUBMIT);
-  atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
-
   SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
   SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat);
 
-  if (pStat->tmrHandle) {
-    taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pStat->tmrHandle, &pItem->tmrId);
+  tdRSmaFetchAndSubmitResult(pItem, pTSchema, suid, pStat, STREAM_INPUT__DATA_SUBMIT);
+  atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE);
+
+  if (smaMgmt.tmrHandle) {
+    taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId);
   } else {
     ASSERT(0);
   }
@@ -702,19 +634,29 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
-  SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
+static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) {
+  SSmaEnv   *pEnv = SMA_RSMA_ENV(pSma);
+  SRSmaStat *pStat = NULL;
   if (!pEnv) {
     // only applicable when rsma env exists
-    return TSDB_CODE_SUCCESS;
+    return NULL;
   }
 
-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
-  SRSmaInfo *pRSmaInfo = NULL;
-
-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
+  pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
+  if (!pStat || !RSMA_INFO_HASH(pStat)) {
+    return NULL;
+  }
 
+  SRSmaInfo *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t));
   if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+    return NULL;
+  }
+  return pRSmaInfo;
+}
+
+static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) {
+  SRSmaInfo *pRSmaInfo = tdGetRSmaInfoBySuid(pSma, suid);
+  if (!pRSmaInfo) {
     smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid);
     return TSDB_CODE_SUCCESS;
   }
@@ -725,8 +667,8 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb
   }
 
   if (inputType == STREAM_INPUT__DATA_SUBMIT) {
-    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], suid, TSDB_RETENTION_L1);
-    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], suid, TSDB_RETENTION_L2);
+    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L1);
+    tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], pRSmaInfo->pTSchema, suid, TSDB_RETENTION_L2);
   }
 
   return TSDB_CODE_SUCCESS;
@@ -939,13 +881,11 @@ _err:
 }
 
 static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) {
-  SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv);
   SRSmaInfo *pRSmaInfo = NULL;
   void      *qTaskInfo = NULL;
 
-  pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &pItem->suid, sizeof(pItem->suid));
-
-  if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) {
+  pRSmaInfo = tdGetRSmaInfoBySuid(pSma, pItem->suid);
+  if (!pRSmaInfo) {
     smaDebug("vgId:%d, no restore as no rsma info for table:%" PRIu64, SMA_VID(pSma), pItem->suid);
     return TSDB_CODE_SUCCESS;
   }
@@ -1350,3 +1290,79 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
   }
   taosReleaseRef(smaMgmt.rsetId, rsmaStat->refId);
 }
+
+/**
+ * @brief trigger to get rsma result
+ *
+ * @param param
+ * @param tmrId
+ */
+static void tdRSmaFetchTrigger(void *param, void *tmrId) {
+  SRSmaInfoItem *pItem = param;
+  SSma          *pSma = NULL;
+  SRSmaStat     *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
+
+  if (!pStat) {
+    smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId,
+             pItem->refId);
+    return;
+  }
+
+  pSma = pStat->pSma;
+
+  // if rsma trigger stat in paused, cancelled or finished, not start fetch task
+  int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
+  switch (rsmaTriggerStat) {
+    case TASK_TRIGGER_STAT_PAUSED:
+    case TASK_TRIGGER_STAT_CANCELLED:
+    case TASK_TRIGGER_STAT_FINISHED: {
+      tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64
+               " refId:%d",
+               SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId);
+      return;
+    }
+    default:
+      break;
+  }
+
+  SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem);
+
+  ASSERT(pRSmaInfo->suid > 0);
+
+  int8_t fetchTriggerStat =
+      atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
+  switch (fetchTriggerStat) {
+    case TASK_TRIGGER_STAT_ACTIVE: {
+      smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
+               pItem->level, pRSmaInfo->suid);
+
+      tdRefSmaStat(pSma, (SSmaStat *)pStat);
+
+      SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
+      qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
+      tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK);
+
+      tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
+    } break;
+    case TASK_TRIGGER_STAT_PAUSED: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
+               SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INACTIVE: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
+               SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+    case TASK_TRIGGER_STAT_INIT: {
+      smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
+               pItem->level, pRSmaInfo->suid);
+    } break;
+    default: {
+      smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
+              SMA_VID(pSma), pItem->level, pRSmaInfo->suid);
+    } break;
+  }
+
+_end:
+  tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__);
+}

From 32c3cecf2abb11c782db49862c66dd50ba4d06f3 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Sun, 10 Jul 2022 11:33:28 +0800
Subject: [PATCH 46/50] refactor(sync): add snapshot writer

---
 source/dnode/vnode/src/vnd/vnodeSync.c | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index a2f5fa6dbb..8d0ae4785b 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -469,11 +469,24 @@ static int32_t vnodeSnapshotDoRead(struct SSyncFSM *pFsm, void *pReader, void **
   return code;
 }
 
-static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void **ppWriter) { return 0; }
+static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void **ppWriter) {
+  SVnode         *pVnode = pFsm->data;
+  SSnapshotParam *pSnapshotParam = pParam;
+  int32_t code = vnodeSnapWriterOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapWriter **)ppWriter);
+  return code;
+}
 
-static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { return 0; }
+static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) {
+  SVnode *pVnode = pFsm->data;
+  int32_t code = vnodeSnapWriterClose(pWriter, isApply);
+  return code;
+}
 
-static int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) { return 0; }
+static int32_t vnodeSnapshotDoWrite(struct SSyncFSM *pFsm, void *pWriter, void *pBuf, int32_t len) {
+  SVnode *pVnode = pFsm->data;
+  int32_t code = vnodeSnapWrite(pWriter, pBuf, len);
+  return code;
+}
 
 static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
   SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM));

From 0c0ec88e5c2ce98f49bca68813a6bbc5f3488663 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 10 Jul 2022 11:35:52 +0800
Subject: [PATCH 47/50] fix: make sdbDump work

---
 tests/test/c/sdbDump.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/test/c/sdbDump.c b/tests/test/c/sdbDump.c
index a62c30660d..42f1bb4062 100644
--- a/tests/test/c/sdbDump.c
+++ b/tests/test/c/sdbDump.c
@@ -24,6 +24,7 @@
 #define TMP_MNODE_DIR           TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode"
 #define TMP_SDB_DATA_DIR        TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "data"
 #define TMP_SDB_SYNC_DIR        TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync"
+#define TMP_SDB_MNODE_JSON      TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "mnode.json"
 #define TMP_SDB_DATA_FILE       TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "data" TD_DIRSEP "sdb.data"
 #define TMP_SDB_RAFT_CFG_FILE   TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_config.json"
 #define TMP_SDB_RAFT_STORE_FILE TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_store.json"
@@ -412,9 +413,11 @@ int32_t parseArgs(int32_t argc, char *argv[]) {
     return -1;
   }
 
+  char mnodeJson[PATH_MAX] = {0};
   char dataFile[PATH_MAX] = {0};
   char raftCfgFile[PATH_MAX] = {0};
   char raftStoreFile[PATH_MAX] = {0};
+  snprintf(mnodeJson, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "mnode.json", tsDataDir);
   snprintf(dataFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "data" TD_DIRSEP "sdb.data", tsDataDir);
   snprintf(raftCfgFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_config.json", tsDataDir);
   snprintf(raftStoreFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_store.json", tsDataDir);
@@ -425,6 +428,8 @@ int32_t parseArgs(int32_t argc, char *argv[]) {
 #ifdef WINDOWS
   taosMulMkDir(TMP_SDB_DATA_DIR);
   taosMulMkDir(TMP_SDB_SYNC_DIR);
+  snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", mnodeJson, TMP_SDB_MNODE_JSON);
+  system(cmd);
   snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", dataFile, TMP_SDB_DATA_FILE);
   system(cmd);
   snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", raftCfgFile, TMP_SDB_RAFT_CFG_FILE);
@@ -436,6 +441,8 @@ int32_t parseArgs(int32_t argc, char *argv[]) {
   system(cmd);
   snprintf(cmd, sizeof(cmd), "mkdir -p %s", TMP_SDB_SYNC_DIR);
   system(cmd);
+  snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", mnodeJson, TMP_SDB_MNODE_JSON);
+  system(cmd);
   snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", dataFile, TMP_SDB_DATA_FILE);
   system(cmd);
   snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", raftCfgFile, TMP_SDB_RAFT_CFG_FILE);

From 94f71012eab521df8140dfcde49c1e1abfe9561d Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 10 Jul 2022 11:49:04 +0800
Subject: [PATCH 48/50] fix: compile error

---
 source/dnode/mnode/impl/test/stb/stb.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp
index 56b8936cf4..42b3f54f82 100644
--- a/source/dnode/mnode/impl/test/stb/stb.cpp
+++ b/source/dnode/mnode/impl/test/stb/stb.cpp
@@ -277,8 +277,6 @@ void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const c
   req.numOfFields = 1;
   req.pFields = taosArrayInit(1, sizeof(SField));
   req.alterType = TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES;
-  req.tagVer = verInBlock;
-  req.colVer = verInBlock;
 
   SField field = {0};
   field.bytes = bytes;

From b4cadcbef2065cbcb7cf207a4a97718f047aee93 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sun, 10 Jul 2022 11:50:56 +0800
Subject: [PATCH 49/50] fix: unitest of sdb

---
 source/dnode/mnode/impl/test/stb/stb.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp
index 42b3f54f82..63bb1bf540 100644
--- a/source/dnode/mnode/impl/test/stb/stb.cpp
+++ b/source/dnode/mnode/impl/test/stb/stb.cpp
@@ -816,7 +816,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
   {
     void*    pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col_not_exist", 20, &contLen, 1);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
-    ASSERT_EQ(pRsp->code, 0);
+    ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST);
 
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);

From 8afb54548fb15a8ce7ddf9f21da96ad78e6e902b Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Sun, 10 Jul 2022 13:21:37 +0800
Subject: [PATCH 50/50] refactor(stream)

---
 source/dnode/vnode/src/tq/tqRead.c      |  6 +--
 source/libs/executor/src/scanoperator.c | 49 ++++++++++++++-----------
 source/libs/wal/src/walRead.c           | 21 +++++------
 3 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c
index 3e5bd0ce57..8753ecc47c 100644
--- a/source/dnode/vnode/src/tq/tqRead.c
+++ b/source/dnode/vnode/src/tq/tqRead.c
@@ -22,8 +22,8 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea
 
   while (1) {
     if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) {
-      tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return", pHandle->consumerId,
-              pHandle->epoch, TD_VID(pTq->pVnode), offset);
+      tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return",
+              pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode), offset);
       *fetchOffset = offset - 1;
       code = -1;
       goto END;
@@ -119,7 +119,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
   while (1) {
     if (!fromProcessedMsg) {
       if (walNextValidMsg(pReader->pWalReader) < 0) {
-        pReader->ver = pReader->pWalReader->curVersion;
+        pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curInvalid;
         ret->offset.type = TMQ_OFFSET__LOG;
         ret->offset.version = pReader->ver;
         ret->fetchType = FETCH_TYPE__NONE;
diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 9427ddba92..d2f63849d0 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -40,8 +40,8 @@ static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta*
                                      const char* dbName);
 
 static int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr,
-                               SSDataBlock* pBlock, const char* idStr);
-static bool processBlockWithProbability(const SSampleExecInfo* pInfo);
+                                      SSDataBlock* pBlock, const char* idStr);
+static bool    processBlockWithProbability(const SSampleExecInfo* pInfo);
 
 bool processBlockWithProbability(const SSampleExecInfo* pInfo) {
 #if 0
@@ -265,7 +265,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
   if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
     SExprSupp* pSup = &pTableScanInfo->pseudoSup;
 
-    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, GET_TASKID(pTaskInfo));
+    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock,
+                                          GET_TASKID(pTaskInfo));
     if (code != TSDB_CODE_SUCCESS) {
       longjmp(pTaskInfo->env, code);
     }
@@ -303,7 +304,7 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction
 }
 
 int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr,
-                            SSDataBlock* pBlock, const char* idStr) {
+                               SSDataBlock* pBlock, const char* idStr) {
   // currently only the tbname pseudo column
   if (numOfPseudoExpr == 0) {
     return TSDB_CODE_SUCCESS;
@@ -313,7 +314,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int
   metaReaderInit(&mr, pHandle->meta, 0);
   int32_t code = metaGetTableEntryByUid(&mr, pBlock->info.uid);
   if (code != TSDB_CODE_SUCCESS) {
-    qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr);
+    qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr);
     metaReaderClear(&mr);
     return terrno;
   }
@@ -697,7 +698,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con
   metaReaderInit(&mr, pMeta, 0);
   int32_t code = metaGetTableEntryByUid(&mr, uid);
   if (code != TSDB_CODE_SUCCESS) {
-    qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr);
+    qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr);
     metaReaderClear(&mr);
     return terrno;
   }
@@ -711,7 +712,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con
     uint64_t suid = mr.me.ctbEntry.suid;
     code = metaGetTableEntryByUid(&mr, suid);
     if (code != TSDB_CODE_SUCCESS) {
-      qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr);
+      qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr);
       metaReaderClear(&mr);
       return terrno;
     }
@@ -738,12 +739,13 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
   }
 
   SBlockDistInfo* pBlockScanInfo = pOperator->info;
-  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
 
   STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN};
-  int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, &blockDistInfo.rowSize, GET_TASKID(pTaskInfo));
+  int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, &blockDistInfo.rowSize,
+                                   GET_TASKID(pTaskInfo));
   if (code != TSDB_CODE_SUCCESS) {
-   longjmp(pTaskInfo->env, code);
+    longjmp(pTaskInfo->env, code);
   }
 
   tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo);
@@ -938,7 +940,8 @@ static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t t
       setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex);
       (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, *pRowIndex, gap, NULL);
     } else {
-      win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, TSDB_ORDER_ASC);
+      win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision,
+                                TSDB_ORDER_ASC);
       setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex);
       (*pRowIndex) +=
           getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
@@ -1219,7 +1222,8 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
 
   // currently only the tbname pseudo column
   if (pInfo->numOfPseudoExpr > 0) {
-    int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, GET_TASKID(pTaskInfo));
+    int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes,
+                                          GET_TASKID(pTaskInfo));
     if (code != TSDB_CODE_SUCCESS) {
       longjmp(pTaskInfo->env, code);
     }
@@ -1269,7 +1273,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
         /*pTaskInfo->streamInfo.lastStatus.version = pTaskInfo->streamInfo.prepareStatus.version - 1;*/
         /*} else {*/
         pTaskInfo->streamInfo.lastStatus = ret.offset;
-        ASSERT(pTaskInfo->streamInfo.lastStatus.version >= pTaskInfo->streamInfo.prepareStatus.version);
+        ASSERT(pTaskInfo->streamInfo.lastStatus.version + 1 >= pTaskInfo->streamInfo.prepareStatus.version);
         /*}*/
         return NULL;
       } else {
@@ -1447,7 +1451,8 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
 
       // currently only the tbname pseudo column
       if (pInfo->numOfPseudoExpr > 0) {
-        code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, GET_TASKID(pTaskInfo));
+        code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes,
+                                      GET_TASKID(pTaskInfo));
         if (code != TSDB_CODE_SUCCESS) {
           longjmp(pTaskInfo->env, code);
         }
@@ -1485,7 +1490,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
     return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes;
 
   } else if (pInfo->blockType == STREAM_INPUT__TABLE_SCAN) {
-    ASSERT(0);
+    /*ASSERT(0);*/
     // check reader last status
     // if not match, reset status
     SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);
@@ -1878,9 +1883,10 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
           metaReaderInit(&mr, pInfo->readHandle.meta, 0);
 
           uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
-          int32_t code = metaGetTableEntryByUid(&mr, suid);
+          int32_t  code = metaGetTableEntryByUid(&mr, suid);
           if (code != TSDB_CODE_SUCCESS) {
-            qError("failed to get super table meta, uid:0x%"PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo));
+            qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno),
+                   GET_TASKID(pTaskInfo));
             metaReaderClear(&mr);
             metaCloseTbCursor(pInfo->pCur);
             pInfo->pCur = NULL;
@@ -2280,9 +2286,10 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
 
   while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) {
     STableKeyInfo* item = taosArrayGet(pInfo->pTableList->pTableList, pInfo->curPos);
-    int32_t code = metaGetTableEntryByUid(&mr, item->uid);
+    int32_t        code = metaGetTableEntryByUid(&mr, item->uid);
     if (code != TSDB_CODE_SUCCESS) {
-      qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo));
+      qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno),
+             GET_TASKID(pTaskInfo));
       metaReaderClear(&mr);
       longjmp(pTaskInfo->env, terrno);
     }
@@ -2570,8 +2577,8 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
 
   // currently only the tbname pseudo column
   if (pTableScanInfo->numOfPseudoExpr > 0) {
-    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pPseudoExpr, pTableScanInfo->numOfPseudoExpr,
-                           pBlock, GET_TASKID(pTaskInfo));
+    int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pPseudoExpr,
+                                          pTableScanInfo->numOfPseudoExpr, pBlock, GET_TASKID(pTaskInfo));
     if (code != TSDB_CODE_SUCCESS) {
       longjmp(pTaskInfo->env, code);
     }
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index 742cbad5fe..2d910a85b8 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -162,8 +162,8 @@ int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) {
   pRead->curVersion = ver;
 
   if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
-    wError("vgId:$d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pRead->pWal->cfg.vgId, ver,
-           pWal->vers.firstVer, pWal->vers.lastVer);
+    wDebug("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pRead->pWal->cfg.vgId,
+           ver, pWal->vers.firstVer, pWal->vers.lastVer);
     terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
     return -1;
   }
@@ -176,13 +176,13 @@ int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) {
   SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
   ASSERT(pRet != NULL);
   if (pRead->curFileFirstVer != pRet->firstVer) {
-    // error code set inner
+    // error code was set inner
     if (walReadChangeFile(pRead, pRet->firstVer) < 0) {
       return -1;
     }
   }
 
-  // error code set inner
+  // error code was set inner
   if (walReadSeekFilePos(pRead, pRet->firstVer, ver) < 0) {
     return -1;
   }
@@ -314,8 +314,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
   code = walValidHeadCksum(pHead);
 
   if (code != 0) {
-    wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed", pRead->pWal->cfg.vgId,
-           ver);
+    wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed", pRead->pWal->cfg.vgId, ver);
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;
   }
@@ -395,8 +394,8 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) {
   }
 
   if (ver > pRead->pWal->vers.lastVer || ver < pRead->pWal->vers.firstVer) {
-    wError("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pRead->pWal->cfg.vgId, ver,
-           pRead->pWal->vers.firstVer, pRead->pWal->vers.lastVer);
+    wError("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pRead->pWal->cfg.vgId,
+           ver, pRead->pWal->vers.firstVer, pRead->pWal->vers.lastVer);
     terrno = TSDB_CODE_WAL_LOG_NOT_EXIST;
     return -1;
   }
@@ -416,8 +415,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) {
 
   code = walValidHeadCksum(pRead->pHead);
   if (code != 0) {
-    wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed", pRead->pWal->cfg.vgId,
-           ver);
+    wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed", pRead->pWal->cfg.vgId, ver);
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;
   }
@@ -453,8 +451,7 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) {
 
   code = walValidBodyCksum(pRead->pHead);
   if (code != 0) {
-    wError("vgId:%d, unexpected wal log index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId,
-           ver);
+    wError("vgId:%d, unexpected wal log index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver);
     pRead->curInvalid = 1;
     terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;