From 1e3fff30e73098b8725b73e47829f36e88010c69 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 12 Jul 2022 12:16:40 +0000
Subject: [PATCH 01/38] more vnode snapshot

---
 source/dnode/vnode/src/inc/vnodeInt.h      |  1 +
 source/dnode/vnode/src/meta/metaSnapshot.c | 67 ++++++++++-----
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 95 ++++++++++++++--------
 3 files changed, 109 insertions(+), 54 deletions(-)

diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 7b298ba830..d6b0dd48e8 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -309,6 +309,7 @@ void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 
 struct SSnapDataHdr {
   int8_t  type;
+  int64_t index;
   int64_t size;
   uint8_t data[];
 };
diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index ac84842e85..900416f1b8 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -26,60 +26,72 @@ struct SMetaSnapReader {
 int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) {
   int32_t          code = 0;
   int32_t          c = 0;
-  SMetaSnapReader* pMetaSnapReader = NULL;
+  SMetaSnapReader* pReader = NULL;
 
   // alloc
-  pMetaSnapReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pMetaSnapReader));
-  if (pMetaSnapReader == NULL) {
+  pReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader));
+  if (pReader == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
-  pMetaSnapReader->pMeta = pMeta;
-  pMetaSnapReader->sver = sver;
-  pMetaSnapReader->ever = ever;
+  pReader->pMeta = pMeta;
+  pReader->sver = sver;
+  pReader->ever = ever;
 
   // impl
-  code = tdbTbcOpen(pMeta->pTbDb, &pMetaSnapReader->pTbc, NULL);
+  code = tdbTbcOpen(pMeta->pTbDb, &pReader->pTbc, NULL);
   if (code) {
+    taosMemoryFree(pReader);
     goto _err;
   }
 
-  code = tdbTbcMoveTo(pMetaSnapReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
+  code = tdbTbcMoveTo(pReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c);
   if (code) {
+    taosMemoryFree(pReader);
     goto _err;
   }
 
-  *ppReader = pMetaSnapReader;
+  metaInfo("vgId:%d vnode snapshot meta reader opened", TD_VID(pMeta->pVnode));
+
+  *ppReader = pReader;
   return code;
 
 _err:
-  metaError("vgId:%d meta snap reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
+  metaError("vgId:%d vnode snapshot meta reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
   *ppReader = NULL;
   return code;
 }
 
 int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) {
+  int32_t code = 0;
+
   tdbTbcClose((*ppReader)->pTbc);
   taosMemoryFree(*ppReader);
   *ppReader = NULL;
-  return 0;
+
+  return code;
 }
 
 int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
+  int32_t     code = 0;
   const void* pKey = NULL;
   const void* pData = NULL;
   int32_t     nKey = 0;
   int32_t     nData = 0;
-  int32_t     code = 0;
+  STbDbKey    key;
 
+  *ppData = NULL;
   for (;;) {
-    code = tdbTbcGet(pReader->pTbc, &pKey, &nKey, &pData, &nData);
-    if (code || ((STbDbKey*)pData)->version > pReader->ever) {
-      code = TSDB_CODE_VND_READ_END;
+    if (tdbTbcGet(pReader->pTbc, &pKey, &nKey, &pData, &nData)) {
       goto _exit;
     }
 
-    if (((STbDbKey*)pData)->version < pReader->sver) {
+    key = ((STbDbKey*)pKey)[0];
+    if (key.version > pReader->ever) {
+      goto _exit;
+    }
+
+    if (key.version < pReader->sver) {
       tdbTbcMoveToNext(pReader->pTbc);
       continue;
     }
@@ -88,17 +100,28 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) {
     break;
   }
 
-  // copy the data
-  if (tRealloc(ppData, sizeof(SSnapDataHdr) + nData) < 0) {
+  ASSERT(pData && nData);
+
+  *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData);
+  if (*ppData == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
-    return code;
+    goto _err;
   }
-  ((SSnapDataHdr*)(*ppData))->type = 0;  // TODO: use macro
-  ((SSnapDataHdr*)(*ppData))->size = nData;
-  memcpy(((SSnapDataHdr*)(*ppData))->data, pData, nData);
+
+  SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
+  pHdr->type = 0;  // TODO: use macro
+  pHdr->size = nData;
+  memcpy(pHdr->data, pData, nData);
+
+  metaInfo("vgId:%d vnode snapshot meta read data, version:%" PRId64 " uid:%" PRId64 " nData:%d",
+           TD_VID(pReader->pMeta->pVnode), key.version, key.uid, nData);
 
 _exit:
   return code;
+
+_err:
+  metaError("vgId:%d vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode), tstrerror(code));
+  return code;
 }
 
 // SMetaSnapWriter ========================================
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 27f30ec787..60801832a9 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -20,13 +20,13 @@ struct SVSnapReader {
   SVnode *pVnode;
   int64_t sver;
   int64_t ever;
+  int64_t index;
   // meta
   int8_t           metaDone;
   SMetaSnapReader *pMetaReader;
   // tsdb
   int8_t           tsdbDone;
   STsdbSnapReader *pTsdbReader;
-  uint8_t         *pData;
 };
 
 int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) {
@@ -42,12 +42,7 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapRe
   pReader->sver = sver;
   pReader->ever = ever;
 
-  code = metaSnapReaderOpen(pVnode->pMeta, sver, ever, &pReader->pMetaReader);
-  if (code) goto _err;
-
-  code = tsdbSnapReaderOpen(pVnode->pTsdb, sver, ever, &pReader->pTsdbReader);
-  if (code) goto _err;
-
+  vInfo("vgId:%d vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever);
   *ppReader = pReader;
   return code;
 
@@ -60,50 +55,75 @@ _err:
 int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
   int32_t code = 0;
 
-  tFree(pReader->pData);
-  if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader);
-  if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader);
-  taosMemoryFree(pReader);
+  // tFree(pReader->pData);
+  // if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader);
+  // if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader);
+  // taosMemoryFree(pReader);
 
+  vInfo("vgId:%d vnode snapshot reader closed", TD_VID(pReader->pVnode));
   return code;
 }
 
 int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) {
   int32_t code = 0;
 
+  // META ==============
   if (!pReader->metaDone) {
-    code = metaSnapRead(pReader->pMetaReader, &pReader->pData);
+    // open reader if not
+    if (pReader->pMetaReader == NULL) {
+      code = metaSnapReaderOpen(pReader->pVnode->pMeta, pReader->sver, pReader->ever, &pReader->pMetaReader);
+      if (code) goto _err;
+    }
+
+    code = metaSnapRead(pReader->pMetaReader, ppData);
     if (code) {
-      if (code == TSDB_CODE_VND_READ_END) {
+      goto _err;
+    } else {
+      if (*ppData) {
+        goto _exit;
+      } else {
         pReader->metaDone = 1;
-      } else {
-        goto _err;
+        code = metaSnapReaderClose(&pReader->pMetaReader);
+        if (code) goto _err;
+
+        vInfo("vgId:%d vnode snapshot meta data read end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index);
       }
-    } else {
-      *ppData = pReader->pData;
-      *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
-      goto _exit;
     }
   }
 
+  // TSDB ==============
   if (!pReader->tsdbDone) {
-    code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData);
-    if (code) {
-      if (code == TSDB_CODE_VND_READ_END) {
-        pReader->tsdbDone = 1;
-      } else {
-        goto _err;
-      }
-    } else {
-      *ppData = pReader->pData;
-      *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
-      goto _exit;
-    }
+    // open if not
+    // if (pReader->pTsdbReader == NULL) {
+    //   code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, &pReader->pTsdbReader);
+    //   if (code) goto _err;
+    // }
+
+    // code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData);
+    // if (code) {
+    //   if (code == TSDB_CODE_VND_READ_END) {
+    //     pReader->tsdbDone = 1;
+    //   } else {
+    //     goto _err;
+    //   }
+    // } else {
+    //   *ppData = pReader->pData;
+    //   *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
+    //   goto _exit;
+    // }
   }
 
-  code = TSDB_CODE_VND_READ_END;
+  *ppData = NULL;
+  *nData = 0;
 
 _exit:
+  if (*ppData) {
+    pReader->index++;
+    ((SSnapDataHdr *)(*ppData))->index = pReader->index;
+    vInfo("vgId:%d vnode snapshot read data, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index);
+  } else {
+    vInfo("vgId:%d vnode snapshot read data end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index);
+  }
   return code;
 
 _err:
@@ -116,6 +136,7 @@ struct SVSnapWriter {
   SVnode *pVnode;
   int64_t sver;
   int64_t ever;
+  int64_t index;
   // meta
   SMetaSnapWriter *pMetaSnapWriter;
   // tsdb
@@ -148,16 +169,22 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr
   pWriter->sver = sver;
   pWriter->ever = ever;
 
+  vInfo("vgId:%d vnode snapshot writer opened", TD_VID(pVnode));
+
+  *ppWriter = pWriter;
   return code;
 
 _err:
   vError("vgId:%d vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code));
+  *ppWriter = NULL;
   return code;
 }
 
 int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
   int32_t code = 0;
 
+  goto _exit;
+
   if (rollback) {
     code = vnodeSnapRollback(pWriter);
     if (code) goto _err;
@@ -166,6 +193,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
     if (code) goto _err;
   }
 
+_exit:
   taosMemoryFree(pWriter);
   return code;
 
@@ -179,6 +207,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
   SSnapDataHdr *pSnapDataHdr = (SSnapDataHdr *)pData;
   SVnode       *pVnode = pWriter->pVnode;
 
+  goto _exit;
+
   ASSERT(pSnapDataHdr->size + sizeof(SSnapDataHdr) == nData);
 
   if (pSnapDataHdr->type == 0) {
@@ -201,6 +231,7 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
     if (code) goto _err;
   }
 
+_exit:
   return code;
 
 _err:

From 98cb6ae092219c66580f098b2bc317adf22fe24c Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Tue, 12 Jul 2022 12:46:27 +0000
Subject: [PATCH 02/38] more vnode snapshot

---
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 54 ++++++++++++----------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 60801832a9..0163ea0449 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -58,9 +58,9 @@ int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
   // tFree(pReader->pData);
   // if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader);
   // if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader);
-  // taosMemoryFree(pReader);
 
   vInfo("vgId:%d vnode snapshot reader closed", TD_VID(pReader->pVnode));
+  taosMemoryFree(pReader);
   return code;
 }
 
@@ -118,9 +118,12 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
 
 _exit:
   if (*ppData) {
+    SSnapDataHdr *pHdr = (SSnapDataHdr *)(*ppData);
+
     pReader->index++;
-    ((SSnapDataHdr *)(*ppData))->index = pReader->index;
-    vInfo("vgId:%d vnode snapshot read data, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index);
+    *nData = sizeof(SSnapDataHdr) + pHdr->size;
+    pHdr->index = pReader->index;
+    vInfo("vgId:%d vnode snapshot read data, nData:%d index:%" PRId64, TD_VID(pReader->pVnode), *nData, pReader->index);
   } else {
     vInfo("vgId:%d vnode snapshot read data end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index);
   }
@@ -194,44 +197,45 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
   }
 
 _exit:
+  vInfo("vgId:%d vnode snapshot writer closed, rollback:%d", TD_VID(pWriter->pVnode), rollback);
   taosMemoryFree(pWriter);
   return code;
 
 _err:
-  vError("vgId:%d vnode snapshow writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code));
+  vError("vgId:%d vnode snapshot writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code));
   return code;
 }
 
 int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
   int32_t       code = 0;
-  SSnapDataHdr *pSnapDataHdr = (SSnapDataHdr *)pData;
+  SSnapDataHdr *pHdr = (SSnapDataHdr *)pData;
   SVnode       *pVnode = pWriter->pVnode;
 
-  goto _exit;
+  // ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData);
 
-  ASSERT(pSnapDataHdr->size + sizeof(SSnapDataHdr) == nData);
+  // if (pHdr->type == 0) {
+  //   // meta
+  //   if (pWriter->pMetaSnapWriter == NULL) {
+  //     code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
+  //     if (code) goto _err;
+  //   }
 
-  if (pSnapDataHdr->type == 0) {
-    // meta
-    if (pWriter->pMetaSnapWriter == NULL) {
-      code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
-      if (code) goto _err;
-    }
+  //   code = metaSnapWrite(pWriter->pMetaSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
+  //   if (code) goto _err;
+  // } else {
+  //   // tsdb
+  //   if (pWriter->pTsdbSnapWriter == NULL) {
+  //     code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter);
+  //     if (code) goto _err;
+  //   }
 
-    code = metaSnapWrite(pWriter->pMetaSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
-    if (code) goto _err;
-  } else {
-    // tsdb
-    if (pWriter->pTsdbSnapWriter == NULL) {
-      code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter);
-      if (code) goto _err;
-    }
-
-    code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
-    if (code) goto _err;
-  }
+  //   code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
+  //   if (code) goto _err;
+  // }
 
 _exit:
+  vInfo("vgId:%d vnode snapshot write data, type:%d index:%" PRId64 " nData:%d", TD_VID(pVnode), pHdr->type,
+        pHdr->index, nData);
   return code;
 
 _err:

From f977a50e9b3cb9569022d3008f9c489060dc2c32 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 13 Jul 2022 01:11:55 +0000
Subject: [PATCH 03/38] more vnode snapshot

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c |  3 +-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 46 +++++++++++-----------
 2 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 06d4a86116..b02e1dabd3 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -232,11 +232,12 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe
     goto _err;
   }
 
+  tsdbInfo("vgId:%d vnode snapshot tsdb reader opened", TD_VID(pTsdb->pVnode));
   *ppReader = pReader;
   return code;
 
 _err:
-  tsdbError("vgId:%d snapshot reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d vnode snapshot tsdb reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   *ppReader = NULL;
   return code;
 }
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 0163ea0449..0e14df0380 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -55,9 +55,13 @@ _err:
 int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
   int32_t code = 0;
 
-  // tFree(pReader->pData);
-  // if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader);
-  // if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader);
+  if (pReader->pTsdbReader) {
+    tsdbSnapReaderClose(&pReader->pTsdbReader);
+  }
+
+  if (pReader->pMetaReader) {
+    metaSnapReaderClose(&pReader->pMetaReader);
+  }
 
   vInfo("vgId:%d vnode snapshot reader closed", TD_VID(pReader->pVnode));
   taosMemoryFree(pReader);
@@ -85,8 +89,6 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
         pReader->metaDone = 1;
         code = metaSnapReaderClose(&pReader->pMetaReader);
         if (code) goto _err;
-
-        vInfo("vgId:%d vnode snapshot meta data read end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index);
       }
     }
   }
@@ -94,23 +96,23 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
   // TSDB ==============
   if (!pReader->tsdbDone) {
     // open if not
-    // if (pReader->pTsdbReader == NULL) {
-    //   code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, &pReader->pTsdbReader);
-    //   if (code) goto _err;
-    // }
+    if (pReader->pTsdbReader == NULL) {
+      code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, &pReader->pTsdbReader);
+      if (code) goto _err;
+    }
 
-    // code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData);
-    // if (code) {
-    //   if (code == TSDB_CODE_VND_READ_END) {
-    //     pReader->tsdbDone = 1;
-    //   } else {
-    //     goto _err;
-    //   }
-    // } else {
-    //   *ppData = pReader->pData;
-    //   *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size;
-    //   goto _exit;
-    // }
+    code = tsdbSnapRead(pReader->pTsdbReader, ppData);
+    if (code) {
+      goto _err;
+    } else {
+      if (*ppData) {
+        goto _exit;
+      } else {
+        pReader->tsdbDone = 1;
+        code = tsdbSnapReaderClose(&pReader->pTsdbReader);
+        if (code) goto _err;
+      }
+    }
   }
 
   *ppData = NULL;
@@ -130,7 +132,7 @@ _exit:
   return code;
 
 _err:
-  vError("vgId:% snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code));
+  vError("vgId:% vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code));
   return code;
 }
 

From 338a7638537358d6fddafd766b5f02d87409a39c Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 13 Jul 2022 02:41:37 +0000
Subject: [PATCH 04/38] more vnode snapshot

---
 source/dnode/vnode/src/inc/tsdb.h          |  2 +-
 source/dnode/vnode/src/tsdb/tsdbCommit.c   |  2 +-
 source/dnode/vnode/src/tsdb/tsdbFS.c       |  4 +-
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 67 ++++++++++++----------
 source/dnode/vnode/src/vnd/vnodeSnapshot.c |  7 ++-
 5 files changed, 46 insertions(+), 36 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 4df9f96514..819076536d 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -202,7 +202,7 @@ int32_t    tsdbFSStateUpsertDelFile(STsdbFSState *pState, SDelFile *pDelFile);
 int32_t    tsdbFSStateUpsertDFileSet(STsdbFSState *pState, SDFileSet *pSet);
 void       tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid);
 SDelFile  *tsdbFSStateGetDelFile(STsdbFSState *pState);
-SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid);
+SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid, int32_t flag);
 // tsdbReaderWriter.c ==============================================================================================
 // SDataFWriter
 int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet);
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index 8a189d98b3..577ef03a8f 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -263,7 +263,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
   taosArrayClear(pCommitter->aBlockIdx);
   tMapDataReset(&pCommitter->oBlockMap);
   tBlockDataReset(&pCommitter->oBlockData);
-  pRSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, pCommitter->commitFid);
+  pRSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, pCommitter->commitFid, TD_EQ);
   if (pRSet) {
     code = tsdbDataFReaderOpen(&pCommitter->pReader, pTsdb, pRSet);
     if (code) goto _err;
diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c
index 53b6735c30..70fbd90646 100644
--- a/source/dnode/vnode/src/tsdb/tsdbFS.c
+++ b/source/dnode/vnode/src/tsdb/tsdbFS.c
@@ -698,6 +698,6 @@ void tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid) {
 
 SDelFile *tsdbFSStateGetDelFile(STsdbFSState *pState) { return pState->pDelFile; }
 
-SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid) {
-  return (SDFileSet *)taosArraySearch(pState->aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ);
+SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid, int32_t flag) {
+  return (SDFileSet *)taosArraySearch(pState->aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, flag);
 }
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index b02e1dabd3..7dc7e29ad7 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -40,10 +40,12 @@ struct STsdbSnapReader {
 
 static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
   int32_t code = 0;
+  STsdb*  pTsdb = pReader->pTsdb;
 
   while (true) {
     if (pReader->pDataFReader == NULL) {
       SDFileSet* pSet = NULL;
+      // taosArraySearch(pTsdb->fs->cState->aDFileSet, &(SDFileSet){.fid = pReader->fid}, tDFileSe)
 
       // search the next data file set to read (todo)
       if (0 /* TODO */) {
@@ -106,10 +108,13 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
   }
 
 _exit:
+  // if (*ppData) {
+  //   tsdbInfo("vgId:%d ");
+  // }
   return code;
 
 _err:
-  tsdbError("vgId:%d snap read data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d vnode snapshot tsdb read data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
@@ -209,14 +214,13 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe
   pReader->sver = sver;
   pReader->ever = ever;
 
+  pReader->fid = INT32_MIN;
   pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
   if (pReader->aBlockIdx == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
-
   pReader->mBlock = tMapDataInit();
-
   code = tBlockDataInit(&pReader->blkData);
   if (code) goto _err;
 
@@ -225,7 +229,6 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
-
   pReader->aDelData = taosArrayInit(0, sizeof(SDelData));
   if (pReader->aDelData == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
@@ -246,37 +249,42 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
   int32_t          code = 0;
   STsdbSnapReader* pReader = *ppReader;
 
-  taosArrayDestroy(pReader->aDelData);
-  taosArrayDestroy(pReader->aDelIdx);
-  if (pReader->pDelFReader) {
-    tsdbDelFReaderClose(&pReader->pDelFReader);
-  }
-  tBlockDataClear(&pReader->blkData);
-  tMapDataClear(&pReader->mBlock);
-  taosArrayDestroy(pReader->aBlockIdx);
   if (pReader->pDataFReader) {
     tsdbDataFReaderClose(&pReader->pDataFReader);
   }
+  taosArrayDestroy(pReader->aBlockIdx);
+  tMapDataClear(&pReader->mBlock);
+  tBlockDataClear(&pReader->blkData);
+
+  if (pReader->pDelFReader) {
+    tsdbDelFReaderClose(&pReader->pDelFReader);
+  }
+  taosArrayDestroy(pReader->aDelIdx);
+  taosArrayDestroy(pReader->aDelData);
+
+  tsdbInfo("vgId:%d vnode snapshot tsdb reader closed", TD_VID(pReader->pTsdb->pVnode));
+
   taosMemoryFree(pReader);
   *ppReader = NULL;
-
   return code;
 }
 
 int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) {
   int32_t code = 0;
 
+  *ppData = NULL;
+
   // read data file
   if (!pReader->dataDone) {
     code = tsdbSnapReadData(pReader, ppData);
     if (code) {
-      if (code == TSDB_CODE_VND_READ_END) {
-        pReader->dataDone = 1;
-      } else {
-        goto _err;
-      }
+      goto _err;
     } else {
-      goto _exit;
+      if (*ppData) {
+        goto _exit;
+      } else {
+        pReader->dataDone = 1;
+      }
     }
   }
 
@@ -284,23 +292,24 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) {
   if (!pReader->delDone) {
     code = tsdbSnapReadDel(pReader, ppData);
     if (code) {
-      if (code == TSDB_CODE_VND_READ_END) {
-        pReader->delDone = 1;
-      } else {
-        goto _err;
-      }
+      goto _err;
     } else {
-      goto _exit;
+      if (*ppData) {
+        goto _exit;
+      } else {
+        pReader->delDone = 1;
+      }
     }
   }
 
-  code = TSDB_CODE_VND_READ_END;
-
 _exit:
+  if (*ppData) {
+  } else {
+  }
   return code;
 
 _err:
-  tsdbError("vgId:%d snapshot read failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d vnode snapshot tsdb read failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
@@ -520,7 +529,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     if (code) goto _err;
 
     pWriter->fid = fid;
-    SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid);
+    SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ);
     // reader
     if (pSet) {
       // open
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 0e14df0380..ed09f0edc3 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -125,7 +125,8 @@ _exit:
     pReader->index++;
     *nData = sizeof(SSnapDataHdr) + pHdr->size;
     pHdr->index = pReader->index;
-    vInfo("vgId:%d vnode snapshot read data, nData:%d index:%" PRId64, TD_VID(pReader->pVnode), *nData, pReader->index);
+    vInfo("vgId:%d vnode snapshot read data,index:%" PRId64 " type:%d nData:%d ", TD_VID(pReader->pVnode),
+          pReader->index, pHdr->type, *nData);
   } else {
     vInfo("vgId:%d vnode snapshot read data end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index);
   }
@@ -236,8 +237,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
   // }
 
 _exit:
-  vInfo("vgId:%d vnode snapshot write data, type:%d index:%" PRId64 " nData:%d", TD_VID(pVnode), pHdr->type,
-        pHdr->index, nData);
+  vInfo("vgId:%d vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->type, nData,
+        pHdr->index);
   return code;
 
 _err:

From b691e76782011d2c612497e05b25379f28e9880d Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 13 Jul 2022 05:51:37 +0000
Subject: [PATCH 05/38] more vnode snapshot

---
 source/dnode/vnode/src/inc/tsdb.h          |   4 +
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c |  85 +++++++++++----
 source/dnode/vnode/src/tsdb/tsdbUtil.c     | 116 +++++++++++++++++++++
 3 files changed, 182 insertions(+), 23 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 819076536d..bfb319dce4 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -126,6 +126,8 @@ void    tColDataClear(void *ph);
 int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
 int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
 int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
+int32_t tPutColData(uint8_t *p, SColData *pColData);
+int32_t tGetColData(uint8_t *p, SColData *pColData);
 // SBlockData
 #define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0)
 #define tBlockDataLastRow(PBLOCKDATA)  tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1)
@@ -142,6 +144,8 @@ int32_t   tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlo
 int32_t   tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest);
 SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx);
 void      tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData);
+int32_t   tPutBlockData(uint8_t *p, SBlockData *pBlockData);
+int32_t   tGetBlockData(uint8_t *p, SBlockData *pBlockData);
 // SDelIdx
 int32_t tPutDelIdx(uint8_t *p, void *ph);
 int32_t tGetDelIdx(uint8_t *p, void *ph);
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 7dc7e29ad7..8bf1b29454 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -29,7 +29,8 @@ struct STsdbSnapReader {
   SBlockIdx*    pBlockIdx;
   SMapData      mBlock;  // SMapData<SBlock>
   int32_t       iBlock;
-  SBlockData    blkData;
+  SBlockData    oBlockData;
+  SBlockData    nBlockData;
   // for del file
   int8_t       delDone;
   SDelFReader* pDelFReader;
@@ -44,16 +45,11 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
 
   while (true) {
     if (pReader->pDataFReader == NULL) {
-      SDFileSet* pSet = NULL;
-      // taosArraySearch(pTsdb->fs->cState->aDFileSet, &(SDFileSet){.fid = pReader->fid}, tDFileSe)
+      SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->cState, pReader->fid, TD_GT);
 
-      // search the next data file set to read (todo)
-      if (0 /* TODO */) {
-        code = TSDB_CODE_VND_READ_END;
-        goto _exit;
-      }
+      if (pSet == NULL) goto _exit;
 
-      // open
+      pReader->fid = pSet->fid;
       code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet);
       if (code) goto _err;
 
@@ -63,6 +59,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
 
       pReader->iBlockIdx = 0;
       pReader->pBlockIdx = NULL;
+
+      tsdbInfo("vgId:%d vnode snapshot tsdb open data file to read, fid:%d", TD_VID(pTsdb->pVnode), pReader->fid);
     }
 
     while (true) {
@@ -75,17 +73,15 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
         pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
         pReader->iBlockIdx++;
 
-        // SBlock
         code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock, NULL);
         if (code) goto _err;
 
         pReader->iBlock = 0;
       }
 
+      SBlock  block;
+      SBlock* pBlock = &block;
       while (true) {
-        SBlock  block;
-        SBlock* pBlock = &block;
-
         if (pReader->iBlock >= pReader->mBlock.nItem) {
           pReader->pBlockIdx = NULL;
           break;
@@ -94,23 +90,63 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
         tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, pBlock, tGetBlock);
         pReader->iBlock++;
 
-        if ((pBlock->minVersion >= pReader->sver && pBlock->minVersion <= pReader->ever) ||
-            (pBlock->maxVersion >= pReader->sver && pBlock->maxVersion <= pReader->ever)) {
-          // overlap (todo)
+        if (pBlock->minVersion > pReader->ever || pBlock->maxVersion < pReader->sver) continue;
 
-          code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->blkData, NULL, NULL);
+        code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->oBlockData, NULL, NULL);
+        if (code) goto _err;
+
+        // filter
+        tBlockDataReset(&pReader->nBlockData);
+        for (int32_t iColData = 0; iColData < taosArrayGetSize(pReader->oBlockData.aIdx); iColData++) {
+          SColData* pColDataO = tBlockDataGetColDataByIdx(&pReader->oBlockData, iColData);
+          SColData* pColDataN = NULL;
+
+          code = tBlockDataAddColData(&pReader->nBlockData, taosArrayGetSize(pReader->nBlockData.aIdx), &pColDataN);
           if (code) goto _err;
 
-          goto _exit;
+          tColDataInit(pColDataN, pColDataO->cid, pColDataO->type, pColDataO->smaOn);
         }
+
+        for (int32_t iRow = 0; iRow < pReader->oBlockData.nRow; iRow++) {
+          TSDBROW row = tsdbRowFromBlockData(&pReader->oBlockData, iRow);
+          int64_t version = TSDBROW_VERSION(&row);
+
+          if (version < pReader->sver || version > pReader->ever) continue;
+
+          code = tBlockDataAppendRow(&pReader->nBlockData, &row, NULL);
+          if (code) goto _err;
+        }
+
+        // org data (todo)
+        int32_t size = sizeof(TABLEID) + tPutBlockData(NULL, &pReader->nBlockData);
+
+        *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size);
+        if (*ppData == NULL) {
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto _err;
+        }
+
+        SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
+        pHdr->type = 1;
+        pHdr->size = size;
+
+        TABLEID* pId = (TABLEID*)(&pHdr[1]);
+        pId->suid = pReader->pBlockIdx->suid;
+        pId->uid = pReader->pBlockIdx->uid;
+
+        tPutBlockData((uint8_t*)(&pId[1]), &pReader->nBlockData);
+
+        tsdbInfo("vgId:%d vnode snapshot read data, fid:%d suid:%" PRId64 " uid:%" PRId64
+                 " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d",
+                 TD_VID(pTsdb->pVnode), pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid,
+                 pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow);
+
+        goto _exit;
       }
     }
   }
 
 _exit:
-  // if (*ppData) {
-  //   tsdbInfo("vgId:%d ");
-  // }
   return code;
 
 _err:
@@ -221,7 +257,9 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe
     goto _err;
   }
   pReader->mBlock = tMapDataInit();
-  code = tBlockDataInit(&pReader->blkData);
+  code = tBlockDataInit(&pReader->oBlockData);
+  if (code) goto _err;
+  code = tBlockDataInit(&pReader->nBlockData);
   if (code) goto _err;
 
   pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
@@ -254,7 +292,8 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
   }
   taosArrayDestroy(pReader->aBlockIdx);
   tMapDataClear(&pReader->mBlock);
-  tBlockDataClear(&pReader->blkData);
+  tBlockDataClear(&pReader->oBlockData);
+  tBlockDataClear(&pReader->nBlockData);
 
   if (pReader->pDelFReader) {
     tsdbDelFReaderClose(&pReader->pDelFReader);
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index dc6c823368..db408df944 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -921,6 +921,76 @@ _exit:
   return code;
 }
 
+int32_t tPutColData(uint8_t *p, SColData *pColData) {
+  int32_t n = 0;
+
+  n += tPutI16v(p ? p + n : p, pColData->cid);
+  n += tPutI8(p ? p + n : p, pColData->type);
+  n += tPutI8(p ? p + n : p, pColData->smaOn);
+  n += tPutI32v(p ? p + n : p, pColData->nVal);
+  n += tPutU8(p ? p + n : p, pColData->flag);
+
+  if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
+  if (pColData->flag != HAS_VALUE) {
+    // bitmap
+
+    int32_t size = BIT2_SIZE(pColData->nVal);
+    if (p) {
+      memcpy(p + n, pColData->pBitMap, size);
+    }
+    n += size;
+  }
+  if (IS_VAR_DATA_TYPE(pColData->type)) {
+    // offset
+
+    int32_t size = sizeof(int32_t) * pColData->nVal;
+    if (p) {
+      memcpy(p + n, pColData->aOffset, size);
+    }
+    n += size;
+  }
+  n += tPutI32v(p ? p + n : p, pColData->nData);
+  if (p) {
+    memcpy(p + n, pColData->pData, pColData->nData);
+  }
+  n += pColData->nData;
+
+_exit:
+  return n;
+}
+
+int32_t tGetColData(uint8_t *p, SColData *pColData) {
+  int32_t n = 0;
+
+  n += tGetI16v(p + n, &pColData->cid);
+  n += tGetI8(p + n, &pColData->type);
+  n += tGetI8(p + n, &pColData->smaOn);
+  n += tGetI32v(p + n, &pColData->nVal);
+  n += tGetU8(p + n, &pColData->flag);
+
+  if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
+  if (pColData->flag != HAS_VALUE) {
+    // bitmap
+
+    int32_t size = BIT2_SIZE(pColData->nVal);
+    pColData->pBitMap = p + n;
+    n += size;
+  }
+  if (IS_VAR_DATA_TYPE(pColData->type)) {
+    // offset
+
+    int32_t size = sizeof(int32_t) * pColData->nVal;
+    pColData->aOffset = (int32_t *)(p + n);
+    n += size;
+  }
+  n += tGetI32v(p + n, &pColData->nData);
+  pColData->pData = p + n;
+  n += pColData->nData;
+
+_exit:
+  return n;
+}
+
 static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) {
   SColData *pColData1 = (SColData *)p1;
   SColData *pColData2 = (SColData *)p2;
@@ -1239,6 +1309,52 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD
   *ppColData = NULL;
 }
 
+int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData) {
+  int32_t n = 0;
+
+  n += tPutI32v(p ? p + n : p, pBlockData->nRow);
+  if (p) {
+    memcpy(p + n, pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow);
+  }
+  n = n + sizeof(int64_t) * pBlockData->nRow;
+  if (p) {
+    memcpy(p + n, pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow);
+  }
+  n = n + sizeof(TSKEY) * pBlockData->nRow;
+
+  int32_t nCol = taosArrayGetSize(pBlockData->aIdx);
+  n += tPutI32v(p ? p + n : p, nCol);
+  for (int32_t iCol = 0; iCol < nCol; iCol++) {
+    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol);
+    n += tPutColData(p ? p + n : p, pColData);
+  }
+
+  return n;
+}
+
+int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData) {
+  int32_t n = 0;
+
+  tBlockDataReset(pBlockData);
+
+  n += tGetI32v(p + n, &pBlockData->nRow);
+  pBlockData->aVersion = (int64_t *)(p + n);
+  n = n + sizeof(int64_t) * pBlockData->nRow;
+  pBlockData->aTSKEY = (TSKEY *)(p + n);
+  n = n + sizeof(TSKEY) * pBlockData->nRow;
+
+  int32_t nCol;
+  n += tGetI32v(p + n, &nCol);
+  for (int32_t iCol = 0; iCol < nCol; iCol++) {
+    SColData *pColData;
+
+    if (tBlockDataAddColData(pBlockData, iCol, &pColData)) return -1;
+    n += tGetColData(p + n, pColData);
+  }
+
+  return n;
+}
+
 // ALGORITHM ==============================
 void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) {
   SColVal  colVal;

From 2fd04241d6a3a418c24130c875ef6c46060f5b8a Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 13 Jul 2022 06:26:13 +0000
Subject: [PATCH 06/38] more vnode snapshot reader

---
 include/util/taoserror.h                   |  1 -
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 85 +++++++++++-----------
 source/dnode/vnode/src/vnd/vnodeSnapshot.c |  4 +-
 source/util/src/terror.c                   |  2 -
 4 files changed, 45 insertions(+), 47 deletions(-)

diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index ce434612c3..a07da1aa45 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -331,7 +331,6 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_VND_INVALID_TABLE_ACTION      TAOS_DEF_ERROR_CODE(0, 0x0519)
 #define TSDB_CODE_VND_COL_ALREADY_EXISTS        TAOS_DEF_ERROR_CODE(0, 0x051a)
 #define TSDB_CODE_VND_TABLE_COL_NOT_EXISTS      TAOS_DEF_ERROR_CODE(0, 0x051b)
-#define TSDB_CODE_VND_READ_END                  TAOS_DEF_ERROR_CODE(0, 0x051c)
 
 // tsdb
 #define TSDB_CODE_TDB_INVALID_TABLE_ID          TAOS_DEF_ERROR_CODE(0, 0x0600)
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 8bf1b29454..fbe14088df 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -34,8 +34,8 @@ struct STsdbSnapReader {
   // for del file
   int8_t       delDone;
   SDelFReader* pDelFReader;
+  SArray*      aDelIdx;  // SArray<SDelIdx>
   int32_t      iDelIdx;
-  SArray*      aDelIdx;   // SArray<SDelIdx>
   SArray*      aDelData;  // SArray<SDelData>
 };
 
@@ -117,7 +117,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
           if (code) goto _err;
         }
 
-        // org data (todo)
+        // org data
+        // compress data (todo)
         int32_t size = sizeof(TABLEID) + tPutBlockData(NULL, &pReader->nBlockData);
 
         *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size);
@@ -137,9 +138,10 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
         tPutBlockData((uint8_t*)(&pId[1]), &pReader->nBlockData);
 
         tsdbInfo("vgId:%d vnode snapshot read data, fid:%d suid:%" PRId64 " uid:%" PRId64
-                 " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d",
+                 " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d size:%d",
                  TD_VID(pTsdb->pVnode), pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid,
-                 pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow);
+                 pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow,
+                 size);
 
         goto _exit;
       }
@@ -161,7 +163,6 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
 
   if (pReader->pDelFReader == NULL) {
     if (pDelFile == NULL) {
-      code = TSDB_CODE_VND_READ_END;
       goto _exit;
     }
 
@@ -176,15 +177,20 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
     pReader->iDelIdx = 0;
   }
 
-  while (pReader->iDelIdx < taosArrayGetSize(pReader->aDelIdx)) {
+  while (true) {
+    if (pReader->iDelIdx >= taosArrayGetSize(pReader->aDelIdx)) {
+      tsdbDelFReaderClose(&pReader->pDelFReader);
+      break;
+    }
+
     SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pReader->aDelIdx, pReader->iDelIdx);
-    int32_t  size = 0;
 
     pReader->iDelIdx++;
 
     code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData, NULL);
     if (code) goto _err;
 
+    int32_t size = 0;
     for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) {
       SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData);
 
@@ -193,46 +199,44 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
       }
     }
 
-    if (size > 0) {
-      int64_t n = 0;
+    if (size == 0) continue;
 
-      size = size + sizeof(SSnapDataHdr) + sizeof(TABLEID);
-      code = tRealloc(ppData, size);
-      if (code) goto _err;
-
-      // SSnapDataHdr
-      SSnapDataHdr* pSnapDataHdr = (SSnapDataHdr*)(*ppData + n);
-      pSnapDataHdr->type = 1;
-      pSnapDataHdr->size = size;  // TODO: size here may incorrect
-      n += sizeof(SSnapDataHdr);
-
-      // TABLEID
-      TABLEID* pId = (TABLEID*)(*ppData + n);
-      pId->suid = pDelIdx->suid;
-      pId->uid = pDelIdx->uid;
-      n += sizeof(*pId);
-
-      // DATA
-      for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) {
-        SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData);
-
-        if (pDelData->version >= pReader->sver && pDelData->version <= pReader->ever) {
-          n += tPutDelData(*ppData + n, pDelData);
-        }
-      }
-
-      goto _exit;
+    // org data
+    size = sizeof(TABLEID) + size;
+    *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size);
+    if (*ppData == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
     }
-  }
 
-  code = TSDB_CODE_VND_READ_END;
-  tsdbDelFReaderClose(&pReader->pDelFReader);
+    SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
+    pHdr->type = 2;
+    pHdr->size = size;
+
+    TABLEID* pId = (TABLEID*)(&pHdr[1]);
+    pId->suid = pDelIdx->suid;
+    pId->uid = pDelIdx->uid;
+    int32_t n = sizeof(SSnapDataHdr) + sizeof(TABLEID);
+    for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) {
+      SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData);
+
+      if (pDelData->version < pReader->sver) continue;
+      if (pDelData->version > pReader->ever) continue;
+
+      n += tPutDelData((*ppData) + n, pDelData);
+    }
+
+    tsdbInfo("vgId:%d vnode snapshot tsdb read del data, suid:%" PRId64 " uid:%d" PRId64 " size:%d",
+             TD_VID(pTsdb->pVnode), pDelIdx->suid, pDelIdx->uid, size);
+
+    break;
+  }
 
 _exit:
   return code;
 
 _err:
-  tsdbError("vgId:%d snap read del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d vnode snapshot tsdb read del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
@@ -342,9 +346,6 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) {
   }
 
 _exit:
-  if (*ppData) {
-  } else {
-  }
   return code;
 
 _err:
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index ed09f0edc3..26dbf75426 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -237,8 +237,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
   // }
 
 _exit:
-  vInfo("vgId:%d vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->type, nData,
-        pHdr->index);
+  vInfo("vgId:%d vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->index,
+        pHdr->type, nData);
   return code;
 
 _err:
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index ef6697b3b5..941f3bf558 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -335,8 +335,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_NOT_EXIST,          "Table does not exists
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TABLE_ACTION,     "Invalid table action")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS,       "Table column already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_COL_NOT_EXISTS,     "Table column not exists")
-TAOS_DEFINE_ERROR(TSDB_CODE_VND_READ_END,                 "Read end")
-
 
 // tsdb
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID,         "Invalid table ID")

From 583742089e0b54a69c6d0ca0e87d8525fc474958 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 13 Jul 2022 08:14:20 +0000
Subject: [PATCH 07/38] more vnode snapshot writer

---
 source/dnode/vnode/src/inc/tsdb.h             | 14 ++--
 source/dnode/vnode/src/tsdb/tsdbCache.c       | 12 ++--
 source/dnode/vnode/src/tsdb/tsdbCommit.c      |  4 +-
 .../dnode/vnode/src/tsdb/tsdbReaderWriter.c   | 28 ++++----
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c    | 50 ++++++++++----
 source/dnode/vnode/src/tsdb/tsdbUtil.c        |  4 +-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c    | 69 ++++++++++---------
 7 files changed, 105 insertions(+), 76 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index bfb319dce4..2e680511c3 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -24,12 +24,12 @@ extern "C" {
 
 // tsdbDebug ================
 // clang-format off
-#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
-#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
-#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}       while(0)
-#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
-#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
-#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSD FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSD ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}     while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TSD WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}       while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TSD ", DEBUG_INFO, 255, __VA_ARGS__); }}            while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSD ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSD ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
 typedef struct TSDBROW       TSDBROW;
@@ -137,7 +137,7 @@ int32_t   tBlockDataInit(SBlockData *pBlockData);
 void      tBlockDataReset(SBlockData *pBlockData);
 int32_t   tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema);
 void      tBlockDataClearData(SBlockData *pBlockData);
-void      tBlockDataClear(SBlockData *pBlockData);
+void      tBlockDataClear(SBlockData *pBlockData, int8_t deepClear);
 int32_t   tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData);
 int32_t   tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema);
 int32_t   tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData);
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index 605f4bf3d5..b3e0695c10 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -476,9 +476,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
       if (--state->iFileSet >= 0) {
         pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet);
       } else {
-        // tBlockDataClear(&state->blockData);
+        // tBlockDataClear(&state->blockData, 1);
         if (state->pBlockData) {
-          tBlockDataClear(state->pBlockData);
+          tBlockDataClear(state->pBlockData, 1);
           state->pBlockData = NULL;
         }
 
@@ -582,8 +582,8 @@ _err:
     state->aBlockIdx = NULL;
   }
   if (state->pBlockData) {
-    // tBlockDataClear(&state->blockData);
-    tBlockDataClear(state->pBlockData);
+    // tBlockDataClear(&state->blockData, 1);
+    tBlockDataClear(state->pBlockData, 1);
     state->pBlockData = NULL;
   }
 
@@ -609,8 +609,8 @@ int32_t clearNextRowFromFS(void *iter) {
     state->aBlockIdx = NULL;
   }
   if (state->pBlockData) {
-    // tBlockDataClear(&state->blockData);
-    tBlockDataClear(state->pBlockData);
+    // tBlockDataClear(&state->blockData, 1);
+    tBlockDataClear(state->pBlockData, 1);
     state->pBlockData = NULL;
   }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index 577ef03a8f..ce14a92734 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -1001,10 +1001,10 @@ _exit:
 static void tsdbCommitDataEnd(SCommitter *pCommitter) {
   taosArrayDestroy(pCommitter->aBlockIdx);
   tMapDataClear(&pCommitter->oBlockMap);
-  tBlockDataClear(&pCommitter->oBlockData);
+  tBlockDataClear(&pCommitter->oBlockData, 1);
   taosArrayDestroy(pCommitter->aBlockIdxN);
   tMapDataClear(&pCommitter->nBlockMap);
-  tBlockDataClear(&pCommitter->nBlockData);
+  tBlockDataClear(&pCommitter->nBlockData, 1);
   tTSchemaDestroy(pCommitter->skmTable.pTSchema);
   tTSchemaDestroy(pCommitter->skmRow.pTSchema);
 }
diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index 4e7a9d3b04..0ed2e12542 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -979,21 +979,21 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl
 
       code = tBlockDataCopy(pBlockData, pBlockData2);
       if (code) {
-        tBlockDataClear(pBlockData1);
-        tBlockDataClear(pBlockData2);
+        tBlockDataClear(pBlockData1, 1);
+        tBlockDataClear(pBlockData2, 1);
         goto _err;
       }
 
       code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData);
       if (code) {
-        tBlockDataClear(pBlockData1);
-        tBlockDataClear(pBlockData2);
+        tBlockDataClear(pBlockData1, 1);
+        tBlockDataClear(pBlockData2, 1);
         goto _err;
       }
     }
 
-    tBlockDataClear(pBlockData1);
-    tBlockDataClear(pBlockData2);
+    tBlockDataClear(pBlockData1, 1);
+    tBlockDataClear(pBlockData2, 1);
   }
 
   tFree(pBuf1);
@@ -1115,29 +1115,29 @@ int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *p
     for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) {
       code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2);
       if (code) {
-        tBlockDataClear(pBlockData1);
-        tBlockDataClear(pBlockData2);
+        tBlockDataClear(pBlockData1, 1);
+        tBlockDataClear(pBlockData2, 1);
         goto _err;
       }
 
       code = tBlockDataCopy(pBlockData, pBlockData2);
       if (code) {
-        tBlockDataClear(pBlockData1);
-        tBlockDataClear(pBlockData2);
+        tBlockDataClear(pBlockData1, 1);
+        tBlockDataClear(pBlockData2, 1);
         goto _err;
       }
 
       // merge two block data
       code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData);
       if (code) {
-        tBlockDataClear(pBlockData1);
-        tBlockDataClear(pBlockData2);
+        tBlockDataClear(pBlockData1, 1);
+        tBlockDataClear(pBlockData2, 1);
         goto _err;
       }
     }
 
-    tBlockDataClear(pBlockData1);
-    tBlockDataClear(pBlockData2);
+    tBlockDataClear(pBlockData1, 1);
+    tBlockDataClear(pBlockData2, 1);
   }
 
   ASSERT(pBlock->nRow == pBlockData->nRow);
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index fbe14088df..563458ad85 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -296,8 +296,8 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
   }
   taosArrayDestroy(pReader->aBlockIdx);
   tMapDataClear(&pReader->mBlock);
-  tBlockDataClear(&pReader->oBlockData);
-  tBlockDataClear(&pReader->nBlockData);
+  tBlockDataClear(&pReader->oBlockData, 1);
+  tBlockDataClear(&pReader->nBlockData, 1);
 
   if (pReader->pDelFReader) {
     tsdbDelFReaderClose(&pReader->pDelFReader);
@@ -555,11 +555,21 @@ _err:
 }
 
 static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
-  int32_t code = 0;
-  STsdb*  pTsdb = pWriter->pTsdb;
-  int64_t skey;  // todo
-  int64_t ekey;  // todo
+  int32_t       code = 0;
+  STsdb*        pTsdb = pWriter->pTsdb;
+  SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
+  TABLEID       id = *(TABLEID*)(&pHdr[1]);
+  int64_t       n;
+  SBlockData    bData = {0};
+  SBlockData*   pBlockData = &bData;
 
+  // decode
+  code = tBlockDataInit(pBlockData);
+  if (code) goto _err;
+  n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData);
+  ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData);
+
+#if 0
   int32_t fid = tsdbKeyFid(skey, pWriter->minutes, pWriter->precision);
   ASSERT(fid == tsdbKeyFid(ekey, pWriter->minutes, pWriter->precision));
 
@@ -600,7 +610,10 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
 
   code = tsdbSnapWriteTableData(pWriter, pData, nData);
   if (code) goto _err;
+#endif
 
+  tsdbInfo("vgId:%d vnode snapshot tsdb write data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d", TD_VID(pTsdb->pVnode),
+           id.suid, id.suid, pBlockData->nRow);
   return code;
 
 _err:
@@ -755,6 +768,12 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
   pWriter->sver = sver;
   pWriter->ever = ever;
 
+  pWriter->minutes = pTsdb->keepCfg.days;
+  pWriter->precision = pTsdb->keepCfg.precision;
+  pWriter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
+  pWriter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
+  pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
+
   *ppWriter = pWriter;
   return code;
 
@@ -793,24 +812,29 @@ _err:
 }
 
 int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
-  int32_t code = 0;
-  int8_t  type = pData[0];
+  int32_t       code = 0;
+  SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
 
   // ts data
-  if (type == 0) {
-    code = tsdbSnapWriteData(pWriter, pData + 1, nData - 1);
+  if (pHdr->type == 1) {
+    code = tsdbSnapWriteData(pWriter, pData, nData);
     if (code) goto _err;
+
+    goto _exit;
   } else {
-    code = tsdbSnapWriteDataEnd(pWriter);
-    if (code) goto _err;
+    if (pWriter->pDataFWriter) {
+      code = tsdbSnapWriteDataEnd(pWriter);
+      if (code) goto _err;
+    }
   }
 
   // del data
-  if (type == 1) {
+  if (pHdr->type == 2) {
     code = tsdbSnapWriteDel(pWriter, pData + 1, nData - 1);
     if (code) goto _err;
   }
 
+_exit:
   return code;
 
 _err:
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index db408df944..7667127fc9 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -1032,11 +1032,11 @@ void tBlockDataReset(SBlockData *pBlockData) {
   taosArrayClear(pBlockData->aIdx);
 }
 
-void tBlockDataClear(SBlockData *pBlockData) {
+void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear) {
   tFree((uint8_t *)pBlockData->aVersion);
   tFree((uint8_t *)pBlockData->aTSKEY);
   taosArrayDestroy(pBlockData->aIdx);
-  taosArrayDestroyEx(pBlockData->aColData, tColDataClear);
+  taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL);
 }
 
 int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema) {
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 26dbf75426..674beb51ec 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -176,7 +176,6 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr
   pWriter->ever = ever;
 
   vInfo("vgId:%d vnode snapshot writer opened", TD_VID(pVnode));
-
   *ppWriter = pWriter;
   return code;
 
@@ -189,15 +188,15 @@ _err:
 int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
   int32_t code = 0;
 
-  goto _exit;
+  // TODO
 
-  if (rollback) {
-    code = vnodeSnapRollback(pWriter);
-    if (code) goto _err;
-  } else {
-    code = vnodeSnapCommit(pWriter);
-    if (code) goto _err;
-  }
+  // if (rollback) {
+  //   code = vnodeSnapRollback(pWriter);
+  //   if (code) goto _err;
+  // } else {
+  //   code = vnodeSnapCommit(pWriter);
+  //   if (code) goto _err;
+  // }
 
 _exit:
   vInfo("vgId:%d vnode snapshot writer closed, rollback:%d", TD_VID(pWriter->pVnode), rollback);
@@ -214,34 +213,40 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
   SSnapDataHdr *pHdr = (SSnapDataHdr *)pData;
   SVnode       *pVnode = pWriter->pVnode;
 
-  // ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData);
+  ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData);
+  ASSERT(pHdr->index == pWriter->index + 1);
+  pWriter->index = pHdr->index;
 
-  // if (pHdr->type == 0) {
-  //   // meta
-  //   if (pWriter->pMetaSnapWriter == NULL) {
-  //     code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
-  //     if (code) goto _err;
-  //   }
-
-  //   code = metaSnapWrite(pWriter->pMetaSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
-  //   if (code) goto _err;
-  // } else {
-  //   // tsdb
-  //   if (pWriter->pTsdbSnapWriter == NULL) {
-  //     code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter);
-  //     if (code) goto _err;
-  //   }
-
-  //   code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
-  //   if (code) goto _err;
-  // }
-
-_exit:
   vInfo("vgId:%d vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->index,
         pHdr->type, nData);
+
+  if (pHdr->type == 0) {
+    // meta
+
+    // if (pWriter->pMetaSnapWriter == NULL) {
+    //   code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
+    //   if (code) goto _err;
+    // }
+
+    // code = metaSnapWrite(pWriter->pMetaSnapWriter, pData , nData);
+    // if (code) goto _err;
+  } else {
+    // tsdb
+
+    if (pWriter->pTsdbSnapWriter == NULL) {
+      code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter);
+      if (code) goto _err;
+    }
+
+    code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData, nData);
+    if (code) goto _err;
+  }
+
+_exit:
   return code;
 
 _err:
-  vError("vgId:%d vnode snapshot write failed since %s", TD_VID(pVnode), tstrerror(code));
+  vError("vgId:%d vnode snapshot write failed since %s, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode),
+         tstrerror(code), pHdr->index, pHdr->type, nData);
   return code;
 }
\ No newline at end of file

From a32c147b3b2bda16abc16b3425fb4d44435e2733 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 13 Jul 2022 09:55:55 +0000
Subject: [PATCH 08/38] more vnode snapshot writer

---
 source/dnode/vnode/src/tsdb/tsdbCommit.c   |  11 +-
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 169 ++++++++++++++-------
 2 files changed, 115 insertions(+), 65 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index ce14a92734..470bff1eae 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -284,16 +284,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
                        .fLast = {.commitID = pCommitter->commitID, .size = 0},
                        .fSma = pRSet->fSma};
   } else {
-    STfs   *pTfs = pTsdb->pVnode->pTfs;
-    SDiskID did = {.level = 0, .id = 0};
-
-    // TODO: alloc a new disk
-    // tfsAllocDisk(pTfs, 0, &did);
-
-    // create the directory
-    tfsMkdirRecurAt(pTfs, pTsdb->path, did);
-
-    wSet = (SDFileSet){.diskId = did,
+    wSet = (SDFileSet){.diskId = (SDiskID){.level = 0, .id = 0},
                        .fid = pCommitter->commitFid,
                        .fHead = {.commitID = pCommitter->commitID, .offset = 0, .size = 0},
                        .fData = {.commitID = pCommitter->commitID, .size = 0},
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 563458ad85..193f4997b0 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -365,36 +365,39 @@ struct STsdbSnapWriter {
   int32_t minRow;
   int32_t maxRow;
   int8_t  cmprAlg;
+  int64_t commitID;
 
   // for data file
+  SBlockData bData;
+
   int32_t       fid;
   SDataFReader* pDataFReader;
-  SArray*       aBlockIdx;
+  SArray*       aBlockIdx;  // SArray<SBlockIdx>
   int32_t       iBlockIdx;
   SBlockIdx*    pBlockIdx;
-  SMapData      mBlock;
+  SMapData      mBlock;  // SMapData<SBlock>
   int32_t       iBlock;
   SBlock*       pBlock;
   SBlock        block;
-  SBlockData    blockData;
+  SBlockData    bDataR;
   int32_t       iRow;
 
   SDataFWriter* pDataFWriter;
-  SArray*       aBlockIdxN;
-  SBlockIdx*    pBlockIdxN;
+  SArray*       aBlockIdxW;  // SArray<SBlockIdx>
+  SBlockIdx*    pBlockIdxW;
   SBlockIdx     blockIdx;
-  SMapData      mBlockN;
-  SBlock*       pBlockN;
-  SBlock        blockN;
-  SBlockData    nBlockData;
+  SMapData      mBlockW;  // SMapData<SBlock>
+  SBlock*       pBlockW;
+  SBlock        blockW;
+  SBlockData    bDataW;
 
   // for del file
   SDelFReader* pDelFReader;
   SDelFWriter* pDelFWriter;
   int32_t      iDelIdx;
-  SArray*      aDelIdx;
+  SArray*      aDelIdxR;
   SArray*      aDelData;
-  SArray*      aDelIdxN;
+  SArray*      aDelIdxW;
 };
 
 static int32_t tsdbSnapRollback(STsdbSnapWriter* pWriter) {
@@ -440,7 +443,7 @@ static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData,
   SBlockData* pBlockData = NULL;  // todo
 
   while (iRow < nRow) {
-    code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
+    code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
     if (code) goto _err;
   }
 
@@ -478,14 +481,14 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData,
   if (pWriter->pBlockIdx == NULL || tTABLEIDCmprFn(&id, pWriter->pBlockIdx) < 0) {
     int32_t c;
 
-    if (pWriter->pBlockIdxN && ((c = tTABLEIDCmprFn(&id, pWriter->pBlockIdxN)) != 0)) {
+    if (pWriter->pBlockIdxW && ((c = tTABLEIDCmprFn(&id, pWriter->pBlockIdxW)) != 0)) {
       ASSERT(c > 0);
 
       code = tsdbSnapWriteTableDataEnd(pWriter);
       if (code) goto _err;
     }
 
-    if (pWriter->pBlockIdxN == NULL) {
+    if (pWriter->pBlockIdxW == NULL) {
       pWriter->pBlockIdx = &pWriter->blockIdx;
       pWriter->pBlockIdx->suid = id.suid;
       pWriter->pBlockIdx->uid = id.uid;
@@ -496,12 +499,12 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData,
     int32_t     nRow = 0;           // todo
     SBlockData* pBlockData = NULL;  // todo
     for (int32_t iRow = 0; iRow < nRow; iRow++) {
-      code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
+      code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
       if (code) goto _err;
 
-      if (pWriter->nBlockData.nRow > pWriter->maxRow * 4 / 5) {
-        code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->nBlockData, NULL, NULL, pWriter->pBlockIdxN,
-                                  pWriter->pBlockN, pWriter->cmprAlg);
+      if (pWriter->bDataW.nRow > pWriter->maxRow * 4 / 5) {
+        code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
+                                  pWriter->pBlockW, pWriter->cmprAlg);
         if (code) goto _err;
       }
     }
@@ -512,7 +515,7 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData,
       if (pWriter->pBlock->last) break;
       if (tBlockCmprFn(&(SBlock){.minKey = {0}, .maxKey = {0}}, pWriter->pBlock) >= 0) break;
 
-      code = tMapDataPutItem(&pWriter->mBlockN, pWriter->pBlock, tPutBlock);
+      code = tMapDataPutItem(&pWriter->mBlockW, pWriter->pBlock, tPutBlock);
       if (code) goto _err;
     }
 
@@ -533,15 +536,15 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData,
       SBlockData* pBlockData = NULL;
 
       for (int32_t iRow = 0; iRow < nRow; iRow++) {
-        code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
+        code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
         if (code) goto _err;
 
-        if (pWriter->nBlockData.nRow >= pWriter->maxRow * 4 / 5) {
-          code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->nBlockData, NULL, NULL, pWriter->pBlockIdxN,
-                                    pWriter->pBlockN, pWriter->cmprAlg);
+        if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) {
+          code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
+                                    pWriter->pBlockW, pWriter->cmprAlg);
           if (code) goto _err;
 
-          tBlockDataClearData(&pWriter->nBlockData);
+          tBlockDataClearData(&pWriter->bDataW);
         }
       }
     }
@@ -560,60 +563,77 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
   SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
   TABLEID       id = *(TABLEID*)(&pHdr[1]);
   int64_t       n;
-  SBlockData    bData = {0};
-  SBlockData*   pBlockData = &bData;
 
   // decode
-  code = tBlockDataInit(pBlockData);
-  if (code) goto _err;
+  SBlockData* pBlockData = &pWriter->bData;
   n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData);
   ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData);
 
-#if 0
-  int32_t fid = tsdbKeyFid(skey, pWriter->minutes, pWriter->precision);
-  ASSERT(fid == tsdbKeyFid(ekey, pWriter->minutes, pWriter->precision));
+  // open file
+  TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData);
+  TSDBKEY keyLast = tBlockDataLastKey(pBlockData);
 
-  // begin
+  int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision);
+  ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision));
   if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) {
-    code = tsdbSnapWriteDataEnd(pWriter);
+    code = tsdbSnapWriteDataEnd(pWriter);  // todo
     if (code) goto _err;
 
     pWriter->fid = fid;
-    SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ);
-    // reader
+
+    // read
+    SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ);  // todo: check nState is valid
     if (pSet) {
-      // open
       code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
       if (code) goto _err;
 
-      // SBlockIdx
       code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx, NULL);
       if (code) goto _err;
     } else {
+      ASSERT(pWriter->pDataFReader == NULL);
       taosArrayClear(pWriter->aBlockIdx);
     }
     pWriter->iBlockIdx = 0;
+    pWriter->pBlockIdx = NULL;
+    tMapDataReset(&pWriter->mBlock);
+    pWriter->iBlock = 0;
+    pWriter->pBlock = NULL;
+    tBlockDataReset(&pWriter->bDataR);
+    pWriter->iRow = 0;
 
-    // writer
-    SDFileSet wSet = {0};
-    if (pSet == NULL) {
-      wSet = (SDFileSet){0};  // todo
+    // write
+    SDFileSet wSet;
+
+    if (pSet) {
+      wSet = (SDFileSet){.diskId = pSet->diskId,
+                         .fid = fid,
+                         .fHead = {.commitID = pWriter->commitID, .offset = 0, .size = 0},
+                         .fData = pSet->fData,
+                         .fLast = {.commitID = pWriter->commitID, .size = 0},
+                         .fSma = pSet->fSma};
     } else {
-      wSet = (SDFileSet){0};  // todo
+      wSet = (SDFileSet){.diskId = (SDiskID){.level = 0, .id = 0},
+                         .fid = fid,
+                         .fHead = {.commitID = pWriter->commitID, .offset = 0, .size = 0},
+                         .fData = {.commitID = pWriter->commitID, .size = 0},
+                         .fLast = {.commitID = pWriter->commitID, .size = 0},
+                         .fSma = {.commitID = pWriter->commitID, .size = 0}};
     }
 
     code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet);
     if (code) goto _err;
 
-    taosArrayClear(pWriter->aBlockIdxN);
+    taosArrayClear(pWriter->aBlockIdxW);
+    pWriter->pBlockIdxW = NULL;
+    tMapDataReset(&pWriter->mBlockW);
+    pWriter->pBlockW = NULL;
+    tBlockDataReset(&pWriter->bDataW);
   }
 
-  code = tsdbSnapWriteTableData(pWriter, pData, nData);
-  if (code) goto _err;
-#endif
+  // write data block (todo)
 
-  tsdbInfo("vgId:%d vnode snapshot tsdb write data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d", TD_VID(pTsdb->pVnode),
-           id.suid, id.suid, pBlockData->nRow);
+  tsdbInfo("vgId:%d vnode snapshot tsdb write data, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d",
+           TD_VID(pTsdb->pVnode), fid, id.suid, id.suid, pBlockData->nRow);
   return code;
 
 _err:
@@ -633,7 +653,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
       code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL);
       if (code) goto _err;
 
-      code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdx, NULL);
+      code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR, NULL);
       if (code) goto _err;
     }
 
@@ -653,8 +673,8 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
     SDelIdx  delIdx;
     int8_t   toBreak = 0;
 
-    if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx)) {
-      pDelIdx = taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx);
+    if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR)) {
+      pDelIdx = taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
     }
 
     if (pDelIdx) {
@@ -695,7 +715,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
     code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx);
     if (code) goto _err;
 
-    if (taosArrayPush(pWriter->aDelIdxN, &delIdx) == NULL) {
+    if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _err;
     }
@@ -716,8 +736,8 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
   STsdb*  pTsdb = pWriter->pTsdb;
 
   if (pWriter->pDelFWriter == NULL) goto _exit;
-  for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx); pWriter->iDelIdx++) {
-    SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx);
+  for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) {
+    SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
 
     code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL);
     if (code) goto _err;
@@ -726,7 +746,7 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
     code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx);
     if (code) goto _err;
 
-    if (taosArrayPush(pWriter->aDelIdx, &delIdx) == NULL) {
+    if (taosArrayPush(pWriter->aDelIdxR, &delIdx) == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _err;
     }
@@ -768,11 +788,50 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
   pWriter->sver = sver;
   pWriter->ever = ever;
 
+  // config
   pWriter->minutes = pTsdb->keepCfg.days;
   pWriter->precision = pTsdb->keepCfg.precision;
   pWriter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
   pWriter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
   pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
+  pWriter->commitID = pTsdb->pVnode->state.commitID;
+
+  // for data file
+  code = tBlockDataInit(&pWriter->bData);
+
+  if (code) goto _err;
+  pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+  if (pWriter->aBlockIdx == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  code = tBlockDataInit(&pWriter->bDataR);
+  if (code) goto _err;
+
+  pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx));
+  if (pWriter->aBlockIdxW == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  code = tBlockDataInit(&pWriter->bDataW);
+  if (code) goto _err;
+
+  // for del file
+  pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx));
+  if (pWriter->aDelIdxR == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->aDelData = taosArrayInit(0, sizeof(SDelData));
+  if (pWriter->aDelData == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pWriter->aDelIdxW = taosArrayInit(0, sizeof(SDelIdx));
+  if (pWriter->aDelIdxW == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
 
   *ppWriter = pWriter;
   return code;

From e6e9926d3887688ada9ca076f8b5fa32eadbe2c8 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 13 Jul 2022 12:02:59 +0000
Subject: [PATCH 09/38] more vnode snapshot writer

---
 source/dnode/vnode/src/inc/tsdb.h          |  5 ---
 source/dnode/vnode/src/tsdb/tsdbCache.c    |  2 --
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 41 ++++++++++++++++++++--
 source/dnode/vnode/src/tsdb/tsdbUtil.c     | 17 ---------
 4 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 2e680511c3..a2ac995c13 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -115,7 +115,6 @@ int32_t tGetBlock(uint8_t *p, void *ph);
 int32_t tBlockCmprFn(const void *p1, const void *p2);
 bool    tBlockHasSma(SBlock *pBlock);
 // SBlockIdx
-void    tBlockIdxReset(SBlockIdx *pBlockIdx);
 int32_t tPutBlockIdx(uint8_t *p, void *ph);
 int32_t tGetBlockIdx(uint8_t *p, void *ph);
 int32_t tCmprBlockIdx(void const *lhs, void const *rhs);
@@ -361,10 +360,6 @@ struct TSDBROW {
 struct SBlockIdx {
   int64_t suid;
   int64_t uid;
-  TSKEY   minKey;
-  TSKEY   maxKey;
-  int64_t minVersion;
-  int64_t maxVersion;
   int64_t offset;
   int64_t size;
 };
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index b3e0695c10..723caca5d7 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -500,9 +500,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
       if (code) goto _err;
 
       /* if (state->pBlockIdx) { */
-      /*   tBlockIdxReset(state->blockIdx); */
       /* } */
-      /* tBlockIdxReset(state->blockIdx); */
       /* code = tMapDataSearch(&state->blockIdxMap, state->pBlockIdxExp, tGetBlockIdx, tCmprBlockIdx,
        * &state->blockIdx);
        */
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 193f4997b0..f3b321dfae 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -383,14 +383,15 @@ struct STsdbSnapWriter {
   int32_t       iRow;
 
   SDataFWriter* pDataFWriter;
-  SArray*       aBlockIdxW;  // SArray<SBlockIdx>
   SBlockIdx*    pBlockIdxW;
   SBlockIdx     blockIdx;
-  SMapData      mBlockW;  // SMapData<SBlock>
   SBlock*       pBlockW;
   SBlock        blockW;
   SBlockData    bDataW;
 
+  SMapData mBlockW;     // SMapData<SBlock>
+  SArray*  aBlockIdxW;  // SArray<SBlockIdx>
+
   // for del file
   SDelFReader* pDelFReader;
   SDelFWriter* pDelFWriter;
@@ -460,6 +461,7 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWrite) {
   return code;
 }
 
+#if 0
 static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t code = 0;
   TABLEID id = {0};  // TODO
@@ -556,6 +558,38 @@ _err:
   tsdbError("vgId:%d tsdb snapshot write table data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
+#endif
+
+static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
+  int32_t     code = 0;
+  SBlockData* pBlockData = &pWriter->bData;
+
+  if (pWriter->pDataFReader == NULL) {
+    // no old data
+
+    // end last table data commit if id not same
+    if (pWriter->pBlockIdxW) {
+      int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id);
+      if (c < 0) {
+        // commit last table data and reset (todo)
+        pWriter->pBlockIdxW = NULL;
+      } else if (c > 0) {
+        ASSERT(0);
+      }
+    }
+
+    // start a new table data if need
+    if (pWriter->pBlockIdxW == NULL) {
+      pWriter->pBlockIdxW = &pWriter->blockIdx;
+      pWriter->pBlockIdxW->suid = id.suid;
+      pWriter->pBlockIdxW->uid = id.uid;
+    }
+
+  } else {
+  }
+
+  return code;
+}
 
 static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t       code = 0;
@@ -630,7 +664,8 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     tBlockDataReset(&pWriter->bDataW);
   }
 
-  // write data block (todo)
+  code = tsdbSnapWriteDataImpl(pWriter, id);
+  if (code) goto _err;
 
   tsdbInfo("vgId:%d vnode snapshot tsdb write data, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d",
            TD_VID(pTsdb->pVnode), fid, id.suid, id.suid, pBlockData->nRow);
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 7667127fc9..be1b848d96 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -189,25 +189,12 @@ static FORCE_INLINE int32_t tGetTSDBKEY(uint8_t *p, TSDBKEY *pKey) {
 }
 
 // SBlockIdx ======================================================
-void tBlockIdxReset(SBlockIdx *pBlockIdx) {
-  pBlockIdx->minKey = TSKEY_MAX;
-  pBlockIdx->maxKey = TSKEY_MIN;
-  pBlockIdx->minVersion = VERSION_MAX;
-  pBlockIdx->maxVersion = VERSION_MIN;
-  pBlockIdx->offset = -1;
-  pBlockIdx->size = -1;
-}
-
 int32_t tPutBlockIdx(uint8_t *p, void *ph) {
   int32_t    n = 0;
   SBlockIdx *pBlockIdx = (SBlockIdx *)ph;
 
   n += tPutI64(p ? p + n : p, pBlockIdx->suid);
   n += tPutI64(p ? p + n : p, pBlockIdx->uid);
-  n += tPutI64(p ? p + n : p, pBlockIdx->minKey);
-  n += tPutI64(p ? p + n : p, pBlockIdx->maxKey);
-  n += tPutI64v(p ? p + n : p, pBlockIdx->minVersion);
-  n += tPutI64v(p ? p + n : p, pBlockIdx->maxVersion);
   n += tPutI64v(p ? p + n : p, pBlockIdx->offset);
   n += tPutI64v(p ? p + n : p, pBlockIdx->size);
 
@@ -220,10 +207,6 @@ int32_t tGetBlockIdx(uint8_t *p, void *ph) {
 
   n += tGetI64(p + n, &pBlockIdx->suid);
   n += tGetI64(p + n, &pBlockIdx->uid);
-  n += tGetI64(p + n, &pBlockIdx->minKey);
-  n += tGetI64(p + n, &pBlockIdx->maxKey);
-  n += tGetI64v(p + n, &pBlockIdx->minVersion);
-  n += tGetI64v(p + n, &pBlockIdx->maxVersion);
   n += tGetI64v(p + n, &pBlockIdx->offset);
   n += tGetI64v(p + n, &pBlockIdx->size);
 

From 20fb3563889666d13546e3ac8960ea85ae8efcdb Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 13 Jul 2022 12:31:18 +0000
Subject: [PATCH 10/38] more vnode snap writer

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 37 ++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index f3b321dfae..8927155173 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -583,12 +583,49 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
       pWriter->pBlockIdxW = &pWriter->blockIdx;
       pWriter->pBlockIdxW->suid = id.suid;
       pWriter->pBlockIdxW->uid = id.uid;
+
+      pWriter->pBlockW = &pWriter->blockW;
+      tBlockReset(pWriter->pBlockW);
+
+      tBlockDataReset(&pWriter->bDataW);
+
+      tMapDataReset(&pWriter->mBlockW);
     }
 
+    // set block schema (todo)
+
+    // add rows
+    for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
+      TSDBROW* pRow = &tsdbRowFromBlockData(pBlockData, iRow);
+
+      code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
+      if (code) goto _err;
+
+      if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) {
+        // write the block to file
+        pWriter->pBlockW->last = 0;
+
+        code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
+                                  pWriter->pBlockW, pWriter->cmprAlg);
+        if (code) goto _err;
+
+        code = tMapDataPutItem(&pWriter->mBlockW, pWriter->pBlockW, tPutBlock);
+        if (code) goto _err;
+
+        // reset
+        tBlockReset(pWriter->pBlockW);
+        tBlockDataReset(&pWriter->bDataW);
+      }
+    }
   } else {
   }
 
   return code;
+
+_err:
+  tsdbError("vgId:%d vnode snapshot tsdb write data impl failed since %s", TD_VID(pWriter->pTsdb->pVnode),
+            tstrerror(code));
+  return code;
 }
 
 static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {

From 25a2d04b85c669c7d4c68f9add1bc26cb767f6a0 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 14 Jul 2022 07:37:53 +0000
Subject: [PATCH 11/38] more vnode snapshot writer

---
 source/dnode/vnode/inc/vnode.h             |   2 +-
 source/dnode/vnode/src/inc/tsdb.h          |   1 +
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 153 ++++++---------------
 source/dnode/vnode/src/tsdb/tsdbUtil.c     |  40 ++++++
 4 files changed, 85 insertions(+), 111 deletions(-)

diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index f0a6b6505d..6b5f795eb0 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -138,7 +138,7 @@ void   *tsdbGetIdx(SMeta *pMeta);
 void   *tsdbGetIvtIdx(SMeta *pMeta);
 
 int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, void **pReader);
-int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray* pTableUids);
+int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids);
 int32_t tsdbLastrowReaderClose(void *pReader);
 int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
 
diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index a2ac995c13..3f667a7465 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -135,6 +135,7 @@ int32_t tGetColData(uint8_t *p, SColData *pColData);
 int32_t   tBlockDataInit(SBlockData *pBlockData);
 void      tBlockDataReset(SBlockData *pBlockData);
 int32_t   tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema);
+int32_t   tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom);
 void      tBlockDataClearData(SBlockData *pBlockData);
 void      tBlockDataClear(SBlockData *pBlockData, int8_t deepClear);
 int32_t   tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData);
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 8927155173..5c5e81cd49 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -383,11 +383,10 @@ struct STsdbSnapWriter {
   int32_t       iRow;
 
   SDataFWriter* pDataFWriter;
-  SBlockIdx*    pBlockIdxW;
-  SBlockIdx     blockIdx;
-  SBlock*       pBlockW;
+  SBlockIdx*    pBlockIdxW;  // NULL when no committing table
   SBlock        blockW;
   SBlockData    bDataW;
+  SBlockIdx     blockIdxW;
 
   SMapData mBlockW;     // SMapData<SBlock>
   SArray*  aBlockIdxW;  // SArray<SBlockIdx>
@@ -455,110 +454,40 @@ _err:
   return code;
 }
 
-static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWrite) {
+static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
   int32_t code = 0;
-  // TODO
-  return code;
-}
 
-#if 0
-static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
-  int32_t code = 0;
-  TABLEID id = {0};  // TODO
+  ASSERT(pWriter->pBlockIdxW != NULL);
 
-  // skip
-  while (pWriter->pBlockIdx && tTABLEIDCmprFn(&id, pWriter->pBlockIdx) < 0) {
-    code = tsdbSnapWriteTableDataEnd(pWriter);
+  // write remain data if has
+  if (pWriter->bDataW.nRow > 0) {
+    if (pWriter->bDataW.nRow >= pWriter->minRow) {
+      pWriter->blockW.last = 0;
+    } else {
+      pWriter->blockW.last = 1;
+    }
+
+    code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
+                              &pWriter->blockW, pWriter->cmprAlg);
     if (code) goto _err;
 
-    pWriter->iBlockIdx++;
-    if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
-      pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
-    } else {
-      pWriter->pBlockIdx = NULL;
-    }
+    code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
+    if (code) goto _err;
   }
 
-  // new or merge
-  if (pWriter->pBlockIdx == NULL || tTABLEIDCmprFn(&id, pWriter->pBlockIdx) < 0) {
-    int32_t c;
+  code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW);
+  if (code) goto _err;
 
-    if (pWriter->pBlockIdxW && ((c = tTABLEIDCmprFn(&id, pWriter->pBlockIdxW)) != 0)) {
-      ASSERT(c > 0);
-
-      code = tsdbSnapWriteTableDataEnd(pWriter);
-      if (code) goto _err;
-    }
-
-    if (pWriter->pBlockIdxW == NULL) {
-      pWriter->pBlockIdx = &pWriter->blockIdx;
-      pWriter->pBlockIdx->suid = id.suid;
-      pWriter->pBlockIdx->uid = id.uid;
-    }
-
-    // loop to write the data
-    TSDBROW*    pRow = NULL;        // todo
-    int32_t     nRow = 0;           // todo
-    SBlockData* pBlockData = NULL;  // todo
-    for (int32_t iRow = 0; iRow < nRow; iRow++) {
-      code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
-      if (code) goto _err;
-
-      if (pWriter->bDataW.nRow > pWriter->maxRow * 4 / 5) {
-        code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
-                                  pWriter->pBlockW, pWriter->cmprAlg);
-        if (code) goto _err;
-      }
-    }
-  } else {
-    // skip
-    while (true) {
-      if (pWriter->pBlock == NULL) break;
-      if (pWriter->pBlock->last) break;
-      if (tBlockCmprFn(&(SBlock){.minKey = {0}, .maxKey = {0}}, pWriter->pBlock) >= 0) break;
-
-      code = tMapDataPutItem(&pWriter->mBlockW, pWriter->pBlock, tPutBlock);
-      if (code) goto _err;
-    }
-
-    if (pWriter->pBlock) {
-      if (pWriter->pBlock->last) {
-        // load the last block and merge with the data (todo)
-      } else {
-        int32_t c = tBlockCmprFn(&(SBlock){0 /*TODO*/}, pWriter->pBlock);
-
-        if (c > 0) {
-          // commit until pWriter->pBlock (todo)
-        } else {
-          // load the block and merge with the data (todo)
-        }
-      }
-    } else {
-      int32_t     nRow = 0;
-      SBlockData* pBlockData = NULL;
-
-      for (int32_t iRow = 0; iRow < nRow; iRow++) {
-        code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL);
-        if (code) goto _err;
-
-        if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) {
-          code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
-                                    pWriter->pBlockW, pWriter->cmprAlg);
-          if (code) goto _err;
-
-          tBlockDataClearData(&pWriter->bDataW);
-        }
-      }
-    }
+  if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
   }
 
   return code;
 
 _err:
-  tsdbError("vgId:%d tsdb snapshot write table data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
-#endif
 
 static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
   int32_t     code = 0;
@@ -567,11 +496,16 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
   if (pWriter->pDataFReader == NULL) {
     // no old data
 
-    // end last table data commit if id not same
+    // end last table write if need
     if (pWriter->pBlockIdxW) {
-      int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id);
+      int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id);
+
       if (c < 0) {
-        // commit last table data and reset (todo)
+        // end last table data write
+        code = tsdbSnapWriteTableDataEnd(pWriter);
+        if (code) goto _err;
+
+        // reset
         pWriter->pBlockIdxW = NULL;
       } else if (c > 0) {
         ASSERT(0);
@@ -580,41 +514,40 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
 
     // start a new table data if need
     if (pWriter->pBlockIdxW == NULL) {
-      pWriter->pBlockIdxW = &pWriter->blockIdx;
+      pWriter->pBlockIdxW = &pWriter->blockIdxW;
       pWriter->pBlockIdxW->suid = id.suid;
       pWriter->pBlockIdxW->uid = id.uid;
 
-      pWriter->pBlockW = &pWriter->blockW;
-      tBlockReset(pWriter->pBlockW);
+      tBlockReset(&pWriter->blockW);
 
       tBlockDataReset(&pWriter->bDataW);
 
       tMapDataReset(&pWriter->mBlockW);
     }
 
-    // set block schema (todo)
+    // set block schema
+    code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData);
+    if (code) goto _err;
 
     // add rows
     for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
-      TSDBROW* pRow = &tsdbRowFromBlockData(pBlockData, iRow);
+      TSDBROW row = tsdbRowFromBlockData(pBlockData, iRow);
 
-      code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
+      code = tBlockDataAppendRow(&pWriter->bDataW, &row, NULL);
       if (code) goto _err;
 
       if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) {
-        // write the block to file
-        pWriter->pBlockW->last = 0;
-
+        pWriter->blockW.last = 0;
         code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
-                                  pWriter->pBlockW, pWriter->cmprAlg);
+                                  &pWriter->blockW, pWriter->cmprAlg);
         if (code) goto _err;
 
-        code = tMapDataPutItem(&pWriter->mBlockW, pWriter->pBlockW, tPutBlock);
+        code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
         if (code) goto _err;
 
         // reset
-        tBlockReset(pWriter->pBlockW);
-        tBlockDataReset(&pWriter->bDataW);
+        tBlockReset(&pWriter->blockW);
+        tBlockDataClearData(&pWriter->bDataW);
       }
     }
   } else {
@@ -647,6 +580,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
   int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision);
   ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision));
   if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) {
+    // end last file data write if need
     code = tsdbSnapWriteDataEnd(pWriter);  // todo
     if (code) goto _err;
 
@@ -697,7 +631,6 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     taosArrayClear(pWriter->aBlockIdxW);
     pWriter->pBlockIdxW = NULL;
     tMapDataReset(&pWriter->mBlockW);
-    pWriter->pBlockW = NULL;
     tBlockDataReset(&pWriter->bDataW);
   }
 
@@ -709,7 +642,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
   return code;
 
 _err:
-  tsdbError("vgId:%d tsdb snapshot write data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d vnode snapshot tsdb write data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index be1b848d96..6320a48e77 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -1132,6 +1132,46 @@ _err:
   return code;
 }
 
+int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom) {
+  int32_t code = 0;
+
+  int32_t iColData = 0;
+  for (int32_t iColDataFrom = 0; iColDataFrom < taosArrayGetSize(pBlockDataFrom->aIdx); iColDataFrom++) {
+    SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColDataFrom);
+
+    while (true) {
+      SColData *pColData;
+      if (iColData < taosArrayGetSize(pBlockData->aIdx)) {
+        pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
+      } else {
+        pColData = NULL;
+      }
+
+      if (pColData == NULL || pColData->cid > pColDataFrom->cid) {
+        code = tBlockDataAddColData(pBlockData, iColData, &pColData);
+        if (code) goto _exit;
+
+        tColDataInit(pColData, pColDataFrom->cid, pColData->type, pColData->smaOn);
+        for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
+          code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type));
+          if (code) goto _exit;
+        }
+
+        iColData++;
+        break;
+      } else if (pColData->cid == pColDataFrom->cid) {
+        iColData++;
+        break;
+      } else {
+        iColData++;
+      }
+    }
+  }
+
+_exit:
+  return code;
+}
+
 int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData) {
   int32_t code = 0;
 

From 4ad539f24291f7f92a4be14c3387ce0dc958e79d Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 14 Jul 2022 16:27:26 +0800
Subject: [PATCH 12/38] test:temp commit

---
 tests/system-test/2-query/queryQnode.py | 25 +++++++++++++++++++++++++
 tests/system-test/2-query/tsbsQuery.py  |  5 +++--
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/tests/system-test/2-query/queryQnode.py b/tests/system-test/2-query/queryQnode.py
index 3fdc09478d..0460ccf135 100644
--- a/tests/system-test/2-query/queryQnode.py
+++ b/tests/system-test/2-query/queryQnode.py
@@ -392,6 +392,31 @@ class TDTestCase:
         tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all  select c0,c1 from stb11_1 where c0>2000;")
         assert unionallQnode==tdSql.queryResult
 
+        queryPolicy=1
+
+        tdSql.execute('alter local  "queryPolicy" "%d"'%queryPolicy)
+        tdSql.query("show local variables;")
+        for i in range(tdSql.queryRows):
+            if tdSql.queryResult[i][0] == "queryPolicy" :
+                if int(tdSql.queryResult[i][1]) == int(queryPolicy):
+                    tdLog.success('alter queryPolicy to %d successfully'%queryPolicy)
+                else :
+                    tdLog.debug(tdSql.queryResult)
+                    tdLog.exit("alter queryPolicy to  %d failed"%queryPolicy)
+        tdSql.execute("reset query cache")
+
+        tdSql.execute("use db1;")
+        tdSql.query("show dnodes;")
+        dnodeId=tdSql.getData(0,0)
+        tdSql.query("select max(c1) from stb10;")
+        assert maxQnode==tdSql.getData(0,0)
+        tdSql.query("select min(c1) from stb11;")
+        assert minQnode==tdSql.getData(0,0)
+        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;")
+        assert unionQnode==tdSql.queryResult
+        tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all  select c0,c1 from stb11_1 where c0>2000;")
+        assert unionallQnode==tdSql.queryResult
+
     # test case : queryPolicy = 2
     def test_case2(self):
         self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 10, 2, 1*10)
diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py
index 8180f511e2..d04993e558 100644
--- a/tests/system-test/2-query/tsbsQuery.py
+++ b/tests/system-test/2-query/tsbsQuery.py
@@ -63,7 +63,8 @@ class TDTestCase:
         tdSql.execute("use db_tsbs")
         
         # test interval and partition
-        tdSql.query(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet; ")
+        tdSql.queryt(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet; ")
+        print(tdSql.queryResult)
         parRows=tdSql.queryRows
         tdSql.query(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet interval(10m); ")
         # tdSql.checkRows(parRows)
@@ -85,7 +86,7 @@ class TDTestCase:
         # tdSql.checkRows(10)
 
         # test partition interval Pseudo time-column
-        tdSql.query("SELECT count(ms1)/144  FROM (SELECT _wstartts as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'  partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;")
+        tdSql.queryt("SELECT count(ms1)/144  FROM (SELECT _wstartts as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'  partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;")
 
 
         # test

From b554559889adec4047d7242089aada54bccb5ae1 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 14 Jul 2022 09:36:23 +0000
Subject: [PATCH 13/38] more vnode snapshot

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 118 +++++++++++++++++++++
 1 file changed, 118 insertions(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 5c5e81cd49..51514f1257 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -489,9 +489,66 @@ _err:
   return code;
 }
 
+static int32_t tsdbSnapWriteTableDataAhead(STsdbSnapWriter* pWriter, TABLEID id) {
+  int32_t code = 0;
+
+  if (pWriter->pDataFReader == NULL) goto _exit;
+
+  while (true) {
+    if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break;
+
+    SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
+    int32_t    c = tTABLEIDCmprFn(pBlockIdx, &id);
+
+    if (c >= 0) break;
+
+    pWriter->iBlockIdx++;
+
+    code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL);
+    if (code) goto _err;
+
+    SBlock block;
+    tMapDataReset(&pWriter->mBlockW);
+    for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) {
+      tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock);
+
+      if (block.last) {
+        code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
+        if (code) goto _err;
+
+        tBlockReset(&block);
+        block.last = 1;
+        code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block,
+                                  pWriter->cmprAlg);
+        if (code) goto _err;
+      }
+
+      code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
+      if (code) goto _err;
+    }
+
+    SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
+    code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, &blockIdx);
+    if (code) goto _err;
+
+    if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+  }
+
+_exit:
+  return code;
+
+_err:
+  return code;
+}
+
 static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
   int32_t     code = 0;
   SBlockData* pBlockData = &pWriter->bData;
+  TSDBKEY     keyFirst = tBlockDataFirstKey(pBlockData);
+  TSDBKEY     keyLast = tBlockDataLastKey(pBlockData);
 
   if (pWriter->pDataFReader == NULL) {
     // no old data
@@ -551,6 +608,67 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
       }
     }
   } else {
+    // has old data
+
+    // TABLE ==================================================
+    // end last table data if id not same (todo)
+    if (pWriter->pBlockIdxW) {
+      int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id);
+      if (c < 0) {
+      } else if (c > 0) {
+        ASSERT(0);
+      }
+    }
+
+    // start new table data if need (todo)
+    if (pWriter->pBlockIdxW == NULL) {
+      // commit table data ahead
+      code = tsdbSnapWriteTableDataAhead(pWriter, id);
+      if (code) goto _err;
+
+      // reader
+      if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
+        pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
+        int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id);
+        if (c) {
+          pWriter->pBlockIdx = NULL;
+        }
+      } else {
+        pWriter->pBlockIdx = NULL;
+      }
+
+      if (pWriter->pBlockIdx) {
+        code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlockW, NULL);
+        if (code) goto _err;
+
+        pWriter->iBlock = 0;
+      }
+
+      // writer
+      pWriter->pBlockIdxW = &pWriter->blockIdxW;
+      pWriter->pBlockIdxW->suid = id.suid;
+      pWriter->pBlockIdxW->uid = id.uid;
+
+      tBlockReset(&pWriter->blockW);
+      tBlockDataReset(&pWriter->bDataW);
+      tMapDataReset(&pWriter->mBlockW);
+    }
+
+    // BLOCK ==================================================
+    // write block ahead
+    while (true) {
+      if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
+
+      SBlock block;
+      tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock);
+
+      if (tsdbKeyCmprFn(&block.maxKey, &keyFirst) >= 0) break;
+
+      pWriter->iBlock++;
+
+      code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
+      if (code) goto _err;
+    }
   }
 
   return code;

From e8a0cf5b75b11f029be943a7b49303f8f05dd53a Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 14 Jul 2022 21:11:39 +0800
Subject: [PATCH 14/38] test:add  test case of tsbs query

---
 tests/pytest/util/sql.py               | 47 ++++++++++++++++++--------
 tests/system-test/2-query/tsbsQuery.py | 15 ++++----
 2 files changed, 41 insertions(+), 21 deletions(-)

diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py
index 585594e035..bc6727a8fc 100644
--- a/tests/pytest/util/sql.py
+++ b/tests/pytest/util/sql.py
@@ -79,22 +79,39 @@ class TDSql:
             self.queryResult = None
             tdLog.info("sql:%s, expect error occured" % (sql))
 
-    def query(self, sql, row_tag=None):
+    def query(self, sql, row_tag=None,queyTimes=10):
         self.sql = sql
-        try:
-            self.cursor.execute(sql)
-            self.queryResult = self.cursor.fetchall()
-            self.queryRows = len(self.queryResult)
-            self.queryCols = len(self.cursor.description)
-        except Exception as e:
-            caller = inspect.getframeinfo(inspect.stack()[1][0])
-            args = (caller.filename, caller.lineno, sql, repr(e))
-            tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
-            traceback.print_exc()
-            raise Exception(repr(e))
-        if row_tag:
-            return self.queryResult
-        return self.queryRows
+        i=1
+        while i <= queyTimes:
+            try:
+                self.cursor.execute(sql)
+                self.queryResult = self.cursor.fetchall()
+                self.queryRows = len(self.queryResult)
+                self.queryCols = len(self.cursor.description)
+                if row_tag:
+                    return self.queryResult
+                return self.queryRows
+            except Exception as e:
+                i+=1
+                tdLog.notice("Try to query again, query times: %d "%i)
+                pass
+        else:
+            try:
+                tdLog.notice("Try the last query ")
+                self.cursor.execute(sql)
+                self.queryResult = self.cursor.fetchall()
+                self.queryRows = len(self.queryResult)
+                self.queryCols = len(self.cursor.description)
+                if row_tag:
+                    return self.queryResult
+                return self.queryRows
+            except Exception as e:    
+                caller = inspect.getframeinfo(inspect.stack()[1][0])
+                args = (caller.filename, caller.lineno, sql, repr(e))
+                tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
+                traceback.print_exc()
+                raise Exception(repr(e))
+
 
     def is_err_sql(self, sql):
         err_flag = True
diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py
index d04993e558..5fe025ea07 100644
--- a/tests/system-test/2-query/tsbsQuery.py
+++ b/tests/system-test/2-query/tsbsQuery.py
@@ -63,11 +63,11 @@ class TDTestCase:
         tdSql.execute("use db_tsbs")
         
         # test interval and partition
-        tdSql.queryt(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet; ")
+        tdSql.query(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet; ")
         print(tdSql.queryResult)
         parRows=tdSql.queryRows
         tdSql.query(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet interval(10m); ")
-        # tdSql.checkRows(parRows)
+        tdSql.checkRows(parRows)
         
         
         # test insert into 
@@ -78,16 +78,19 @@ class TDTestCase:
 
 
         # test paitition interval fill
-        # tdSql.query("SELECT name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings   WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'   partition by name interval(10m) fill(value,0) ;")
+        tdSql.query("SELECT name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings   WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'   partition by name interval(10m) fill(value,0) ;")
 
 
-        # # test partition interval limit 
-        # tdSql.query("SELECT ts,model,floor(2*(sum(nzs)/count(nzs)))/floor(2*(sum(nzs)/count(nzs))) AS broken_down FROM (SELECT ts,model, status/status AS nzs FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' ) WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'   partition BY model,ts interval(10m) limit 10;")
+        # test partition interval limit  (PRcore-TD-17410)
+        # tdSql.query("select name,driver from (SELECT name,driver,fleet ,avg(velocity) as mean_velocity FROM readings partition BY name,driver,fleet interval (10m) limit 1);")
         # tdSql.checkRows(10)
 
         # test partition interval Pseudo time-column
-        tdSql.queryt("SELECT count(ms1)/144  FROM (SELECT _wstartts as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'  partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;")
+        tdSql.query("SELECT count(ms1)/144  FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'  partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;")
 
+        #PR TD-17413
+        tdSql.query("SELECT ms1  FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'  partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-01T00:01:01Z';")
+        tdSql.checkRows(0)
 
         # test
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring

From d554bf6588c6cf35f6a315e05b9333efb076e1af Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Thu, 14 Jul 2022 21:22:43 +0800
Subject: [PATCH 15/38] test:modify test frame that add  multiple retries for
 sql execution and query

---
 tests/pytest/util/sql.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py
index bc6727a8fc..4961355f06 100644
--- a/tests/pytest/util/sql.py
+++ b/tests/pytest/util/sql.py
@@ -283,16 +283,27 @@ class TDSql:
                 time.sleep(1)
                 continue
 
-    def execute(self, sql):
+    def execute(self, sql,queyTimes=10):
         self.sql = sql
-        try:
-            self.affectedRows = self.cursor.execute(sql)
-        except Exception as e:
-            caller = inspect.getframeinfo(inspect.stack()[1][0])
-            args = (caller.filename, caller.lineno, sql, repr(e))
-            tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
-            raise Exception(repr(e))
-        return self.affectedRows
+        i=1
+        while i <= queyTimes:
+            try:
+                self.affectedRows = self.cursor.execute(sql)
+                return self.affectedRows
+            except Exception as e:
+                i+=1
+                tdLog.notice("Try to execute sql again, query times: %d "%i)
+                pass
+        else:               
+            try:
+                tdLog.notice("Try the last execute sql ")
+                self.affectedRows = self.cursor.execute(sql)
+                return self.affectedRows
+            except Exception as e:
+                caller = inspect.getframeinfo(inspect.stack()[1][0])
+                args = (caller.filename, caller.lineno, sql, repr(e))
+                tdLog.notice("%s(%d) failed: sql:%s, %s" % args)
+                raise Exception(repr(e))
 
     def checkAffectedRows(self, expectAffectedRows):
         if self.affectedRows != expectAffectedRows:

From 7b2dc59860a1a083bb928fba9dd267839a80ab29 Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Fri, 15 Jul 2022 10:42:47 +0800
Subject: [PATCH 16/38] test: add test case for precison

---
 tests/system-test/2-query/db.py | 63 +++++++++++++++++++++++++++++++++
 tests/system-test/fulltest.sh   |  1 +
 2 files changed, 64 insertions(+)
 create mode 100644 tests/system-test/2-query/db.py

diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py
new file mode 100644
index 0000000000..a4d603bada
--- /dev/null
+++ b/tests/system-test/2-query/db.py
@@ -0,0 +1,63 @@
+import taos
+import sys
+import datetime
+import inspect
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+import random
+
+
+class TDTestCase:
+    updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143,
+                     "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143,
+                     "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "udfDebugFlag": 143}
+
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def case1(self):
+        tdSql.execute("create database if not exists dbms precision 'ms'")
+        tdSql.execute("create database if not exists dbus precision 'us'")
+        tdSql.execute("create database if not exists dbns precision 'ns'")
+
+        tdSql.execute("create table dbms.ntb (ts timestamp, c1 int, c2 bigint)")
+        tdSql.execute("create table dbus.ntb (ts timestamp, c1 int, c2 bigint)")
+        tdSql.execute("create table dbns.ntb (ts timestamp, c1 int, c2 bigint)")
+        
+        tdSql.execute("insert into dbms.ntb values ('2022-01-01 08:00:00.001', 1, 2)")
+        tdSql.execute("insert into dbms.ntb values ('2022-01-01 08:00:00.002', 3, 4)")
+        
+        tdSql.execute("insert into dbus.ntb values ('2022-01-01 08:00:00.000001', 1, 2)")
+        tdSql.execute("insert into dbus.ntb values ('2022-01-01 08:00:00.000002', 3, 4)")
+        
+        tdSql.execute("insert into dbns.ntb values ('2022-01-01 08:00:00.000000001', 1, 2)")
+        tdSql.execute("insert into dbns.ntb values ('2022-01-01 08:00:00.000000002', 3, 4)")
+        
+        tdSql.query("select count(c1) from dbms.ntb interval(1a)")
+        tdSql.checkRows(2)
+        
+        tdSql.query("select count(c1) from dbus.ntb interval(1u)")
+        tdSql.checkRows(2)
+        
+        tdSql.query("select count(c1) from dbns.ntb interval(1b)")
+        tdSql.checkRows(2)
+
+    def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========start case1 run ...............")
+
+        self.case1()
+
+        tdLog.printNoPrefix("==========end case1 run ...............")
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index 8bafe3c966..7a72dc75b2 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -29,6 +29,7 @@ python3 ./test.py -f 1-insert/block_wise.py
 python3 ./test.py -f 1-insert/create_retentions.py
 python3 ./test.py -f 1-insert/table_param_ttl.py
 
+python3 ./test.py -f 2-query/db.py
 python3 ./test.py -f 2-query/between.py
 python3 ./test.py -f 2-query/distinct.py
 python3 ./test.py -f 2-query/varchar.py

From ec2a069185934e9db7441092fd0c0a7b87a6333f Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 15 Jul 2022 06:53:56 +0000
Subject: [PATCH 17/38] more vnode snapshot writer

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 280 ++++++++++++---------
 source/dnode/vnode/src/tsdb/tsdbUtil.c     |   6 +-
 2 files changed, 164 insertions(+), 122 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 51514f1257..d9a67ce487 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -377,10 +377,9 @@ struct STsdbSnapWriter {
   SBlockIdx*    pBlockIdx;
   SMapData      mBlock;  // SMapData<SBlock>
   int32_t       iBlock;
-  SBlock*       pBlock;
-  SBlock        block;
-  SBlockData    bDataR;
+  SBlockData*   pBlockData;
   int32_t       iRow;
+  SBlockData    bDataR;
 
   SDataFWriter* pDataFWriter;
   SBlockIdx*    pBlockIdxW;  // NULL when no committing table
@@ -550,127 +549,171 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
   TSDBKEY     keyFirst = tBlockDataFirstKey(pBlockData);
   TSDBKEY     keyLast = tBlockDataLastKey(pBlockData);
 
-  if (pWriter->pDataFReader == NULL) {
-    // no old data
+  // TABLE ====================================
 
-    // end last table write if need
-    if (pWriter->pBlockIdxW) {
-      int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id);
-
-      if (c < 0) {
-        // end last table data write
-        code = tsdbSnapWriteTableDataEnd(pWriter);
-        if (code) goto _err;
-
-        // reset
-        pWriter->pBlockIdxW = NULL;
-      } else if (c > 0) {
-        ASSERT(0);
-      }
-    }
-
-    // start a new table data if need
-    if (pWriter->pBlockIdxW == NULL) {
-      pWriter->pBlockIdxW = &pWriter->blockIdxW;
-      pWriter->pBlockIdxW->suid = id.suid;
-      pWriter->pBlockIdxW->uid = id.uid;
-
-      tBlockReset(&pWriter->blockW);
-
-      tBlockDataReset(&pWriter->bDataW);
-
-      tMapDataReset(&pWriter->mBlockW);
-    }
-
-    // set block schema
-    code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData);
-    if (code) goto _err;
-
-    // add rows
-    for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
-      TSDBROW row = tsdbRowFromBlockData(pBlockData, iRow);
-
-      code = tBlockDataAppendRow(&pWriter->bDataW, &row, NULL);
+  // end last table write if should
+  if (pWriter->pBlockIdxW) {
+    int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id);
+    if (c < 0) {
+      // end
+      code = tsdbSnapWriteTableDataEnd(pWriter);
       if (code) goto _err;
 
-      if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) {
-        pWriter->blockW.last = 0;
-        code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
-                                  &pWriter->blockW, pWriter->cmprAlg);
-        if (code) goto _err;
-
-        code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
-        if (code) goto _err;
-
-        // reset
-        tBlockReset(&pWriter->blockW);
-        tBlockDataClearData(&pWriter->bDataW);
-      }
-    }
-  } else {
-    // has old data
-
-    // TABLE ==================================================
-    // end last table data if id not same (todo)
-    if (pWriter->pBlockIdxW) {
-      int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id);
-      if (c < 0) {
-      } else if (c > 0) {
-        ASSERT(0);
-      }
-    }
-
-    // start new table data if need (todo)
-    if (pWriter->pBlockIdxW == NULL) {
-      // commit table data ahead
-      code = tsdbSnapWriteTableDataAhead(pWriter, id);
-      if (code) goto _err;
-
-      // reader
-      if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
-        pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
-        int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id);
-        if (c) {
-          pWriter->pBlockIdx = NULL;
-        }
-      } else {
-        pWriter->pBlockIdx = NULL;
-      }
-
-      if (pWriter->pBlockIdx) {
-        code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlockW, NULL);
-        if (code) goto _err;
-
-        pWriter->iBlock = 0;
-      }
-
-      // writer
-      pWriter->pBlockIdxW = &pWriter->blockIdxW;
-      pWriter->pBlockIdxW->suid = id.suid;
-      pWriter->pBlockIdxW->uid = id.uid;
-
-      tBlockReset(&pWriter->blockW);
-      tBlockDataReset(&pWriter->bDataW);
-      tMapDataReset(&pWriter->mBlockW);
-    }
-
-    // BLOCK ==================================================
-    // write block ahead
-    while (true) {
-      if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
-
-      SBlock block;
-      tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock);
-
-      if (tsdbKeyCmprFn(&block.maxKey, &keyFirst) >= 0) break;
-
-      pWriter->iBlock++;
-
-      code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
-      if (code) goto _err;
+      // reset
+      pWriter->pBlockIdxW = NULL;
+    } else if (c > 0) {
+      ASSERT(0);
     }
   }
 
+  // start new table data write if need
+  if (pWriter->pBlockIdxW == NULL) {
+    // write table data ahead
+    code = tsdbSnapWriteTableDataAhead(pWriter, id);
+    if (code) goto _err;
+
+    // reader
+    if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
+      ASSERT(pWriter->pDataFReader);
+
+      pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlock);
+      int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id);
+      if (c) {
+        ASSERT(c > 0);
+        pWriter->pBlockIdx = NULL;
+      } else {
+        pWriter->iBlockIdx++;
+      }
+    } else {
+      pWriter->pBlockIdx = NULL;
+    }
+
+    if (pWriter->pBlockIdx) {
+      code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock, NULL);
+      if (code) goto _err;
+    } else {
+      tMapDataReset(&pWriter->mBlock);
+    }
+    pWriter->iBlock = 0;
+    pWriter->pBlockData = NULL;
+    pWriter->iRow = 0;
+
+    // writer
+    pWriter->pBlockIdxW = &pWriter->blockIdxW;
+    pWriter->pBlockIdxW->suid = id.suid;
+    pWriter->pBlockIdxW->uid = id.uid;
+
+    tBlockReset(&pWriter->blockW);
+    tBlockDataReset(&pWriter->bDataW);
+    tMapDataReset(&pWriter->mBlockW);
+  }
+
+  ASSERT(pWriter->pBlockIdxW && pWriter->pBlockIdxW->suid == id.suid && pWriter->pBlockIdxW->uid == id.uid);
+  ASSERT(pWriter->pBlockIdx == NULL || (pWriter->pBlockIdx->suid == id.suid && pWriter->pBlockIdx->uid == id.uid));
+
+  // BLOCK ====================================
+  int32_t  iRow = 0;
+  TSDBROW* pRow = &tsdbRowFromBlockData(pBlockData, iRow);
+  while (true) {
+    if (pRow == NULL) break;
+
+    if (pWriter->pBlockData) {
+      ASSERT(pWriter->iRow < pWriter->pBlockData->nRow);
+
+      int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow));
+
+      ASSERT(c);
+
+      if (c < 0) {
+        code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
+        if (code) goto _err;
+
+        iRow++;
+        if (iRow < pWriter->pBlockData->nRow) {
+          pRow = &tsdbRowFromBlockData(pBlockData, iRow);
+        } else {
+          pRow = NULL;
+        }
+      } else if (c > 0) {
+        code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL);
+        if (code) goto _err;
+
+        pWriter->iRow++;
+        if (pWriter->iRow >= pWriter->pBlockData->nRow) {
+          pWriter->pBlockData = NULL;
+        }
+      }
+    } else {
+      SBlock block;
+
+      if (pWriter->iBlock < pWriter->mBlock.nItem) {
+        tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock);
+        int32_t c;
+
+        c = tsdbKeyCmprFn(&block.maxKey, &TSDBROW_KEY(pRow));
+
+        ASSERT(c);
+
+        if (c < 0) {
+          code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
+          if (code) goto _err;
+
+          pWriter->iBlock++;
+          continue;
+        }
+
+        c = tsdbKeyCmprFn(&block.minKey, &TSDBROW_KEY(pRow));
+
+        ASSERT(c);
+
+        if (c > 0) {
+          code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
+          if (code) goto _err;
+
+          iRow++;
+          if (iRow < pWriter->pBlockData->nRow) {
+            pRow = &tsdbRowFromBlockData(pBlockData, iRow);
+          } else {
+            pRow = NULL;
+          }
+
+          goto _check_write;
+        }
+
+        pWriter->pBlockData = &pWriter->bDataR;
+        code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL);
+        if (code) goto _err;
+
+        pWriter->iRow = 0;
+      }
+
+      code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
+      if (code) goto _err;
+
+      iRow++;
+      if (iRow < pWriter->pBlockData->nRow) {
+        pRow = &tsdbRowFromBlockData(pBlockData, iRow);
+      } else {
+        pRow = NULL;
+      }
+    }
+
+  _check_write:
+    if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue;
+
+  _write_block:
+    code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdx, &pWriter->blockW,
+                              pWriter->cmprAlg);
+    if (code) goto _err;
+
+    code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
+    if (code) goto _err;
+
+    tBlockReset(&pWriter->blockW);
+    tBlockDataClearData(&pWriter->bDataW);
+  }
+
+_exit:
   return code;
 
 _err:
@@ -720,7 +763,6 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     pWriter->pBlockIdx = NULL;
     tMapDataReset(&pWriter->mBlock);
     pWriter->iBlock = 0;
-    pWriter->pBlock = NULL;
     tBlockDataReset(&pWriter->bDataR);
     pWriter->iRow = 0;
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 6320a48e77..04368ddfb7 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -36,15 +36,15 @@ int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(u
 
   // alloc
   code = tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem);
-  if (code) goto _err;
+  if (code) goto _exit;
   code = tRealloc(&pMapData->pData, pMapData->nData);
-  if (code) goto _err;
+  if (code) goto _exit;
 
   // put
   pMapData->aOffset[nItem] = offset;
   tPutItemFn(pMapData->pData + offset, pItem);
 
-_err:
+_exit:
   return code;
 }
 

From 7e45878581e0a06ca2c19eefc122cf11f8a026b9 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 15 Jul 2022 10:03:18 +0000
Subject: [PATCH 18/38] more vnode snapshot

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 342 +++++++++++----------
 1 file changed, 182 insertions(+), 160 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index d9a67ce487..932cf9f9ae 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -488,52 +488,42 @@ _err:
   return code;
 }
 
-static int32_t tsdbSnapWriteTableDataAhead(STsdbSnapWriter* pWriter, TABLEID id) {
+static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* pBlockIdx) {
   int32_t code = 0;
 
-  if (pWriter->pDataFReader == NULL) goto _exit;
+  code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL);
+  if (code) goto _err;
 
-  while (true) {
-    if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break;
+  // SBlockData
+  SBlock block;
+  tMapDataReset(&pWriter->mBlockW);
+  for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) {
+    tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock);
 
-    SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
-    int32_t    c = tTABLEIDCmprFn(pBlockIdx, &id);
+    if (block.last) {
+      code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
+      if (code) goto _err;
 
-    if (c >= 0) break;
-
-    pWriter->iBlockIdx++;
-
-    code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL);
-    if (code) goto _err;
-
-    SBlock block;
-    tMapDataReset(&pWriter->mBlockW);
-    for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) {
-      tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock);
-
-      if (block.last) {
-        code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
-        if (code) goto _err;
-
-        tBlockReset(&block);
-        block.last = 1;
-        code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block,
-                                  pWriter->cmprAlg);
-        if (code) goto _err;
-      }
-
-      code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
+      tBlockReset(&block);
+      block.last = 1;
+      code =
+          tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, pWriter->cmprAlg);
       if (code) goto _err;
     }
 
-    SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
-    code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, &blockIdx);
+    code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
     if (code) goto _err;
+  }
 
-    if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      goto _err;
-    }
+  // SBlock
+  SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
+  code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, &blockIdx);
+  if (code) goto _err;
+
+  // SBlockIdx
+  if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
   }
 
 _exit:
@@ -543,14 +533,135 @@ _err:
   return code;
 }
 
-static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
+static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
+  int32_t     code = 0;
+  SBlockData* pBlockData = &pWriter->bData;
+  int32_t     iRow = 0;
+  TSDBROW     row;
+  TSDBROW*    pRow = &row;
+
+  // correct schema
+  code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData);
+  if (code) goto _err;
+
+  // loop to merge
+  *pRow = tsdbRowFromBlockData(pBlockData, iRow);
+  while (true) {
+    if (pRow == NULL) break;
+
+    if (pWriter->pBlockData) {
+      ASSERT(pWriter->iRow < pWriter->pBlockData->nRow);
+
+      int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow));
+
+      ASSERT(c);
+
+      if (c < 0) {
+        code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
+        if (code) goto _err;
+
+        iRow++;
+        if (iRow < pWriter->pBlockData->nRow) {
+          *pRow = tsdbRowFromBlockData(pBlockData, iRow);
+        } else {
+          pRow = NULL;
+        }
+      } else if (c > 0) {
+        code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL);
+        if (code) goto _err;
+
+        pWriter->iRow++;
+        if (pWriter->iRow >= pWriter->pBlockData->nRow) {
+          pWriter->pBlockData = NULL;
+        }
+      }
+    } else {
+      TSDBKEY key = TSDBROW_KEY(pRow);
+
+      while (true) {
+        if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
+
+        SBlock  block;
+        int32_t c;
+
+        tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock);
+
+        if (block.last) {
+          pWriter->pBlockData = &pWriter->bDataR;
+
+          code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL);
+          if (code) goto _err;
+          pWriter->iRow = 0;
+
+          pWriter->iBlock++;
+          break;
+        }
+
+        c = tsdbKeyCmprFn(&block.maxKey, &key);
+
+        ASSERT(c);
+
+        if (c > 0) break;
+
+        if (pWriter->bDataW.nRow) {
+          pWriter->blockW.last = 0;
+          code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
+                                    &pWriter->blockW, pWriter->cmprAlg);
+          if (code) goto _err;
+
+          code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
+          if (code) goto _err;
+
+          tBlockReset(&pWriter->blockW);
+          tBlockDataClearData(&pWriter->bDataW);
+        }
+
+        code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
+        if (code) goto _err;
+
+        pWriter->iBlock++;
+      }
+
+      if (pWriter->pBlockData) continue;
+
+      code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
+      if (code) goto _err;
+
+      iRow++;
+      if (iRow < pBlockData->nRow) {
+        *pRow = tsdbRowFromBlockData(pBlockData, iRow);
+      } else {
+        pRow = NULL;
+      }
+    }
+
+  _check_write:
+    if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue;
+
+  _write_block:
+    code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdx, &pWriter->blockW,
+                              pWriter->cmprAlg);
+    if (code) goto _err;
+
+    code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
+    if (code) goto _err;
+
+    tBlockReset(&pWriter->blockW);
+    tBlockDataClearData(&pWriter->bDataW);
+  }
+
+  return code;
+
+_err:
+  return code;
+}
+
+static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) {
   int32_t     code = 0;
   SBlockData* pBlockData = &pWriter->bData;
   TSDBKEY     keyFirst = tBlockDataFirstKey(pBlockData);
   TSDBKEY     keyLast = tBlockDataLastKey(pBlockData);
 
-  // TABLE ====================================
-
   // end last table write if should
   if (pWriter->pBlockIdxW) {
     int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id);
@@ -569,23 +680,33 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
   // start new table data write if need
   if (pWriter->pBlockIdxW == NULL) {
     // write table data ahead
-    code = tsdbSnapWriteTableDataAhead(pWriter, id);
-    if (code) goto _err;
+    while (true) {
+      if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break;
+
+      SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
+      int32_t    c = tTABLEIDCmprFn(pBlockIdx, &id);
+
+      if (c >= 0) break;
+
+      code = tsdbSnapMoveWriteTableData(pWriter, pBlockIdx);
+      if (code) goto _err;
+
+      pWriter->iBlockIdx++;
+    }
 
     // reader
     if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
       ASSERT(pWriter->pDataFReader);
 
-      pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlock);
-      int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdx, &id);
-      if (c) {
-        ASSERT(c > 0);
-        pWriter->pBlockIdx = NULL;
-      } else {
+      SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlock);
+      int32_t    c = tTABLEIDCmprFn(pBlockIdx, &id);
+
+      ASSERT(c >= 0);
+
+      if (c == 0) {
+        pWriter->pBlockIdx = pBlockIdx;
         pWriter->iBlockIdx++;
       }
-    } else {
-      pWriter->pBlockIdx = NULL;
     }
 
     if (pWriter->pBlockIdx) {
@@ -611,107 +732,8 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
   ASSERT(pWriter->pBlockIdxW && pWriter->pBlockIdxW->suid == id.suid && pWriter->pBlockIdxW->uid == id.uid);
   ASSERT(pWriter->pBlockIdx == NULL || (pWriter->pBlockIdx->suid == id.suid && pWriter->pBlockIdx->uid == id.uid));
 
-  // BLOCK ====================================
-  int32_t  iRow = 0;
-  TSDBROW* pRow = &tsdbRowFromBlockData(pBlockData, iRow);
-  while (true) {
-    if (pRow == NULL) break;
-
-    if (pWriter->pBlockData) {
-      ASSERT(pWriter->iRow < pWriter->pBlockData->nRow);
-
-      int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow));
-
-      ASSERT(c);
-
-      if (c < 0) {
-        code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
-        if (code) goto _err;
-
-        iRow++;
-        if (iRow < pWriter->pBlockData->nRow) {
-          pRow = &tsdbRowFromBlockData(pBlockData, iRow);
-        } else {
-          pRow = NULL;
-        }
-      } else if (c > 0) {
-        code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL);
-        if (code) goto _err;
-
-        pWriter->iRow++;
-        if (pWriter->iRow >= pWriter->pBlockData->nRow) {
-          pWriter->pBlockData = NULL;
-        }
-      }
-    } else {
-      SBlock block;
-
-      if (pWriter->iBlock < pWriter->mBlock.nItem) {
-        tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock);
-        int32_t c;
-
-        c = tsdbKeyCmprFn(&block.maxKey, &TSDBROW_KEY(pRow));
-
-        ASSERT(c);
-
-        if (c < 0) {
-          code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
-          if (code) goto _err;
-
-          pWriter->iBlock++;
-          continue;
-        }
-
-        c = tsdbKeyCmprFn(&block.minKey, &TSDBROW_KEY(pRow));
-
-        ASSERT(c);
-
-        if (c > 0) {
-          code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
-          if (code) goto _err;
-
-          iRow++;
-          if (iRow < pWriter->pBlockData->nRow) {
-            pRow = &tsdbRowFromBlockData(pBlockData, iRow);
-          } else {
-            pRow = NULL;
-          }
-
-          goto _check_write;
-        }
-
-        pWriter->pBlockData = &pWriter->bDataR;
-        code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL);
-        if (code) goto _err;
-
-        pWriter->iRow = 0;
-      }
-
-      code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
-      if (code) goto _err;
-
-      iRow++;
-      if (iRow < pWriter->pBlockData->nRow) {
-        pRow = &tsdbRowFromBlockData(pBlockData, iRow);
-      } else {
-        pRow = NULL;
-      }
-    }
-
-  _check_write:
-    if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue;
-
-  _write_block:
-    code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdx, &pWriter->blockW,
-                              pWriter->cmprAlg);
-    if (code) goto _err;
-
-    code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
-    if (code) goto _err;
-
-    tBlockReset(&pWriter->blockW);
-    tBlockDataClearData(&pWriter->bDataW);
-  }
+  code = tsdbSnapWriteTableDataImpl(pWriter);
+  if (code) goto _err;
 
 _exit:
   return code;
@@ -723,11 +745,10 @@ _err:
 }
 
 static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
-  int32_t       code = 0;
-  STsdb*        pTsdb = pWriter->pTsdb;
-  SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
-  TABLEID       id = *(TABLEID*)(&pHdr[1]);
-  int64_t       n;
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+  TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr));
+  int64_t n;
 
   // decode
   SBlockData* pBlockData = &pWriter->bData;
@@ -742,13 +763,13 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
   ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision));
   if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) {
     // end last file data write if need
-    code = tsdbSnapWriteDataEnd(pWriter);  // todo
+    code = tsdbSnapWriteDataEnd(pWriter);
     if (code) goto _err;
 
     pWriter->fid = fid;
 
     // read
-    SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ);  // todo: check nState is valid
+    SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ);
     if (pSet) {
       code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
       if (code) goto _err;
@@ -763,8 +784,9 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     pWriter->pBlockIdx = NULL;
     tMapDataReset(&pWriter->mBlock);
     pWriter->iBlock = 0;
-    tBlockDataReset(&pWriter->bDataR);
+    pWriter->pBlockData = NULL;
     pWriter->iRow = 0;
+    tBlockDataReset(&pWriter->bDataR);
 
     // write
     SDFileSet wSet;
@@ -789,12 +811,12 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     if (code) goto _err;
 
     taosArrayClear(pWriter->aBlockIdxW);
-    pWriter->pBlockIdxW = NULL;
     tMapDataReset(&pWriter->mBlockW);
+    pWriter->pBlockIdxW = NULL;
     tBlockDataReset(&pWriter->bDataW);
   }
 
-  code = tsdbSnapWriteDataImpl(pWriter, id);
+  code = tsdbSnapWriteTableData(pWriter, id);
   if (code) goto _err;
 
   tsdbInfo("vgId:%d vnode snapshot tsdb write data, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d",

From 989159f0ba4510c2a07eb1dfad0796b9f2854269 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Fri, 15 Jul 2022 13:02:24 +0000
Subject: [PATCH 19/38] vnod snapshot done

---
 source/dnode/vnode/src/tsdb/tsdbRead.c     |  23 ++-
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 184 +++++++++++++++------
 2 files changed, 145 insertions(+), 62 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index f0aea0cefb..7b4127e591 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -63,10 +63,10 @@ typedef struct SBlockLoadSuppInfo {
 } SBlockLoadSuppInfo;
 
 typedef struct SFilesetIter {
-  int32_t          numOfFiles;  // number of total files
-  int32_t          index;       // current accessed index in the list
-  SArray*          pFileList;   // data file list
-  int32_t          order;
+  int32_t numOfFiles;  // number of total files
+  int32_t index;       // current accessed index in the list
+  SArray* pFileList;   // data file list
+  int32_t order;
 } SFilesetIter;
 
 typedef struct SFileDataBlockInfo {
@@ -830,8 +830,8 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
   SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
 
-  int32_t  code = tsdbReadColData(pReader->pFileReader, &pBlockScanInfo->blockIdx, pBlock, pSupInfo->colIds, numOfCols,
-                                  pBlockData, NULL, NULL);
+  int32_t code = tsdbReadColData(pReader->pFileReader, &pBlockScanInfo->blockIdx, pBlock, pSupInfo->colIds, numOfCols,
+                                 pBlockData, NULL, NULL);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -1991,7 +1991,7 @@ static TSDBKEY getCurrentKeyInBuf(SDataBlockIter* pBlockIter, STsdbReader* pRead
 
 static int32_t moveToNextFile(STsdbReader* pReader, int32_t* numOfBlocks) {
   SReaderStatus* pStatus = &pReader->status;
-  SArray* pIndexList = taosArrayInit(4, sizeof(SBlockIdx));
+  SArray*        pIndexList = taosArrayInit(4, sizeof(SBlockIdx));
 
   while (1) {
     bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader);
@@ -3006,14 +3006,14 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
 
   code = doLoadFileBlockData(pReader, &pStatus->blockIter, pBlockScanInfo, &pStatus->fileBlockData);
   if (code != TSDB_CODE_SUCCESS) {
-    tBlockDataClear(&pStatus->fileBlockData);
+    tBlockDataClear(&pStatus->fileBlockData, 1);
 
     terrno = code;
     return NULL;
   }
 
   copyBlockDataToSDataBlock(pReader, pBlockScanInfo);
-  tBlockDataClear(&pStatus->fileBlockData);
+  tBlockDataClear(&pStatus->fileBlockData, 1);
   return pReader->pResBlock->pDataBlock;
 }
 
@@ -3132,8 +3132,8 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
       hasNext = (pBlockIter->numOfBlocks > 0);
     }
 
-//    tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables,
-//              pReader->pFileGroup->fid, pReader->idStr);
+    //    tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables,
+    //              pReader->pFileGroup->fid, pReader->idStr);
   }
 
   return code;
@@ -3204,4 +3204,3 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6
 
   return TSDB_CODE_SUCCESS;
 }
-
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 932cf9f9ae..e561bb9d22 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -411,30 +411,6 @@ static int32_t tsdbSnapCommit(STsdbSnapWriter* pWriter) {
   return code;
 }
 
-static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
-  int32_t code = 0;
-  STsdb*  pTsdb = pWriter->pTsdb;
-
-  if (pWriter->pDataFWriter == NULL) goto _exit;
-
-  // TODO
-
-  code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 0);
-  if (code) goto _err;
-
-  if (pWriter->pDataFReader) {
-    code = tsdbDataFReaderClose(&pWriter->pDataFReader);
-    if (code) goto _err;
-  }
-
-_exit:
-  return code;
-
-_err:
-  tsdbError("vgId:%d tsdb snapshot writer data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
-  return code;
-}
-
 static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t     code = 0;
   int32_t     iRow = 0;           // todo
@@ -456,14 +432,41 @@ _err:
 static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
   int32_t code = 0;
 
-  ASSERT(pWriter->pBlockIdxW != NULL);
+  ASSERT(pWriter->pDataFWriter);
+
+  if (pWriter->pBlockIdxW == NULL) goto _exit;
+
+  // consume remain rows
+  if (pWriter->pBlockData) {
+    ASSERT(pWriter->iRow < pWriter->pBlockData->nRow);
+    while (pWriter->iRow < pWriter->pBlockData->nRow) {
+      code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL);
+      if (code) goto _err;
+
+      if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) {
+        pWriter->blockW.last = 0;
+        code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
+                                  &pWriter->blockW, pWriter->cmprAlg);
+        if (code) goto _err;
+
+        code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
+        if (code) goto _err;
+
+        tBlockReset(&pWriter->blockW);
+        tBlockDataClearData(&pWriter->bDataW);
+      }
+
+      pWriter->iRow++;
+    }
+  }
 
   // write remain data if has
   if (pWriter->bDataW.nRow > 0) {
-    if (pWriter->bDataW.nRow >= pWriter->minRow) {
-      pWriter->blockW.last = 0;
-    } else {
-      pWriter->blockW.last = 1;
+    pWriter->blockW.last = 0;
+    if (pWriter->bDataW.nRow < pWriter->minRow) {
+      if (pWriter->iBlock > pWriter->mBlock.nItem) {
+        pWriter->blockW.last = 1;
+      }
     }
 
     code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
@@ -474,14 +477,40 @@ static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
     if (code) goto _err;
   }
 
+  while (true) {
+    if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
+
+    SBlock block;
+    tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock);
+
+    if (block.last) {
+      code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
+      if (code) goto _err;
+
+      tBlockReset(&block);
+      block.last = 1;
+      code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block,
+                                pWriter->cmprAlg);
+      if (code) goto _err;
+    }
+
+    code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
+    if (code) goto _err;
+
+    pWriter->iBlock++;
+  }
+
+  // SBlock
   code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW);
   if (code) goto _err;
 
+  // SBlockIdx
   if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
 
+_exit:
   return code;
 
 _err:
@@ -601,25 +630,39 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
 
         ASSERT(c);
 
-        if (c > 0) break;
+        if (c < 0) {
+          if (pWriter->bDataW.nRow) {
+            pWriter->blockW.last = 0;
+            code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
+                                      &pWriter->blockW, pWriter->cmprAlg);
+            if (code) goto _err;
 
-        if (pWriter->bDataW.nRow) {
-          pWriter->blockW.last = 0;
-          code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
-                                    &pWriter->blockW, pWriter->cmprAlg);
+            code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
+            if (code) goto _err;
+
+            tBlockReset(&pWriter->blockW);
+            tBlockDataClearData(&pWriter->bDataW);
+          }
+
+          code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
           if (code) goto _err;
 
-          code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock);
-          if (code) goto _err;
+          pWriter->iBlock++;
+        } else {
+          c = tsdbKeyCmprFn(&tBlockDataLastKey(pBlockData), &block.minKey);
 
-          tBlockReset(&pWriter->blockW);
-          tBlockDataClearData(&pWriter->bDataW);
+          ASSERT(c);
+
+          if (c < 0) break;
+
+          pWriter->pBlockData = &pWriter->bDataR;
+          code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL);
+          if (code) goto _err;
+          pWriter->iRow = 0;
+
+          pWriter->iBlock++;
+          break;
         }
-
-        code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock);
-        if (code) goto _err;
-
-        pWriter->iBlock++;
       }
 
       if (pWriter->pBlockData) continue;
@@ -744,6 +787,45 @@ _err:
   return code;
 }
 
+static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
+  int32_t code = 0;
+  STsdb*  pTsdb = pWriter->pTsdb;
+
+  if (pWriter->pDataFWriter == NULL) goto _exit;
+
+  code = tsdbSnapWriteTableDataEnd(pWriter);
+  if (code) goto _err;
+
+  while (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
+    code = tsdbSnapMoveWriteTableData(pWriter, (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx));
+    if (code) goto _err;
+
+    pWriter->iBlockIdx++;
+  }
+
+  code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW, NULL);
+  if (code) goto _err;
+
+  code = tsdbFSStateUpsertDFileSet(pTsdb->fs->nState, tsdbDataFWriterGetWSet(pWriter->pDataFWriter));
+  if (code) goto _err;
+
+  code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1);
+  if (code) goto _err;
+
+  if (pWriter->pDataFReader) {
+    code = tsdbDataFReaderClose(&pWriter->pDataFReader);
+    if (code) goto _err;
+  }
+
+_exit:
+  tsdbError("vgId:%d vnode snapshot tsdb writer data end", TD_VID(pTsdb->pVnode));
+  return code;
+
+_err:
+  tsdbError("vgId:%d vnode snapshot tsdb writer data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
 static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t code = 0;
   STsdb*  pTsdb = pWriter->pTsdb;
@@ -845,23 +927,23 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
     }
 
     // writer
-    SDelFile delFile = {.commitID = pTsdb->pVnode->state.commitID, .offset = 0, .size = 0};
+    SDelFile delFile = {.commitID = pWriter->commitID, .offset = 0, .size = 0};
     code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb);
     if (code) goto _err;
   }
 
   // process the del data
-  TABLEID id = {0};  // todo
+  TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr));
 
   while (true) {
     SDelIdx* pDelIdx = NULL;
-    int64_t  n = 0;
+    int64_t  n = sizeof(SSnapDataHdr) + sizeof(TABLEID);
     SDelData delData;
     SDelIdx  delIdx;
     int8_t   toBreak = 0;
 
     if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR)) {
-      pDelIdx = taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
+      pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
     }
 
     if (pDelIdx) {
@@ -914,7 +996,7 @@ _exit:
   return code;
 
 _err:
-  tsdbError("vgId:%d tsdb snapshot write del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d vnode snapshot tsdb write del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
@@ -923,6 +1005,7 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
   STsdb*  pTsdb = pWriter->pTsdb;
 
   if (pWriter->pDelFWriter == NULL) goto _exit;
+
   for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) {
     SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
 
@@ -954,10 +1037,11 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
   }
 
 _exit:
+  tsdbInfo("vgId:%d vnode snapshot tsdb write del end", TD_VID(pTsdb->pVnode));
   return code;
 
 _err:
-  tsdbError("vgId:%d tsdb snapshow write del end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d vnode snapshot tsdb write del end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
@@ -1076,7 +1160,7 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
 
   // del data
   if (pHdr->type == 2) {
-    code = tsdbSnapWriteDel(pWriter, pData + 1, nData - 1);
+    code = tsdbSnapWriteDel(pWriter, pData, nData);
     if (code) goto _err;
   }
 

From 725fa46142cfe04d498beb83532e2e2d7a32ec86 Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Fri, 15 Jul 2022 21:08:17 +0800
Subject: [PATCH 20/38] test:add  test case of tsbs query

---
 tests/system-test/2-query/tsbsQuery.py | 40 ++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 6 deletions(-)

diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py
index 5fe025ea07..b52b0bffef 100644
--- a/tests/system-test/2-query/tsbsQuery.py
+++ b/tests/system-test/2-query/tsbsQuery.py
@@ -40,9 +40,9 @@ class TDTestCase:
                     f"insert into rct{j} values ( {ts+i*10000}, {80+i}, {90+i}, {85+i}, {30+i*10}, {1.2*i}, {221+i*2}, {20+i*0.2}, {1500+i*20}, {150+i*2},{5+i} )"
                 )
                 tdSql.execute(
-                    f"insert into dct{j} values ( {ts+i*10000}, {1+i*0.1},{1400+i*15}, {1+i},{1500+i*20}, {150+i*2},{5+i} )"
+                    f"insert into dct{j} values ( {ts+i*10000}, {1+i*0.1},{1400+i*15}, {i},{1500+i*20}, {150+i*2},{5+i} )"
                 )
-
+        tdSql.execute("insert into dct9 (ts,fuel_state) values('2021-07-13 14:06:33.123Z',1.2) ;")
     # def check_avg(self ,origin_query , check_query):
     #     avg_result = tdSql.getResult(origin_query)
     #     origin_result = tdSql.getResult(check_query)
@@ -60,6 +60,7 @@ class TDTestCase:
 
 
     def tsbsIotQuery(self):
+        
         tdSql.execute("use db_tsbs")
         
         # test interval and partition
@@ -88,11 +89,38 @@ class TDTestCase:
         # test partition interval Pseudo time-column
         tdSql.query("SELECT count(ms1)/144  FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'  partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;")
 
-        #PR TD-17413
-        tdSql.query("SELECT ms1  FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'  partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-01T00:01:01Z';")
-        tdSql.checkRows(0)
+        # 1 high-load:
+        # tdSql.query("SELECT ts,name,driver,current_load,load_capacity FROM (SELECT last(ts) as ts,name,driver, current_load,load_capacity FROM diagnostics  WHERE fleet = 'South'   partition by name,driver) WHERE current_load>= (0.9 * load_capacity) partition by name ORDER BY name desc, ts DESC;")
 
-        # test
+        # tdSql.query("SELECT ts,name,driver,current_load,load_capacity FROM (SELECT last(ts) as ts,name,driver, current_load,load_capacity FROM diagnostics  WHERE fleet = 'South'   partition by name,driver) WHERE current_load>= (0.9 * load_capacity) partition by name ORDER BY name ;")
+
+        # 2 stationary-trucks 
+        tdSql.query("select name,driver from (SELECT name,driver,fleet ,avg(velocity) as mean_velocity FROM readings WHERE ts > '2016-01-01T15:07:21Z' AND ts <= '2016-01-01T16:17:21Z'    partition BY name,driver,fleet interval(10m) LIMIT 1)")
+        tdSql.query("select name,driver from (SELECT name,driver,fleet ,avg(velocity) as mean_velocity FROM readings WHERE ts > '2016-01-01T15:07:21Z' AND ts <= '2016-01-01T16:17:21Z'    partition BY name,driver,fleet interval(10m) LIMIT 1) WHERE fleet = 'West' AND mean_velocity < 1000 partition BY name")
+
+        # 3 long-driving-sessions
+        # tdSql.query("SELECT name,driver FROM(SELECT name,driver,count(*) AS ten_min FROM(SELECT _wstart as ts,name,driver,avg(velocity) as mean_velocity FROM readings  where ts > '2016-01-01T00:00:34Z' AND ts <= '2016-01-01T04:00:34Z'     partition BY name,driver interval(10m)) WHERE mean_velocity > 1 GROUP BY name,driver) WHERE ten_min > 22 ;")
+
+
+        #4 long-daily-sessions
+        tdSql.query("SELECT name,driver FROM(SELECT name,driver,count(*) AS ten_min FROM(SELECT name,driver,avg(velocity) as mean_velocity FROM readings WHERE fleet ='West' AND ts > '2016-01-01T12:31:37Z' AND ts <= '2016-01-05T12:31:37Z' partition BY name,driver  interval(10m) ) WHERE mean_velocity > 1 GROUP BY name,driver) WHERE ten_min > 60")
+
+        # 5. avg-daily-driving-duration
+        tdSql.query("select _wstart as ts,fleet,name,driver,count(mv)/6 as hours_driven from ( select _wstart as ts,fleet,name,driver,avg(velocity) as mv from readings where ts > '2016-01-01T00:00:00Z'     and ts < '2016-01-05T00:00:01Z'    partition by fleet,name,driver interval(10m)) where ts > '2016-01-01T00:00:00Z'  and ts < '2016-01-05T00:00:01Z'    partition by fleet,name,driver interval(1d) ;")
+
+
+        # 6. avg-daily-driving-session        
+        #taosc core dumped
+        tdSql.execute("create table random_measure2_1 (ts timestamp,ela float, name binary(40))")
+        tdSql.query("SELECT ts,diff(mv) AS difka  FROM (SELECT ts,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings   WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'   partition by name,ts interval(10m) fill(value,0))  GROUP BY name,ts;")
+        tdSql.query("SELECT _wstart,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings   WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'   partition by name interval(10m) fill(value,0)")
+
+        # 7. avg-load
+        tdSql.query("SELECT fleet, model,avg(ml) AS mean_load_percentage FROM (SELECT fleet, model,current_load/load_capacity AS ml FROM diagnostics partition BY name, fleet, model) partition BY fleet, model order by  fleet ;")
+
+        # 8. daily-activity   
+        tdSql.query(" SELECT model,ms1  FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'  partition by model, fleet interval(10m) fill(value,0)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z'  AND ms1<1;")
+        tdSql.query("SELECT _wstart,model,fleet,count(ms1)/144  FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z'  partition by model, fleet interval(10m) fill(value,0)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z'  AND ms1<1 partition by model, fleet interval(1d) ;")
     def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
         tdLog.printNoPrefix("==========step1:create database and table,insert data  ==============")
         self.prepareData()

From 8228808a2ad80dc66a66c08924f46219527fd953 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Sat, 16 Jul 2022 03:12:47 +0000
Subject: [PATCH 21/38] more vnode snapshot writer

---
 source/dnode/vnode/src/meta/metaSnapshot.c | 20 +++--------
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 23 +++++--------
 source/dnode/vnode/src/vnd/vnodeSnapshot.c | 40 ++++++++--------------
 3 files changed, 26 insertions(+), 57 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 900416f1b8..2143a3f085 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -131,18 +131,6 @@ struct SMetaSnapWriter {
   int64_t ever;
 };
 
-static int32_t metaSnapRollback(SMetaSnapWriter* pWriter) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
-static int32_t metaSnapCommit(SMetaSnapWriter* pWriter) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
 int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) {
   int32_t          code = 0;
   SMetaSnapWriter* pWriter;
@@ -171,10 +159,9 @@ int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
   SMetaSnapWriter* pWriter = *ppWriter;
 
   if (rollback) {
-    code = metaSnapRollback(pWriter);
-    if (code) goto _err;
+    ASSERT(0);
   } else {
-    code = metaSnapCommit(pWriter);
+    code = metaCommit(pWriter->pMeta);
     if (code) goto _err;
   }
   taosMemoryFree(pWriter);
@@ -199,9 +186,10 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
   code = metaHandleEntry(pMeta, &metaEntry);
   if (code) goto _err;
 
+  tDecoderClear(pDecoder);
   return code;
 
 _err:
-  metaError("vgId:%d meta snapshot write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
+  metaError("vgId:%d vnode snapshot meta write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
   return code;
 }
\ No newline at end of file
diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index e561bb9d22..8ca921c6e2 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -399,18 +399,6 @@ struct STsdbSnapWriter {
   SArray*      aDelIdxW;
 };
 
-static int32_t tsdbSnapRollback(STsdbSnapWriter* pWriter) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
-static int32_t tsdbSnapCommit(STsdbSnapWriter* pWriter) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
 static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t     code = 0;
   int32_t     iRow = 0;           // todo
@@ -738,6 +726,7 @@ static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) {
     }
 
     // reader
+    pWriter->pBlockIdx = NULL;
     if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
       ASSERT(pWriter->pDataFReader);
 
@@ -1104,6 +1093,9 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
     goto _err;
   }
 
+  code = tsdbFSBegin(pTsdb->fs);
+  if (code) goto _err;
+
   *ppWriter = pWriter;
   return code;
 
@@ -1118,7 +1110,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
   STsdbSnapWriter* pWriter = *ppWriter;
 
   if (rollback) {
-    code = tsdbSnapRollback(pWriter);
+    code = tsdbFSRollback(pWriter->pTsdb->fs);
     if (code) goto _err;
   } else {
     code = tsdbSnapWriteDataEnd(pWriter);
@@ -1127,7 +1119,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
     code = tsdbSnapWriteDelEnd(pWriter);
     if (code) goto _err;
 
-    code = tsdbSnapCommit(pWriter);
+    code = tsdbFSCommit(pWriter->pTsdb->fs);
     if (code) goto _err;
   }
 
@@ -1137,7 +1129,8 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
   return code;
 
 _err:
-  tsdbError("vgId:%d tsdb snapshot writer close failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d vnode snapshot tsdb writer close failed since %s", TD_VID(pWriter->pTsdb->pVnode),
+            tstrerror(code));
   return code;
 }
 
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 674beb51ec..c42c080fb8 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -149,18 +149,6 @@ struct SVSnapWriter {
   STsdbSnapWriter *pTsdbSnapWriter;
 };
 
-static int32_t vnodeSnapRollback(SVSnapWriter *pWriter) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
-static int32_t vnodeSnapCommit(SVSnapWriter *pWriter) {
-  int32_t code = 0;
-  // TODO
-  return code;
-}
-
 int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) {
   int32_t       code = 0;
   SVSnapWriter *pWriter = NULL;
@@ -188,15 +176,15 @@ _err:
 int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) {
   int32_t code = 0;
 
-  // TODO
+  if (pWriter->pMetaSnapWriter) {
+    code = metaSnapWriterClose(&pWriter->pMetaSnapWriter, rollback);
+    if (code) goto _err;
+  }
 
-  // if (rollback) {
-  //   code = vnodeSnapRollback(pWriter);
-  //   if (code) goto _err;
-  // } else {
-  //   code = vnodeSnapCommit(pWriter);
-  //   if (code) goto _err;
-  // }
+  if (pWriter->pTsdbSnapWriter) {
+    code = tsdbSnapWriterClose(&pWriter->pTsdbSnapWriter, rollback);
+    if (code) goto _err;
+  }
 
 _exit:
   vInfo("vgId:%d vnode snapshot writer closed, rollback:%d", TD_VID(pWriter->pVnode), rollback);
@@ -223,13 +211,13 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
   if (pHdr->type == 0) {
     // meta
 
-    // if (pWriter->pMetaSnapWriter == NULL) {
-    //   code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
-    //   if (code) goto _err;
-    // }
+    if (pWriter->pMetaSnapWriter == NULL) {
+      code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter);
+      if (code) goto _err;
+    }
 
-    // code = metaSnapWrite(pWriter->pMetaSnapWriter, pData , nData);
-    // if (code) goto _err;
+    code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData);
+    if (code) goto _err;
   } else {
     // tsdb
 

From ff07377fa37a68f51f4cc59c6340ea7b6de942b0 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Sat, 16 Jul 2022 03:40:20 +0000
Subject: [PATCH 22/38] finish vnode snapshot

---
 source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
index 8ca921c6e2..4886d874a9 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c
@@ -641,14 +641,15 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
 
           ASSERT(c);
 
-          if (c < 0) break;
+          if (c > 0) {
+            pWriter->pBlockData = &pWriter->bDataR;
+            code =
+                tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL);
+            if (code) goto _err;
+            pWriter->iRow = 0;
 
-          pWriter->pBlockData = &pWriter->bDataR;
-          code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL);
-          if (code) goto _err;
-          pWriter->iRow = 0;
-
-          pWriter->iBlock++;
+            pWriter->iBlock++;
+          }
           break;
         }
       }
@@ -684,6 +685,8 @@ static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
   return code;
 
 _err:
+  tsdbError("vgId:%d vnode snapshot tsdb write table data impl failed since %s", TD_VID(pWriter->pTsdb->pVnode),
+            tstrerror(code));
   return code;
 }
 

From d730bc3b0006fb1758c77f14a20bde9f57c40299 Mon Sep 17 00:00:00 2001
From: tomchon <haoran920c@163.com>
Date: Sat, 16 Jul 2022 12:15:52 +0800
Subject: [PATCH 23/38] test:modify testcase

---
 .../tsim/sync/3Replica5VgElect3mnodedrop.sim  | 204 +++++++++---------
 1 file changed, 102 insertions(+), 102 deletions(-)

diff --git a/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim
index ae02a23c9b..507d01595d 100644
--- a/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim
+++ b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim
@@ -110,21 +110,21 @@ if $rows != $vgroups then
   return -1
 endi
 
-if $data[0][4] == LEADER then
-   if $data[0][6] == FOLLOWER then
-      if $data[0][8] == FOLLOWER then
+if $data[0][4] == leader then
+   if $data[0][6] == follower then
+      if $data[0][8] == follower then
          print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] 
       endi
    endi
-elif $data[0][6] == LEADER then
-   if $data[0][4] == FOLLOWER then
-      if $data[0][8] == FOLLOWER then
+elif $data[0][6] == leader then
+   if $data[0][4] == follower then
+      if $data[0][8] == follower then
          print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] 
       endi
    endi
-elif $data[0][8] == LEADER then
-   if $data[0][4] == FOLLOWER then
-      if $data[0][6] == FOLLOWER then
+elif $data[0][8] == leader then
+   if $data[0][4] == follower then
+      if $data[0][6] == follower then
          print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] 
       endi
    endi
@@ -132,21 +132,21 @@ else
    goto check_vg_ready
 endi
 
-if $data[1][4] == LEADER then
-   if $data[1][6] == FOLLOWER then
-      if $data[1][8] == FOLLOWER then
+if $data[1][4] == leader then
+   if $data[1][6] == follower then
+      if $data[1][8] == follower then
          print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] 
       endi
    endi
-elif $data[1][6] == LEADER then
-   if $data[1][4] == FOLLOWER then
-      if $data[1][8] == FOLLOWER then
+elif $data[1][6] == leader then
+   if $data[1][4] == follower then
+      if $data[1][8] == follower then
          print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] 
       endi
    endi
-elif $data[1][8] == LEADER then
-   if $data[1][4] == FOLLOWER then
-      if $data[1][6] == FOLLOWER then
+elif $data[1][8] == leader then
+   if $data[1][4] == follower then
+      if $data[1][6] == follower then
          print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] 
       endi
    endi
@@ -154,21 +154,21 @@ else
    goto check_vg_ready
 endi
 
-if $data[2][4] == LEADER then
-   if $data[2][6] == FOLLOWER then
-      if $data[2][8] == FOLLOWER then
+if $data[2][4] == leader then
+   if $data[2][6] == follower then
+      if $data[2][8] == follower then
          print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] 
       endi
    endi
-elif $data[2][6] == LEADER then
-   if $data[2][4] == FOLLOWER then
-      if $data[2][8] == FOLLOWER then
+elif $data[2][6] == leader then
+   if $data[2][4] == follower then
+      if $data[2][8] == follower then
          print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] 
       endi
    endi
-elif $data[2][8] == LEADER then
-   if $data[2][4] == FOLLOWER then
-      if $data[2][6] == FOLLOWER then
+elif $data[2][8] == leader then
+   if $data[2][4] == follower then
+      if $data[2][6] == follower then
          print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] 
       endi
    endi
@@ -176,21 +176,21 @@ else
    goto check_vg_ready
 endi
 
-if $data[3][4] == LEADER then
-   if $data[3][6] == FOLLOWER then
-      if $data[3][8] == FOLLOWER then
+if $data[3][4] == leader then
+   if $data[3][6] == follower then
+      if $data[3][8] == follower then
          print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] 
       endi
    endi
-elif $data[3][6] == LEADER then
-   if $data[3][4] == FOLLOWER then
-      if $data[3][8] == FOLLOWER then
+elif $data[3][6] == leader then
+   if $data[3][4] == follower then
+      if $data[3][8] == follower then
          print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] 
       endi
    endi
-elif $data[3][8] == LEADER then
-   if $data[3][4] == FOLLOWER then
-      if $data[3][6] == FOLLOWER then
+elif $data[3][8] == leader then
+   if $data[3][4] == follower then
+      if $data[3][6] == follower then
          print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] 
       endi
    endi
@@ -198,21 +198,21 @@ else
    goto check_vg_ready
 endi
 
-if $data[4][4] == LEADER then
-   if $data[4][6] == FOLLOWER then
-      if $data[4][8] == FOLLOWER then
+if $data[4][4] == leader then
+   if $data[4][6] == follower then
+      if $data[4][8] == follower then
          print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] 
       endi
    endi
-elif $data[4][6] == LEADER then
-   if $data[4][4] == FOLLOWER then
-      if $data[4][8] == FOLLOWER then
+elif $data[4][6] == leader then
+   if $data[4][4] == follower then
+      if $data[4][8] == follower then
          print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] 
       endi
    endi
-elif $data[4][8] == LEADER then
-   if $data[4][4] == FOLLOWER then
-      if $data[4][6] == FOLLOWER then
+elif $data[4][8] == leader then
+   if $data[4][4] == follower then
+      if $data[4][6] == follower then
          print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] 
       endi
    endi
@@ -305,13 +305,13 @@ if $data[0][0] != 1 then
   return -1
 endi
 
-if $data[0][2] != LEADER then
+if $data[0][2] != leader then
   goto check_mnode_ready_2
 endi
-if $data[1][2] != FOLLOWER then
+if $data[1][2] != follower then
   goto check_mnode_ready_2
 endi
-if $data[2][2] != FOLLOWER then
+if $data[2][2] != follower then
   goto check_mnode_ready_2
 endi
 
@@ -337,21 +337,21 @@ if $rows != $vgroups then
   return -1
 endi
 
-if $data[0][4] == LEADER then
-   if $data[0][6] == FOLLOWER then
-      if $data[0][8] == FOLLOWER then
+if $data[0][4] == leader then
+   if $data[0][6] == follower then
+      if $data[0][8] == follower then
          print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] 
       endi
    endi
-elif $data[0][6] == LEADER then
-   if $data[0][4] == FOLLOWER then
-      if $data[0][8] == FOLLOWER then
+elif $data[0][6] == leader then
+   if $data[0][4] == follower then
+      if $data[0][8] == follower then
          print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] 
       endi
    endi
-elif $data[0][8] == LEADER then
-   if $data[0][4] == FOLLOWER then
-      if $data[0][6] == FOLLOWER then
+elif $data[0][8] == leader then
+   if $data[0][4] == follower then
+      if $data[0][6] == follower then
          print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] 
       endi
    endi
@@ -359,21 +359,21 @@ else
    goto check_vg_ready1
 endi
 
-if $data[1][4] == LEADER then
-   if $data[1][6] == FOLLOWER then
-      if $data[1][8] == FOLLOWER then
+if $data[1][4] == leader then
+   if $data[1][6] == follower then
+      if $data[1][8] == follower then
          print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] 
       endi
    endi
-elif $data[1][6] == LEADER then
-   if $data[1][4] == FOLLOWER then
-      if $data[1][8] == FOLLOWER then
+elif $data[1][6] == leader then
+   if $data[1][4] == follower then
+      if $data[1][8] == follower then
          print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] 
       endi
    endi
-elif $data[1][8] == LEADER then
-   if $data[1][4] == FOLLOWER then
-      if $data[1][6] == FOLLOWER then
+elif $data[1][8] == leader then
+   if $data[1][4] == follower then
+      if $data[1][6] == follower then
          print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] 
       endi
    endi
@@ -381,21 +381,21 @@ else
    goto check_vg_ready1
 endi
 
-if $data[2][4] == LEADER then
-   if $data[2][6] == FOLLOWER then
-      if $data[2][8] == FOLLOWER then
+if $data[2][4] == leader then
+   if $data[2][6] == follower then
+      if $data[2][8] == follower then
          print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] 
       endi
    endi
-elif $data[2][6] == LEADER then
-   if $data[2][4] == FOLLOWER then
-      if $data[2][8] == FOLLOWER then
+elif $data[2][6] == leader then
+   if $data[2][4] == follower then
+      if $data[2][8] == follower then
          print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] 
       endi
    endi
-elif $data[2][8] == LEADER then
-   if $data[2][4] == FOLLOWER then
-      if $data[2][6] == FOLLOWER then
+elif $data[2][8] == leader then
+   if $data[2][4] == follower then
+      if $data[2][6] == follower then
          print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] 
       endi
    endi
@@ -403,21 +403,21 @@ else
    goto check_vg_ready1
 endi
 
-if $data[3][4] == LEADER then
-   if $data[3][6] == FOLLOWER then
-      if $data[3][8] == FOLLOWER then
+if $data[3][4] == leader then
+   if $data[3][6] == follower then
+      if $data[3][8] == follower then
          print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] 
       endi
    endi
-elif $data[3][6] == LEADER then
-   if $data[3][4] == FOLLOWER then
-      if $data[3][8] == FOLLOWER then
+elif $data[3][6] == leader then
+   if $data[3][4] == follower then
+      if $data[3][8] == follower then
          print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] 
       endi
    endi
-elif $data[3][8] == LEADER then
-   if $data[3][4] == FOLLOWER then
-      if $data[3][6] == FOLLOWER then
+elif $data[3][8] == leader then
+   if $data[3][4] == follower then
+      if $data[3][6] == follower then
          print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] 
       endi
    endi
@@ -425,21 +425,21 @@ else
    goto check_vg_ready1
 endi
 
-if $data[4][4] == LEADER then
-   if $data[4][6] == FOLLOWER then
-      if $data[4][8] == FOLLOWER then
+if $data[4][4] == leader then
+   if $data[4][6] == follower then
+      if $data[4][8] == follower then
          print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] 
       endi
    endi
-elif $data[4][6] == LEADER then
-   if $data[4][4] == FOLLOWER then
-      if $data[4][8] == FOLLOWER then
+elif $data[4][6] == leader then
+   if $data[4][4] == follower then
+      if $data[4][8] == follower then
          print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] 
       endi
    endi
-elif $data[4][8] == LEADER then
-   if $data[4][4] == FOLLOWER then
-      if $data[4][6] == FOLLOWER then
+elif $data[4][8] == leader then
+   if $data[4][4] == follower then
+      if $data[4][6] == follower then
          print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] 
       endi
    endi
@@ -558,27 +558,27 @@ if $data[0][0] != 1 then
   return -1
 endi
 
-if $data[0][2] == LEADER then
-   if $data[1][2] != FOLLOWER then
+if $data[0][2] == leader then
+   if $data[1][2] != follower then
      goto check_mnode_ready_3
    endi
-   if $data[2][2] != FOLLOWER then
+   if $data[2][2] != follower then
    goto check_mnode_ready_3
    endi
 endi
-if $data[1][2] == LEADER then
-   if $data[0][2] != FOLLOWER then
+if $data[1][2] == leader then
+   if $data[0][2] != follower then
      goto check_mnode_ready_3
    endi
-   if $data[2][2] != FOLLOWER then
+   if $data[2][2] != follower then
    goto check_mnode_ready_3
    endi
 endi
-if $data[2][2] == LEADER then
-   if $data[1][2] != FOLLOWER then
+if $data[2][2] == leader then
+   if $data[1][2] != follower then
      goto check_mnode_ready_3
    endi
-   if $data[0][2] != FOLLOWER then
+   if $data[0][2] != follower then
    goto check_mnode_ready_3
    endi
 endi

From a760fb8071b96f59a6b535112b6be1371d29a819 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Sat, 16 Jul 2022 05:48:09 +0000
Subject: [PATCH 24/38] fix coredump

---
 source/dnode/vnode/src/meta/metaSnapshot.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c
index 2143a3f085..85261d302e 100644
--- a/source/dnode/vnode/src/meta/metaSnapshot.c
+++ b/source/dnode/vnode/src/meta/metaSnapshot.c
@@ -180,7 +180,7 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
   SMetaEntry metaEntry = {0};
   SDecoder*  pDecoder = &(SDecoder){0};
 
-  tDecoderInit(pDecoder, pData, nData);
+  tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
   metaDecodeEntry(pDecoder, &metaEntry);
 
   code = metaHandleEntry(pMeta, &metaEntry);

From a57d2417c350d59ae9b14874e69964aa1afe3be7 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Sat, 16 Jul 2022 14:01:49 +0800
Subject: [PATCH 25/38] feat(query): add hyperloglog scalar version

TD-17344
---
 include/libs/scalar/scalar.h        | 1 +
 source/libs/function/src/builtins.c | 5 +++--
 source/libs/scalar/src/sclfunc.c    | 4 ++++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h
index 5cbe380234..34b455379e 100644
--- a/include/libs/scalar/scalar.h
+++ b/include/libs/scalar/scalar.h
@@ -111,6 +111,7 @@ int32_t derivativeScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalar
 int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 31776610b6..6f31448360 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -2369,6 +2369,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getHLLFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = hllFunction,
+    .sprocessFunc = hllScalarFunction,
     .finalizeFunc = hllFinalize,
     .invertFunc   = NULL,
     .combineFunc  = hllCombine,
@@ -2463,7 +2464,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "tail",
     .type = FUNCTION_TYPE_TAIL,
-    .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | 
+    .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC |
                       FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateTail,
     .getEnvFunc   = getTailFuncEnv,
@@ -2474,7 +2475,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "unique",
     .type = FUNCTION_TYPE_UNIQUE,
-    .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | 
+    .classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC |
                       FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
     .translateFunc = translateUnique,
     .getEnvFunc   = getUniqueFuncEnv,
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 9687528136..1442b5b5b5 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -2421,3 +2421,7 @@ int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *
 int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   return avgScalarFunction(pInput, inputNum, pOutput);
 }
+
+int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return countScalarFunction(pInput, inputNum, pOutput);
+}

From eeb870924189d1e38617a36b238fc11bec3e1b3c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 16 Jul 2022 14:05:26 +0800
Subject: [PATCH 26/38] fix(query): add returned code check.

---
 source/libs/executor/src/scanoperator.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c
index 6aacb0ee01..001ddf9d5b 100644
--- a/source/libs/executor/src/scanoperator.c
+++ b/source/libs/executor/src/scanoperator.c
@@ -520,6 +520,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
 
     int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader,
                                   GET_TASKID(pTaskInfo));
+    if (code != TSDB_CODE_SUCCESS) {
+      longjmp(pTaskInfo->env, code);
+      return NULL;
+    }
   }
 
   SSDataBlock* result = doTableScanGroup(pOperator);

From 6d63d1b5b811b38b2748bdef9f5e86504e6c4450 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Sat, 16 Jul 2022 14:07:32 +0800
Subject: [PATCH 27/38] fix count scalar function behavior

---
 source/libs/scalar/src/sclfunc.c | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 1442b5b5b5..339ea101a6 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -1746,20 +1746,14 @@ int32_t countScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
   SColumnInfoData *pOutputData = pOutput->columnData;
 
   int64_t *out = (int64_t *)pOutputData->pData;
-  bool hasNull = false;
   *out = 0;
   for (int32_t i = 0; i < pInput->numOfRows; ++i) {
     if (colDataIsNull_s(pInputData, i)) {
-      hasNull = true;
-      break;
+      continue;
     }
     (*out)++;
   }
 
-  if (hasNull) {
-    colDataAppendNULL(pOutputData, 0);
-  }
-
   pOutput->numOfRows = 1;
   return TSDB_CODE_SUCCESS;
 }

From 2d1428eadd2465abfd275479986defe0a37420d7 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Sat, 16 Jul 2022 14:13:45 +0800
Subject: [PATCH 28/38] feat(query): add csum function scalar version

TD-17344
---
 include/libs/scalar/scalar.h        | 1 +
 source/libs/function/src/builtins.c | 1 +
 source/libs/scalar/src/sclfunc.c    | 4 ++++
 3 files changed, 6 insertions(+)

diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h
index 34b455379e..a6ed944632 100644
--- a/include/libs/scalar/scalar.h
+++ b/include/libs/scalar/scalar.h
@@ -112,6 +112,7 @@ int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
 int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 6f31448360..185f529963 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -2438,6 +2438,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getCsumFuncEnv,
     .initFunc     = functionSetup,
     .processFunc  = csumFunction,
+    .sprocessFunc = csumScalarFunction,
     .finalizeFunc = NULL
   },
   {
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index 339ea101a6..b94379c7b5 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -2419,3 +2419,7 @@ int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
 int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   return countScalarFunction(pInput, inputNum, pOutput);
 }
+
+int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return sumScalarFunction(pInput, inputNum, pOutput);
+}

From 55407c50e559f9d6e5dc040bdfd28c4b2894e81c Mon Sep 17 00:00:00 2001
From: Haojun Liao <hjliao@taosdata.com>
Date: Sat, 16 Jul 2022 14:44:31 +0800
Subject: [PATCH 29/38] fix(query): disable the optimized implementation of
 first/last/lastrow function.

---
 source/libs/function/src/builtinsimpl.c | 54 +++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index 12b796c5ca..a9a569480d 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -2723,6 +2723,9 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
 
   int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
 
+  //  please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function.
+  //  we will use this opt implementation in an new version that is only available in scan subplan
+#if 0
   if (blockDataOrder == TSDB_ORDER_ASC) {
     // filter according to current result firstly
     if (pResInfo->numOfRes > 0) {
@@ -2770,6 +2773,22 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
       }
     }
   }
+#else
+  for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
+    if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
+      continue;
+    }
+
+    numOfElems++;
+
+    char* data = colDataGetData(pInputCol, i);
+    TSKEY cts = getRowPTs(pInput->pPTS, i);
+    if (pResInfo->numOfRes == 0 || pInfo->ts > cts) {
+      doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data);
+      pResInfo->numOfRes = 1;
+    }
+  }
+#endif
 
   SET_VAL(pResInfo, numOfElems, 1);
   return TSDB_CODE_SUCCESS;
@@ -2801,6 +2820,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
 
   int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
 
+  //  please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function.
+#if 0
   if (blockDataOrder == TSDB_ORDER_ASC) {
     for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) {
       if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
@@ -2833,6 +2854,22 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
       break;
     }
   }
+#else
+  for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+    if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
+      continue;
+    }
+
+    numOfElems++;
+
+    char* data = colDataGetData(pInputCol, i);
+    TSKEY cts = getRowPTs(pInput->pPTS, i);
+    if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
+      doSaveCurrentVal(pCtx, i, cts, type, data);
+      pResInfo->numOfRes = 1;
+    }
+  }
+#endif
 
   SET_VAL(pResInfo, numOfElems, 1);
   return TSDB_CODE_SUCCESS;
@@ -2988,6 +3025,9 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
 
   int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
 
+#if 0
+  // the optimized version only function if all tuples in one block are monotonious increasing or descreasing.
+  // this is NOT always works if project operator exists in downstream.
   if (blockDataOrder == TSDB_ORDER_ASC) {
     for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) {
       char* data = colDataGetData(pInputCol, i);
@@ -2997,6 +3037,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
       if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
         doSaveLastrow(pCtx, data, i, cts, pInfo);
       }
+
       break;
     }
   } else {  // descending order
@@ -3011,7 +3052,19 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
       break;
     }
   }
+#else
+  for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
+    char* data = colDataGetData(pInputCol, i);
+    TSKEY cts = getRowPTs(pInput->pPTS, i);
+    numOfElems++;
 
+    if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
+      doSaveLastrow(pCtx, data, i, cts, pInfo);
+      pResInfo->numOfRes = 1;
+    }
+  }
+
+#endif
   SET_VAL(pResInfo, numOfElems, 1);
   return TSDB_CODE_SUCCESS;
 }
@@ -5926,6 +5979,7 @@ int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) {
     TSKEY cts = getRowPTs(pInput->pPTS, i);
     if (pResInfo->numOfRes == 0 || pInfo->ts < cts) {
       doSaveLastrow(pCtx, data, i, cts, pInfo);
+      pResInfo->numOfRes = 1;
     }
   }
 

From 9ea66e08af630173c2613da473b4b62012287b90 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Sat, 16 Jul 2022 14:58:55 +0800
Subject: [PATCH 30/38] enh: change redirect to network_unavail

---
 source/client/src/clientMain.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index 62052457fd..b4c2615f70 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -142,6 +142,7 @@ void taos_close(TAOS *taos) {
 
 int taos_errno(TAOS_RES *res) {
   if (res == NULL || TD_RES_TMQ_META(res)) {
+    if (terrno == TSDB_CODE_RPC_REDIRECT) errno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     return terrno;
   }
 
@@ -149,11 +150,13 @@ int taos_errno(TAOS_RES *res) {
     return 0;
   }
 
+  if (pRequest->code == TSDB_CODE_RPC_REDIRECT) pRequest->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
   return ((SRequestObj *)res)->code;
 }
 
 const char *taos_errstr(TAOS_RES *res) {
   if (res == NULL || TD_RES_TMQ_META(res)) {
+    if (terrno == TSDB_CODE_RPC_REDIRECT) errno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     return (const char *)tstrerror(terrno);
   }
 
@@ -165,6 +168,7 @@ const char *taos_errstr(TAOS_RES *res) {
   if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
     return pRequest->msgBuf;
   } else {
+    if (pRequest->code == TSDB_CODE_RPC_REDIRECT) pRequest->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     return (const char *)tstrerror(pRequest->code);
   }
 }

From 3b094cd21a451662d66660d2c96993470190f400 Mon Sep 17 00:00:00 2001
From: Shengliang Guan <slguan@taosdata.com>
Date: Sat, 16 Jul 2022 15:02:04 +0800
Subject: [PATCH 31/38] test: valgrind case

---
 tests/script/tsim/valgrind/basic3.sim      | 139 +++++----------------
 tests/script/tsim/valgrind/checkError2.sim |  30 ++++-
 2 files changed, 56 insertions(+), 113 deletions(-)

diff --git a/tests/script/tsim/valgrind/basic3.sim b/tests/script/tsim/valgrind/basic3.sim
index 7fba66c468..6a42a8eb7f 100644
--- a/tests/script/tsim/valgrind/basic3.sim
+++ b/tests/script/tsim/valgrind/basic3.sim
@@ -1,13 +1,10 @@
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
-system sh/deploy.sh -n dnode2 -i 2
+system sh/cfg.sh -n dnode1 -c debugflag -v 131
 system sh/exec.sh -n dnode1 -s start -v
-system sh/exec.sh -n dnode2 -s start -v
 sql connect
 
-print =============== add dnode2 into cluster
-sql create dnode $hostname port 7200
-
+print =============== step1: create drop show dnodes
 $x = 0
 step1:
 	$x = $x + 1
@@ -18,135 +15,59 @@ step1:
 	endi
 sql show dnodes
 print ---> $data00 $data01 $data02 $data03 $data04 $data05
-print ---> $data10 $data11 $data12 $data13 $data14 $data15
-if $rows != 2 then
+if $rows != 1 then
   return -1
 endi
 if $data(1)[4] != ready then
   goto step1
 endi
-if $data(2)[4] != ready then
-  goto step1
-endi
-
-print =============== create database, stable, table
-sql create database db vgroups 3
-sql use db
-sql create table stb (ts timestamp, c int) tags (t int)
-sql create table t0 using stb tags (0)
-sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10));
-
-print =============== run show xxxx
-sql show dnodes
-if $rows != 2 then 
-  return -1
-endi
-
-sql show mnodes
-if $rows != 1 then 
-  return -1
-endi
 
+print =============== step2: create db
+sql create database d1 vgroups 2 buffer 3
 sql show databases
-if $rows != 3 then
-  return -1
-endi
+sql use d1
+sql show vgroups
 
+print =============== step3: create show stable
+sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
 sql show stables
 if $rows != 1 then 
   return -1
 endi
 
+print =============== step4: create show table
+sql create table ct1 using stb tags(1000)
+sql create table ct2 using stb tags(2000)
+sql create table ct3 using stb tags(3000)
 sql show tables
-if $rows != 2 then 
-  return -1
-endi
-
-sql show users
-if $rows != 1 then 
-  return -1
-endi
-
-sql show vgroups
 if $rows != 3 then 
   return -1
 endi
 
-print =============== run select * from information_schema.xxxx
-sql select * from information_schema.`dnodes`
-if $rows != 2 then 
-  return -1
-endi
+print =============== step5: insert data
+sql insert into ct1 values(now+0d, 10, 2.0, 3.0)
+sql insert into ct1 values(now+1d, 11, 2.1, 3.1)(now+2d, -12, -2.2, -3.2)(now+3d, -13, -2.3, -3.3)
+sql insert into ct2 values(now+0d, 10, 2.0, 3.0)
+sql insert into ct2 values(now+1d, 11, 2.1, 3.1)(now+2d, -12, -2.2, -3.2)(now+3d, -13, -2.3, -3.3)
+sql insert into ct3 values('2022-01-01 00:00:00.000', 10, 2.0, 3.0)
 
-sql select * from information_schema.`mnodes`
-if $rows != 1 then 
-  return -1
-endi
+print =============== step6: query data
+sql select * from ct1 where ts < now -1d and ts > now +1d 
+sql select * from stb where ts < now -1d and ts > now +1d 
+sql select * from ct1 where ts < now -1d and ts > now +1d order by ts desc 
+sql select * from stb where ts < now -1d and ts > now +1d order by ts desc 
 
-sql select * from information_schema.user_databases
-if $rows != 3 then
-  return -1
-endi
-
-sql select * from information_schema.user_stables
-if $rows != 1 then 
-  return -1
-endi
-
-sql select * from information_schema.user_tables
-if $rows != 31 then
-  return -1
-endi
-
-sql select * from information_schema.user_users
-if $rows != 1 then 
-  return -1
-endi
-
-sql select * from information_schema.`vgroups`
-if $rows != 3 then 
-  return -1
-endi
-
-sql show variables;
-if $rows != 4 then
-  return -1
-endi
-
-sql show dnode 1 variables;
-if $rows <= 0 then
-  return -1
-endi
-
-sql show local variables;
-if $rows <= 0 then
-  return -1
-endi
-
-print ==== stop dnode1 and dnode2, and restart dnodes
+_OVER:
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-system sh/exec.sh -n dnode2 -s stop -x SIGINT
+print =============== check
+$null=
 
-print =============== check dnode1
 system_content sh/checkValgrind.sh -n dnode1 
 print cmd return result ----> [ $system_content ]
-if $system_content <= 0 then
-  return 0
+if $system_content > 1 then
+  return -1
 endi 
 
-$null=
 if $system_content == $null then
-  return 0
-endi 
-
-print =============== check dnode2
-system_content sh/checkValgrind.sh -n dnode2 
-print cmd return result ----> [ $system_content ]
-if $system_content <= 0 then
-  return 0
-endi 
-
-$null=
-if $system_content == $null then
-  return 0
+  return -1
 endi 
diff --git a/tests/script/tsim/valgrind/checkError2.sim b/tests/script/tsim/valgrind/checkError2.sim
index 3a82218015..9746fe9ecf 100644
--- a/tests/script/tsim/valgrind/checkError2.sim
+++ b/tests/script/tsim/valgrind/checkError2.sim
@@ -23,7 +23,7 @@ if $data(1)[4] != ready then
 endi
 
 print =============== step2: create db
-sql create database d1 vgroups 1 buffer 3
+sql create database d1 vgroups 2 buffer 3
 sql show databases
 sql use d1
 sql show vgroups
@@ -44,24 +44,32 @@ if $rows != 3 then
   return -1
 endi
 
-print =============== step5: insert data
+print =============== step5: insert data (null / update)
 sql insert into ct1 values(now+0s, 10, 2.0, 3.0)
-sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
+sql insert into ct1 values(now+1s, 11, 2.1, NULL)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
 sql insert into ct2 values(now+0s, 10, 2.0, 3.0)
 sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3)
-sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0)
+sql insert into ct3 values('2021-01-01 00:00:00.000', NULL, NULL, 3.0)
+sql insert into ct3 values('2022-03-02 16:59:00.010', 3  , 4, 5), ('2022-03-02 16:59:00.010', 33 , 4, 5), ('2022-04-01 16:59:00.011', 4,  4, 5), ('2022-04-01 16:59:00.011', 6,  4, 5), ('2022-03-06 16:59:00.013', 8,  4, 5);
+sql insert into ct3 values('2022-03-02 16:59:00.010', 103, 1, 2), ('2022-03-02 16:59:00.010', 303, 3, 4), ('2022-04-01 16:59:00.011', 40, 5, 6), ('2022-04-01 16:59:00.011', 60, 4, 5), ('2022-03-06 16:59:00.013', 80, 4, 5);
 
 print =============== step6: query data
 sql select * from ct1
 sql select * from stb
 sql select c1, c2, c3 from ct1
 sql select ts, c1, c2, c3 from stb
+sql select * from ct1 where ts < now -1d and ts > now +1d 
+sql select * from stb where ts < now -1d and ts > now +1d 
+#sql select * from ct1 where ts < now -1d and ts > now +1d order by ts desc 
+#sql select * from stb where ts < now -1d and ts > now +1d order by ts desc 
 
 print =============== step7: count
 sql select count(*) from ct1;
 sql select count(*) from stb;
 sql select count(ts), count(c1), count(c2), count(c3) from ct1
 sql select count(ts), count(c1), count(c2), count(c3) from stb
+sql select count(*) from ct1 where ts < now -1d and ts > now +1d 
+sql select count(*) from stb where ts < now -1d and ts > now +1d
 
 print =============== step8: func
 sql select first(ts), first(c1), first(c2), first(c3) from ct1
@@ -69,6 +77,20 @@ sql select min(c1), min(c2), min(c3) from ct1
 sql select max(c1), max(c2), max(c3) from ct1
 sql select sum(c1), sum(c2), sum(c3) from ct1
 
+print =============== step9: insert select
+#sql create table ct4 using stb tags(4000);
+#sql insert into ct4 select * from ct1;
+#sql select * from ct4;
+#sql insert into ct4 select ts,c1,c2,c3 from stb;
+
+#sql create table tb1 (ts timestamp, c1 int, c2 float, c3 double);
+#sql insert into tb1 (ts, c1, c2, c3) select * from ct1;
+#sql select * from tb1;
+
+#sql create table tb2 (ts timestamp, f1 binary(10), c1 int, c2 double);
+#sql insert into tb2 (c2, c1, ts) select c2+1, c1, ts+3 from ct2;
+#sql select * from tb2;
+
 _OVER:
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 print =============== check

From 790cecf12306965f9d8a28a207b8d08f04e61333 Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Sat, 16 Jul 2022 15:04:45 +0800
Subject: [PATCH 32/38] feat(query): add diff function scalar version

---
 include/libs/scalar/scalar.h        | 1 +
 source/libs/function/src/builtins.c | 1 +
 source/libs/scalar/src/sclfunc.c    | 4 ++++
 3 files changed, 6 insertions(+)

diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h
index a6ed944632..dfdb69ee3c 100644
--- a/include/libs/scalar/scalar.h
+++ b/include/libs/scalar/scalar.h
@@ -113,6 +113,7 @@ int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *
 int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 185f529963..10e0808c4d 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -2408,6 +2408,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .getEnvFunc   = getDiffFuncEnv,
     .initFunc     = diffFunctionSetup,
     .processFunc  = diffFunction,
+    .sprocessFunc = diffScalarFunction,
     .finalizeFunc = functionFinalize
   },
   {
diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c
index b94379c7b5..3f26cd46f8 100644
--- a/source/libs/scalar/src/sclfunc.c
+++ b/source/libs/scalar/src/sclfunc.c
@@ -2408,6 +2408,10 @@ int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam
   return nonCalcScalarFunction(pInput, inputNum, pOutput);
 }
 
+int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
+  return nonCalcScalarFunction(pInput, inputNum, pOutput);
+}
+
 int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) {
   return avgScalarFunction(pInput, inputNum, pOutput);
 }

From 6f29fd4020f1ecabbd78140f84b3353fdb124d66 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Sat, 16 Jul 2022 15:34:22 +0800
Subject: [PATCH 33/38] fix(sync): snapshot begin index

---
 source/dnode/vnode/src/vnd/vnodeOpen.c        |  2 +-
 source/libs/sync/inc/syncInt.h                |  1 +
 source/libs/sync/src/syncAppendEntries.c      |  5 +-
 source/libs/sync/src/syncAppendEntriesReply.c | 16 +++-
 source/libs/sync/src/syncMain.c               | 11 +++
 tests/script/tsim/sync/vnodesnapshot-test.sim | 81 -------------------
 6 files changed, 32 insertions(+), 84 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index 0914827950..1645aa37f9 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -119,7 +119,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   taosRealPath(tdir, NULL, sizeof(tdir));
 
 // for test tsdb snapshot
-#if 0
+#if 1
   pVnode->config.walCfg.segSize = 200;
   pVnode->config.walCfg.retentionSize = 2000;
 #endif
diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h
index 1f26033f63..0cbd7d36b2 100644
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@@ -223,6 +223,7 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode);
 
 // snapshot --------------
 bool syncNodeHasSnapshot(SSyncNode* pSyncNode);
+void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode);
 
 SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode);
 SyncTerm  syncNodeGetLastTerm(SSyncNode* pSyncNode);
diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c
index dca80f6826..9678b335fd 100644
--- a/source/libs/sync/src/syncAppendEntries.c
+++ b/source/libs/sync/src/syncAppendEntries.c
@@ -711,6 +711,9 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
         syncNodeEventLog(ths, logBuf);
       } while (0);
 
+      // maybe update commit index by snapshot
+      syncNodeMaybeUpdateCommitBySnapshot(ths);
+
       // prepare response msg
       SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
       pReply->srcId = ths->myRaftId;
@@ -718,7 +721,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc
       pReply->term = ths->pRaftStore->currentTerm;
       pReply->privateTerm = ths->pNewNodeReceiver->privateTerm;
       pReply->success = false;
-      pReply->matchIndex = SYNC_INDEX_INVALID;
+      pReply->matchIndex = ths->commitIndex;
 
       // msg event log
       do {
diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c
index c18c2b4d38..bb55e3d9d6 100644
--- a/source/libs/sync/src/syncAppendEntriesReply.c
+++ b/source/libs/sync/src/syncAppendEntriesReply.c
@@ -147,6 +147,15 @@ static void syncNodeStartSnapshotOnce(SSyncNode* ths, SyncIndex beginIndex, Sync
 int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) {
   int32_t ret = 0;
 
+  // print log
+  do {
+    char logBuf[256];
+    snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, term:%lu, match:%ld, success:%d", pMsg->term,
+             pMsg->matchIndex, pMsg->success);
+    syncNodeEventLog(ths, logBuf);
+
+  } while (0);
+
   // if already drop replica, do not process
   if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) {
     syncNodeEventLog(ths, "recv sync-append-entries-reply,  maybe replica already dropped");
@@ -238,7 +247,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
         SSnapshot oldSnapshot;
         ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &oldSnapshot);
         SyncTerm newSnapshotTerm = oldSnapshot.lastApplyTerm;
-        syncNodeStartSnapshotOnce(ths, SYNC_INDEX_BEGIN, nextIndex, newSnapshotTerm, pMsg);
+        syncNodeStartSnapshotOnce(ths, pMsg->matchIndex + 1, nextIndex, newSnapshotTerm, pMsg);
 
         // get sender
         SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId));
@@ -256,6 +265,11 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
     }
     syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex);
 
+    SyncIndex oldMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId));
+    if (pMsg->matchIndex > oldMatchIndex) {
+      syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex);
+    }
+
     // event log, update next-index
     do {
       char    host[64];
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index b6c01f2923..bb7454ea6f 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -1083,6 +1083,17 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) {
   return pSyncNode;
 }
 
+void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode) {
+  if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
+    SSnapshot snapshot;
+    int32_t   code = pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
+    ASSERT(code == 0);
+    if (snapshot.lastApplyIndex > pSyncNode->commitIndex) {
+      pSyncNode->commitIndex = snapshot.lastApplyIndex;
+    }
+  }
+}
+
 void syncNodeStart(SSyncNode* pSyncNode) {
   // start raft
   if (pSyncNode->replicaNum == 1) {
diff --git a/tests/script/tsim/sync/vnodesnapshot-test.sim b/tests/script/tsim/sync/vnodesnapshot-test.sim
index 8589078b50..2f0eccf02e 100644
--- a/tests/script/tsim/sync/vnodesnapshot-test.sim
+++ b/tests/script/tsim/sync/vnodesnapshot-test.sim
@@ -170,84 +170,3 @@ if $rows != 100 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
-system sh/exec.sh -n dnode2 -s stop -x SIGINT
-system sh/exec.sh -n dnode3 -s stop -x SIGINT
-system sh/exec.sh -n dnode4 -s stop -x SIGINT
-########################################################
-
-
-########################################################
-print ===> start dnode1 dnode3 dnode4
-system sh/exec.sh -n dnode1 -s start
-#system sh/exec.sh -n dnode2 -s start
-system sh/exec.sh -n dnode3 -s start
-system sh/exec.sh -n dnode4 -s start
-
-sleep 7000
-
-print =============== query data 
-sql connect
-sql use db
-sql select * from ct1
-print rows: $rows
-print $data00 $data01 $data02
-if $rows != 100 then
-  return -1
-endi
-
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
-#system sh/exec.sh -n dnode2 -s stop -x SIGINT
-system sh/exec.sh -n dnode3 -s stop -x SIGINT
-system sh/exec.sh -n dnode4 -s stop -x SIGINT
-########################################################
-
-
-########################################################
-print ===> start dnode1 dnode2 dnode4
-system sh/exec.sh -n dnode1 -s start
-system sh/exec.sh -n dnode2 -s start
-#system sh/exec.sh -n dnode3 -s start
-system sh/exec.sh -n dnode4 -s start
-
-sleep 3000
-
-print =============== query data 
-sql select * from ct1
-print rows: $rows
-print $data00 $data01 $data02
-if $rows != 100 then
-  return -1
-endi
-
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
-system sh/exec.sh -n dnode2 -s stop -x SIGINT
-#system sh/exec.sh -n dnode3 -s stop -x SIGINT
-system sh/exec.sh -n dnode4 -s stop -x SIGINT
-########################################################
-
-
-########################################################
-print ===> start dnode1 dnode2 dnode3 
-system sh/exec.sh -n dnode1 -s start
-system sh/exec.sh -n dnode2 -s start
-system sh/exec.sh -n dnode3 -s start
-#system sh/exec.sh -n dnode4 -s start
-
-sleep 3000
-
-print =============== query data 
-sql select * from ct1
-print rows: $rows
-print $data00 $data01 $data02
-if $rows != 100 then
-  return -1
-endi
-
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
-system sh/exec.sh -n dnode2 -s stop -x SIGINT
-system sh/exec.sh -n dnode3 -s stop -x SIGINT
-#system sh/exec.sh -n dnode4 -s stop -x SIGINT
-########################################################
-
-

From 92ec6ccea0a711b8abe0a70773f94044918dba40 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Sat, 16 Jul 2022 15:37:48 +0800
Subject: [PATCH 34/38] enh: change redirect to network_unavail

---
 source/client/src/clientMain.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c
index b4c2615f70..77b31011a3 100644
--- a/source/client/src/clientMain.c
+++ b/source/client/src/clientMain.c
@@ -142,7 +142,7 @@ void taos_close(TAOS *taos) {
 
 int taos_errno(TAOS_RES *res) {
   if (res == NULL || TD_RES_TMQ_META(res)) {
-    if (terrno == TSDB_CODE_RPC_REDIRECT) errno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+    if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     return terrno;
   }
 
@@ -150,13 +150,13 @@ int taos_errno(TAOS_RES *res) {
     return 0;
   }
 
-  if (pRequest->code == TSDB_CODE_RPC_REDIRECT) pRequest->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
-  return ((SRequestObj *)res)->code;
+  return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL
+                                                              : ((SRequestObj *)res)->code;
 }
 
 const char *taos_errstr(TAOS_RES *res) {
   if (res == NULL || TD_RES_TMQ_META(res)) {
-    if (terrno == TSDB_CODE_RPC_REDIRECT) errno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+    if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL;
     return (const char *)tstrerror(terrno);
   }
 
@@ -168,8 +168,8 @@ const char *taos_errstr(TAOS_RES *res) {
   if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
     return pRequest->msgBuf;
   } else {
-    if (pRequest->code == TSDB_CODE_RPC_REDIRECT) pRequest->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
-    return (const char *)tstrerror(pRequest->code);
+    return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL)
+                                                    : (const char *)tstrerror(pRequest->code);
   }
 }
 

From 5cefb68ddefc875316d5deca96c42dbaa541d1e1 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Sat, 16 Jul 2022 16:00:16 +0800
Subject: [PATCH 35/38] fix(sync): snapshot end index

---
 source/libs/sync/src/syncAppendEntriesReply.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c
index bb55e3d9d6..5ad2cb8e94 100644
--- a/source/libs/sync/src/syncAppendEntriesReply.c
+++ b/source/libs/sync/src/syncAppendEntriesReply.c
@@ -247,7 +247,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
         SSnapshot oldSnapshot;
         ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &oldSnapshot);
         SyncTerm newSnapshotTerm = oldSnapshot.lastApplyTerm;
-        syncNodeStartSnapshotOnce(ths, pMsg->matchIndex + 1, nextIndex, newSnapshotTerm, pMsg);
+        syncNodeStartSnapshotOnce(ths, pMsg->matchIndex + 1, oldSnapshot.lastApplyIndex, newSnapshotTerm, pMsg);
 
         // get sender
         SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId));

From 1a9f21c724e049272ef188a5bdb8692129e8a39f Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Sat, 16 Jul 2022 08:02:39 +0000
Subject: [PATCH 36/38] fix coredump

---
 source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 04368ddfb7..f318c69c6f 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -1151,7 +1151,7 @@ int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFr
         code = tBlockDataAddColData(pBlockData, iColData, &pColData);
         if (code) goto _exit;
 
-        tColDataInit(pColData, pColDataFrom->cid, pColData->type, pColData->smaOn);
+        tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn);
         for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
           code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type));
           if (code) goto _exit;

From 49c953fa0cb2b07e18818e2b2b7b726f214540bb Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Sat, 16 Jul 2022 16:10:50 +0800
Subject: [PATCH 37/38] refactor(sync): disable wal cfg

---
 source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c
index 1645aa37f9..0914827950 100644
--- a/source/dnode/vnode/src/vnd/vnodeOpen.c
+++ b/source/dnode/vnode/src/vnd/vnodeOpen.c
@@ -119,7 +119,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   taosRealPath(tdir, NULL, sizeof(tdir));
 
 // for test tsdb snapshot
-#if 1
+#if 0
   pVnode->config.walCfg.segSize = 200;
   pVnode->config.walCfg.retentionSize = 2000;
 #endif

From c978bd10ec38e73803d88c1abb96b134056e6ee4 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Sat, 16 Jul 2022 16:23:56 +0800
Subject: [PATCH 38/38] fix(sync): do not sender snapshot when there's wal

---
 source/libs/sync/src/syncAppendEntriesReply.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c
index 5ad2cb8e94..5137922522 100644
--- a/source/libs/sync/src/syncAppendEntriesReply.c
+++ b/source/libs/sync/src/syncAppendEntriesReply.c
@@ -247,7 +247,14 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie
         SSnapshot oldSnapshot;
         ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &oldSnapshot);
         SyncTerm newSnapshotTerm = oldSnapshot.lastApplyTerm;
-        syncNodeStartSnapshotOnce(ths, pMsg->matchIndex + 1, oldSnapshot.lastApplyIndex, newSnapshotTerm, pMsg);
+
+        SyncIndex endIndex;
+        if (ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex + 1)) {
+          endIndex = nextIndex;
+        } else {
+          endIndex = oldSnapshot.lastApplyIndex;
+        }
+        syncNodeStartSnapshotOnce(ths, pMsg->matchIndex + 1, endIndex, newSnapshotTerm, pMsg);
 
         // get sender
         SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId));