From 733db123ad8d073db22a5cb661a8796f83df4719 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 19 Jul 2023 13:24:59 +0000
Subject: [PATCH 01/36] refactor code

---
 source/dnode/vnode/src/vnd/vnodeSnapshot.c    | 49 +++++++++----------
 source/libs/stream/src/streamBackendRocksdb.c | 10 ++--
 2 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 70d74268c5..496a151c6e 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -239,31 +239,30 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
       }
     }
   }
-  // if (!pReader->streamStateDone) {
-  //   if (pReader->pStreamStateReader == NULL) {
-  //     code =
-  //         streamStateSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->sver,
-  //         &pReader->pStreamStateReader);
-  //     if (code) {
-  //       pReader->streamStateDone = 1;
-  //       pReader->pStreamStateReader = NULL;
-  //       goto _err;
-  //     }
-  //   }
-  //   code = streamStateSnapRead(pReader->pStreamStateReader, ppData);
-  //   if (code) {
-  //     goto _err;
-  //   } else {
-  //     if (*ppData) {
-  //       goto _exit;
-  //     } else {
-  //       pReader->streamStateDone = 1;
-  //       code = streamStateSnapReaderClose(pReader->pStreamStateReader);
-  //       if (code) goto _err;
-  //       pReader->pStreamStateReader = NULL;
-  //     }
-  //   }
-  // }
+  if (!pReader->streamStateDone) {
+    if (pReader->pStreamStateReader == NULL) {
+      code =
+          streamStateSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->sver, &pReader->pStreamStateReader);
+      if (code) {
+        pReader->streamStateDone = 1;
+        pReader->pStreamStateReader = NULL;
+        goto _err;
+      }
+    }
+    code = streamStateSnapRead(pReader->pStreamStateReader, ppData);
+    if (code) {
+      goto _err;
+    } else {
+      if (*ppData) {
+        goto _exit;
+      } else {
+        pReader->streamStateDone = 1;
+        code = streamStateSnapReaderClose(pReader->pStreamStateReader);
+        if (code) goto _err;
+        pReader->pStreamStateReader = NULL;
+      }
+    }
+  }
 
   // RSMA ==============
   if (VND_IS_RSMA(pReader->pVnode) && !pReader->rsmaDone) {
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 0c385e2362..5ddfdef806 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -188,10 +188,10 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) {
 
   // chkpId = 0;
   char* state = taosMemoryCalloc(1, strlen(path) + 32);
-  sprintf(state, "%s/%s", path, "state");
+  sprintf(state, "%s%s%s", path, TD_DIRSEP, "state");
   if (chkpId != 0) {
     char* chkp = taosMemoryCalloc(1, strlen(path) + 64);
-    sprintf(chkp, "%s/%s/checkpoint%" PRId64 "", path, "checkpoints", chkpId);
+    sprintf(chkp, "%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, "checkpoints", TD_DIRSEP, chkpId);
     if (taosIsDir(chkp) && isValidCheckpoint(chkp)) {
       if (taosIsDir(state)) {
         // remove dir if exists
@@ -462,7 +462,7 @@ int32_t delObsoleteCheckpoint(void* arg, const char* path) {
   for (int i = 0; i < taosArrayGetSize(checkpointDel); i++) {
     int64_t id = *(int64_t*)taosArrayGet(checkpointDel, i);
     char    tbuf[256] = {0};
-    sprintf(tbuf, "%s/checkpoint%" PRId64 "", path, id);
+    sprintf(tbuf, "%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, id);
     if (taosIsDir(tbuf)) {
       taosRemoveDir(tbuf);
     }
@@ -483,7 +483,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) {
 
   int32_t len = strlen(pMeta->path) + 30;
   char*   checkpointPath = taosMemoryCalloc(1, len);
-  sprintf(checkpointPath, "%s/%s", pMeta->path, "checkpoints");
+  sprintf(checkpointPath, "%s%s%s", pMeta->path, TD_DIRSEP, "checkpoints");
 
   if (!taosDirExist(checkpointPath)) {
     // no checkpoint, nothing to load
@@ -530,7 +530,7 @@ int32_t streamBackendDoCheckpoint(void* arg, uint64_t checkpointId) {
   int32_t      code = -1;
 
   char path[256] = {0};
-  sprintf(path, "%s/%s", pMeta->path, "checkpoints");
+  sprintf(path, "%s%s%s", pMeta->path, TD_DIRSEP, "checkpoints");
   code = taosMulModeMkDir(path, 0755);
   if (code != 0) {
     qError("failed to prepare checkpoint dir, path:%s, reason:%s", path, tstrerror(code));

From 6e5be0d30edae01c866db77df0e6d49c7be71632 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 20 Jul 2023 03:49:32 +0000
Subject: [PATCH 02/36] fix stream state transfer

---
 include/libs/stream/tstream.h                 |  4 +++-
 source/dnode/vnode/src/tq/tqStreamStateSnap.c | 16 ++++++++++++++--
 source/libs/stream/src/streamBackendRocksdb.c |  4 ++--
 source/libs/stream/src/streamSnapshot.c       |  2 ++
 source/libs/stream/src/streamState.c          |  5 +++--
 5 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 354f202e55..efc7901d4a 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -367,7 +367,9 @@ typedef struct SStreamMeta {
   SHashObj*     pTaskBackendUnique;
   TdThreadMutex backendMutex;
 
-  int32_t  chkptNotReadyTasks;
+  int32_t chkptNotReadyTasks;
+
+  int64_t checkpointId;
   SArray*  checkpointSaved;
   SArray*  checkpointInUse;
   int32_t  checkpointCap;
diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
index f7bae25043..4c70f3f7b9 100644
--- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c
+++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
@@ -41,13 +41,25 @@ int32_t streamStateSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
+
+  SStreamMeta* meta = pTq->pStreamMeta;
   pReader->pTq = pTq;
   pReader->sver = sver;
   pReader->ever = ever;
 
+  int64_t checkpointId = meta ? meta->checkpointId : 0;
+
   SStreamSnapReader* pSnapReader = NULL;
-  sprintf(tdir, "%s%s%s%s%s", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM, TD_DIRSEP, "checkpoints");
-  if (streamSnapReaderOpen(pTq, sver, ever, tdir, &pSnapReader) == 0) {
+
+  // restore from checkpoint if checkpointid != 0
+  if (checkpointId != 0) {
+    sprintf(tdir, "%s%s%s%s%s%scheckpoint%" PRId64 "", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM, TD_DIRSEP, "checkpoints",
+            TD_DIRSEP, checkpointId);
+  } else {
+    sprintf(tdir, "%s%s%s%s%s", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM, TD_DIRSEP, "state");
+  }
+
+  if (streamSnapReaderOpen(pTq, sver, checkpointId, tdir, &pSnapReader) == 0) {
     pReader->complete = 1;
   } else {
     code = -1;
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 5ddfdef806..f643e5b1e8 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -163,8 +163,8 @@ int32_t copyFiles(const char* src, const char* dst) {
     char* name = taosGetDirEntryName(de);
     if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue;
 
-    sprintf(absSrcPath, "%s/%s", src, name);
-    sprintf(absDstPath, "%s/%s", dst, name);
+    sprintf(absSrcPath, "%s%s%s", src, TD_DIRSEP, name);
+    sprintf(absDstPath, "%s%s%s", dst, TD_DIRSEP, name);
     if (!taosDirEntryIsDir(de)) {
       code = taosCopyFile(absSrcPath, absDstPath);
       if (code == -1) {
diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c
index 1a66c00389..ce82268f7e 100644
--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@@ -64,6 +64,7 @@ struct SStreamSnapReader {
   int64_t           sver;
   int64_t           ever;
   SStreamSnapHandle handle;
+  int64_t           checkpointId;
 };
 struct SStreamSnapWriter {
   void*             pMeta;
@@ -217,6 +218,7 @@ int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t ever, char* path
   if (pReader == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  pReader->handle.checkpointId = ever;
   // const char* path = NULL;
   if (streamSnapHandleInit(&pReader->handle, (char*)path) < 0) {
     taosMemoryFree(pReader);
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index 0a4f73a67c..ad18cdc091 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -108,7 +108,7 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz
   SStreamTask* pStreamTask = pTask;
   char         statePath[1024];
   if (!specPath) {
-    sprintf(statePath, "%s/%d", path, pStreamTask->id.taskId);
+    sprintf(statePath, "%s%s%d", path, TD_DIRSEP, pStreamTask->id.taskId);
   } else {
     memset(statePath, 0, 1024);
     tstrncpy(statePath, path, 1024);
@@ -762,7 +762,8 @@ int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVa
 
 int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) {
 #ifdef USE_ROCKSDB
-  qDebug("===stream===delete skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, key->win.skey,key->win.ekey, key->groupId);
+  qDebug("===stream===delete skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64, key->win.skey, key->win.ekey,
+         key->groupId);
   return streamStateSessionDel_rocksdb(pState, key);
 #else
   SStateSessionKey sKey = {.key = *key, .opNum = pState->number};

From 519d3b74d1191f436c34df7be278808a139698fd Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 20 Jul 2023 09:51:18 +0000
Subject: [PATCH 03/36] fix stream state transfer

---
 source/dnode/vnode/src/tq/tqStreamStateSnap.c | 10 +---
 source/libs/stream/src/streamSnapshot.c       | 59 ++++++++++++-------
 source/libs/tdb/test/tdbPageRecycleTest.cpp   |  2 +-
 3 files changed, 40 insertions(+), 31 deletions(-)

diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
index 4c70f3f7b9..e0d58176ed 100644
--- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c
+++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
@@ -51,15 +51,7 @@ int32_t streamStateSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS
 
   SStreamSnapReader* pSnapReader = NULL;
 
-  // restore from checkpoint if checkpointid != 0
-  if (checkpointId != 0) {
-    sprintf(tdir, "%s%s%s%s%s%scheckpoint%" PRId64 "", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM, TD_DIRSEP, "checkpoints",
-            TD_DIRSEP, checkpointId);
-  } else {
-    sprintf(tdir, "%s%s%s%s%s", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM, TD_DIRSEP, "state");
-  }
-
-  if (streamSnapReaderOpen(pTq, sver, checkpointId, tdir, &pSnapReader) == 0) {
+  if (streamSnapReaderOpen(pTq, sver, checkpointId, pTq->path, &pSnapReader) == 0) {
     pReader->complete = 1;
   } else {
     code = -1;
diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c
index ce82268f7e..50187cc01e 100644
--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@@ -79,31 +79,42 @@ const char*    ROCKSDB_CURRENT = "CURRENT";
 const char*    ROCKSDB_CHECKPOINT_META = "CHECKPOINT";
 static int64_t kBlockSize = 64 * 1024;
 
-int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path);
+int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path, int64_t chkpId);
 void    streamSnapHandleDestroy(SStreamSnapHandle* handle);
 
 // static void streamBuildFname(char* path, char* file, char* fullname)
 
 #define STREAM_ROCKSDB_BUILD_FULLNAME(path, file, fullname) \
   do {                                                      \
-    sprintf(fullname, "%s/%s", path, file);                 \
+    sprintf(fullname, "%s%s%s", path, TD_DIRSEP, file);     \
   } while (0)
 
-int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path) {
+int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chkpId) {
   // impl later
+  int   len = strlen(path);
+  char* tdir = taosMemoryCalloc(1, len + 128);
+  memcpy(tdir, path, len);
+
+  if (chkpId != 0) {
+    sprintf(tdir, "%s%s%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, "stream", TD_DIRSEP, "checkpoints", TD_DIRSEP,
+            chkpId);
+  } else {
+    sprintf(tdir, "%s%s%s%s%s", path, TD_DIRSEP, "stream", TD_DIRSEP, "state");
+  }
   int32_t code = 0;
 
-  TdDirPtr pDir = taosOpenDir(path);
+  TdDirPtr pDir = taosOpenDir(tdir);
   if (NULL == pDir) {
+    qError("stream-state failed to open %s", tdir);
     goto _err;
   }
 
   SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile));
   pHandle->pBackendFile = pFile;
-  pHandle->checkpointId = 0;
+  pHandle->checkpointId = chkpId;
   pHandle->seraial = 0;
 
-  pFile->path = taosStrdup(path);
+  pFile->path = tdir;
   pFile->pSst = taosArrayInit(16, sizeof(void*));
 
   TdDirEntryPtr pDirEntry;
@@ -135,7 +146,9 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path) {
   taosCloseDir(&pDir);
 
   if (pFile->pCurrent == NULL) {
+    qError("stream-state failed to open %s, reason: no valid file", tdir);
     code = -1;
+    tdir = NULL;
     goto _err;
   }
   SArray* list = taosArrayInit(64, sizeof(SBackendFileItem));
@@ -176,11 +189,13 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path) {
   pHandle->pFileList = list;
 
   char  fullname[256] = {0};
-  char* file = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx);
+  char* file = ((SBackendFileItem*)taosArrayGet(pHandle->pFileList, pHandle->currFileIdx))->name;
   STREAM_ROCKSDB_BUILD_FULLNAME(pFile->path, file, fullname);
 
   pHandle->fd = taosOpenFile(fullname, TD_FILE_READ);
   if (pHandle->fd == NULL) {
+    qError("stream-state failed to open %s, reason: %s", tdir, tstrerror(errno));
+    tdir = NULL;
     goto _err;
   }
   pHandle->seraial = 0;
@@ -188,6 +203,7 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path) {
   return 0;
 _err:
   streamSnapHandleDestroy(pHandle);
+  taosMemoryFreeClear(tdir);
 
   code = -1;
   return code;
@@ -195,32 +211,33 @@ _err:
 
 void streamSnapHandleDestroy(SStreamSnapHandle* handle) {
   SBanckendFile* pFile = handle->pBackendFile;
-  taosMemoryFree(pFile->pCheckpointMeta);
-  taosMemoryFree(pFile->pCurrent);
-  taosMemoryFree(pFile->pMainfest);
-  taosMemoryFree(pFile->pOptions);
-  taosMemoryFree(pFile->path);
-  for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) {
-    char* sst = taosArrayGetP(pFile->pSst, i);
-    taosMemoryFree(sst);
+  if (pFile) {
+    taosMemoryFree(pFile->pCheckpointMeta);
+    taosMemoryFree(pFile->pCurrent);
+    taosMemoryFree(pFile->pMainfest);
+    taosMemoryFree(pFile->pOptions);
+    taosMemoryFree(pFile->path);
+    for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) {
+      char* sst = taosArrayGetP(pFile->pSst, i);
+      taosMemoryFree(sst);
+    }
+    taosArrayDestroy(pFile->pSst);
+    taosMemoryFree(pFile);
   }
-  taosArrayDestroy(pFile->pSst);
-  taosMemoryFree(pFile);
 
   taosArrayDestroy(handle->pFileList);
   taosCloseFile(&handle->fd);
   return;
 }
 
-int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t ever, char* path, SStreamSnapReader** ppReader) {
+int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t chkpId, char* path, SStreamSnapReader** ppReader) {
   // impl later
   SStreamSnapReader* pReader = taosMemoryCalloc(1, sizeof(SStreamSnapReader));
   if (pReader == NULL) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  pReader->handle.checkpointId = ever;
-  // const char* path = NULL;
-  if (streamSnapHandleInit(&pReader->handle, (char*)path) < 0) {
+
+  if (streamSnapHandleInit(&pReader->handle, (char*)path, chkpId) < 0) {
     taosMemoryFree(pReader);
     return -1;
   }
diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp
index 4d7b314917..d740bd0f94 100644
--- a/source/libs/tdb/test/tdbPageRecycleTest.cpp
+++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp
@@ -804,7 +804,7 @@ TEST(TdbPageRecycleTest, recycly_delete_interior_ofp_nocommit) {
       // sprintf(&key[count - 2], "%c", i);
       key[count - 2] = '0' + i;
 
-      ret = tdbTbInsert(pDb, key, count, NULL, NULL, txn);
+      ret = tdbTbInsert(pDb, key, count, NULL, 0, txn);
       GTEST_ASSERT_EQ(ret, 0);
     }
   }

From fc2d6744e290121826b9827bc1e4e3916a654eba Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 20 Jul 2023 10:52:34 +0000
Subject: [PATCH 04/36] fix stream state transfer

---
 source/libs/stream/src/streamSnapshot.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c
index 50187cc01e..086da597f6 100644
--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@@ -198,6 +198,7 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk
     tdir = NULL;
     goto _err;
   }
+  qDebug("open file %s, current offset %" PRId64 "", file, (int64_t)0);
   pHandle->seraial = 0;
   pHandle->offset = 0;
   return 0;
@@ -261,24 +262,37 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si
 
   SBackendFileItem* item = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx);
 
+  qDebug("start to read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
+         (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
+
   uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize);
   int64_t  nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset);
   if (nread == -1) {
     code = TAOS_SYSTEM_ERROR(terrno);
     qError("stream snap failed to read snap, file name:%s, type:%d, reason:%s", item->name, item->type,
            tstrerror(code));
+    qDebug("failed to read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
+           (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
     return code;
     // handle later
     return -1;
   } else if (nread > 0 && nread <= kBlockSize) {
     // left bytes less than kBlockSize
+    qDebug("read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
+           (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
     pHandle->offset += nread;
     if (pHandle->offset >= item->size || nread < kBlockSize) {
       taosCloseFile(&pHandle->fd);
+      qDebug("close file no.%d, move to next file, next file no.%d", pHandle->currFileIdx, pHandle->currFileIdx + 1);
       pHandle->offset = 0;
       pHandle->currFileIdx += 1;
     }
   } else {
+    qDebug("no data read, close file no.%d, move to next file, open and read", pHandle->currFileIdx);
+    taosCloseFile(&pHandle->fd);
+    pHandle->offset = 0;
+    pHandle->currFileIdx += 1;
+
     if (pHandle->currFileIdx >= taosArrayGetSize(pHandle->pFileList)) {
       // finish
       *ppData = NULL;
@@ -292,6 +306,9 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si
 
     nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset);
     pHandle->offset += nread;
+
+    qDebug("read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
+           (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
   }
 
   SStreamSnapBlockHdr* pHdr = (SStreamSnapBlockHdr*)buf;

From 82df77eba8b6bed1e0a9f0471f3e60d9f4126439 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 20 Jul 2023 11:52:31 +0000
Subject: [PATCH 05/36] fix stream state transfer

---
 source/libs/stream/src/streamSnapshot.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c
index 086da597f6..a036bcd877 100644
--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@@ -157,7 +157,9 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk
   // current
   item.name = pFile->pCurrent;
   item.type = ROCKSDB_CURRENT_TYPE;
-  taosStatFile(pFile->pCurrent, &item.size, NULL);
+  if (taosStatFile(pFile->pCurrent, &item.size, NULL) != 0) {
+    qError("stream-state failed to get file size: %s", pFile->pCurrent);
+  }
   taosArrayPush(list, &item);
   // mainfest
   item.name = pFile->pMainfest;

From 6a27e81964e22e6ef44f780f74282dd98be87112 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 24 Jul 2023 02:05:14 +0000
Subject: [PATCH 06/36] fix stream state transfer

---
 source/libs/stream/src/streamSnapshot.c | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c
index a036bcd877..44aa69a070 100644
--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@@ -200,7 +200,7 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk
     tdir = NULL;
     goto _err;
   }
-  qDebug("open file %s, current offset %" PRId64 "", file, (int64_t)0);
+  qDebug("stream-state open file %s, current offset %" PRId64 "", file, (int64_t)0);
   pHandle->seraial = 0;
   pHandle->offset = 0;
   return 0;
@@ -264,33 +264,34 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si
 
   SBackendFileItem* item = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx);
 
-  qDebug("start to read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
+  qDebug("stream-state start to read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
          (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
 
   uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize);
   int64_t  nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset);
   if (nread == -1) {
     code = TAOS_SYSTEM_ERROR(terrno);
-    qError("stream snap failed to read snap, file name:%s, type:%d, reason:%s", item->name, item->type,
+    qError("stream-state snap failed to read snap, file name:%s, type:%d, reason:%s", item->name, item->type,
            tstrerror(code));
-    qDebug("failed to read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
-           (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
+    qDebug("stream-state failed to read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d",
+           item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
     return code;
     // handle later
     return -1;
   } else if (nread > 0 && nread <= kBlockSize) {
     // left bytes less than kBlockSize
-    qDebug("read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
+    qDebug("stream-state read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
            (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
     pHandle->offset += nread;
     if (pHandle->offset >= item->size || nread < kBlockSize) {
       taosCloseFile(&pHandle->fd);
-      qDebug("close file no.%d, move to next file, next file no.%d", pHandle->currFileIdx, pHandle->currFileIdx + 1);
+      qDebug("stream-state close file no.%d, move to next file, next file no.%d", pHandle->currFileIdx,
+             pHandle->currFileIdx + 1);
       pHandle->offset = 0;
       pHandle->currFileIdx += 1;
     }
   } else {
-    qDebug("no data read, close file no.%d, move to next file, open and read", pHandle->currFileIdx);
+    qDebug("stream-state no data read, close file no.%d, move to next file, open and read", pHandle->currFileIdx);
     taosCloseFile(&pHandle->fd);
     pHandle->offset = 0;
     pHandle->currFileIdx += 1;

From 0c2e86f416ebf4cc171048f7f1b68268737361db Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 25 Jul 2023 10:17:30 +0000
Subject: [PATCH 07/36] fix stream state transfer

---
 source/libs/stream/src/streamSnapshot.c | 40 ++++++++++++++++++-------
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c
index 44aa69a070..43cb6511b8 100644
--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@@ -89,6 +89,17 @@ void    streamSnapHandleDestroy(SStreamSnapHandle* handle);
     sprintf(fullname, "%s%s%s", path, TD_DIRSEP, file);     \
   } while (0)
 
+int32_t getFileSize(char* path, char* name, int64_t* sz) {
+  int ret = 0;
+
+  char* fullname = taosMemoryCalloc(1, strlen(path) + 32);
+  sprintf(fullname, "%s%s%s", path, TD_DIRSEP, name);
+
+  ret = taosStatFile(fullname, sz, NULL);
+  taosMemoryFree(fullname);
+
+  return ret;
+}
 int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chkpId) {
   // impl later
   int   len = strlen(path);
@@ -157,32 +168,32 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk
   // current
   item.name = pFile->pCurrent;
   item.type = ROCKSDB_CURRENT_TYPE;
-  if (taosStatFile(pFile->pCurrent, &item.size, NULL) != 0) {
-    qError("stream-state failed to get file size: %s", pFile->pCurrent);
-  }
+  getFileSize(pFile->path, item.name, &item.size);
   taosArrayPush(list, &item);
+
   // mainfest
   item.name = pFile->pMainfest;
   item.type = ROCKSDB_MAINFEST_TYPE;
-  taosStatFile(pFile->pMainfest, &item.size, NULL);
+  getFileSize(pFile->path, item.name, &item.size);
   taosArrayPush(list, &item);
+
   // options
   item.name = pFile->pOptions;
   item.type = ROCKSDB_OPTIONS_TYPE;
-  taosStatFile(pFile->pOptions, &item.size, NULL);
+  getFileSize(pFile->path, item.name, &item.size);
   taosArrayPush(list, &item);
   // sst
   for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) {
     char* sst = taosArrayGetP(pFile->pSst, i);
     item.name = sst;
     item.type = ROCKSDB_SST_TYPE;
-    taosStatFile(sst, &item.size, NULL);
+    getFileSize(pFile->path, item.name, &item.size);
     taosArrayPush(list, &item);
   }
   // meta
   item.name = pFile->pCheckpointMeta;
   item.type = ROCKSDB_CHECKPOINT_META_TYPE;
-  taosStatFile(pFile->pCheckpointMeta, &item.size, NULL);
+  getFileSize(pFile->path, item.name, &item.size);
   taosArrayPush(list, &item);
 
   pHandle->pBackendFile = pFile;
@@ -305,8 +316,8 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si
     item = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx);
     char fullname[256] = {0};
     STREAM_ROCKSDB_BUILD_FULLNAME(pFile->path, item->name, fullname);
-    pHandle->fd = taosOpenFile(fullname, TD_FILE_READ);
 
+    pHandle->fd = taosOpenFile(fullname, TD_FILE_READ);
     nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset);
     pHandle->offset += nread;
 
@@ -349,7 +360,12 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path
   pHandle->pFileList = list;
   pHandle->currFileIdx = 0;
   pHandle->offset = 0;
-  pHandle->fd = taosOpenFile(taosArrayGet(pHandle->pFileList, pHandle->currFileIdx), TD_FILE_WRITE);
+
+  char  fullname[256] = {0};
+  char* name = ((SBackendFileItem*)taosArrayGet(pHandle->pFileList, pHandle->currFileIdx))->name;
+  STREAM_ROCKSDB_BUILD_FULLNAME(pFile->path, name, fullname);
+
+  pHandle->fd = taosOpenFile(fullname, TD_FILE_WRITE);
   *ppWriter = pWriter;
   return 0;
 }
@@ -378,8 +394,9 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa
     item.type = pHdr->type;
     taosArrayPush(pHandle->pFileList, &item);
 
-    char  fullname[256] = {0};
-    char* name = ((SBackendFileItem*)taosArrayGet(pHandle->pFileList, taosArrayGetSize(pHandle->pFileList) - 1))->name;
+    char fullname[256] = {0};
+
+    char* name = ((SBackendFileItem*)taosArrayGet(pHandle->pFileList, pHandle->currFileIdx))->name;
     STREAM_ROCKSDB_BUILD_FULLNAME(pFile->path, name, fullname);
     pHandle->fd = taosOpenFile(fullname, TD_FILE_WRITE);
 
@@ -406,6 +423,7 @@ int32_t streamSnapWriterClose(SStreamSnapWriter* pWriter, int8_t rollback) {
     qDebug("stream snap get file list, %s", buf);
     taosMemoryFree(buf);
   }
+
   for (int i = 0; i < taosArrayGetSize(handle->pFileList); i++) {
     SBackendFileItem* item = taosArrayGet(handle->pFileList, i);
     taosMemoryFree(item->name);

From 22c85fcbad47faf3e445548f09a2504b7ab6d642 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 25 Jul 2023 13:07:49 +0000
Subject: [PATCH 08/36] fix stream state transfer

---
 source/libs/stream/src/streamSnapshot.c | 84 ++++++++++++-------------
 1 file changed, 39 insertions(+), 45 deletions(-)

diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c
index 43cb6511b8..5ae0f6f30e 100644
--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@@ -89,7 +89,7 @@ void    streamSnapHandleDestroy(SStreamSnapHandle* handle);
     sprintf(fullname, "%s%s%s", path, TD_DIRSEP, file);     \
   } while (0)
 
-int32_t getFileSize(char* path, char* name, int64_t* sz) {
+int32_t streamGetFileSize(char* path, char* name, int64_t* sz) {
   int ret = 0;
 
   char* fullname = taosMemoryCalloc(1, strlen(path) + 32);
@@ -100,6 +100,13 @@ int32_t getFileSize(char* path, char* name, int64_t* sz) {
 
   return ret;
 }
+
+TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) {
+  char fullname[256] = {0};
+  STREAM_ROCKSDB_BUILD_FULLNAME(path, name, fullname);
+  return taosOpenFile(fullname, opt);
+}
+
 int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chkpId) {
   // impl later
   int   len = strlen(path);
@@ -140,7 +147,7 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk
       continue;
     }
     if (strlen(name) >= strlen(ROCKSDB_OPTIONS) && 0 == strncmp(name, ROCKSDB_OPTIONS, strlen(ROCKSDB_OPTIONS))) {
-      pFile->pMainfest = taosStrdup(name);
+      pFile->pOptions = taosStrdup(name);
       continue;
     }
     if (strlen(name) >= strlen(ROCKSDB_CHECKPOINT_META) &&
@@ -168,50 +175,39 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk
   // current
   item.name = pFile->pCurrent;
   item.type = ROCKSDB_CURRENT_TYPE;
-  getFileSize(pFile->path, item.name, &item.size);
+  streamGetFileSize(pFile->path, item.name, &item.size);
   taosArrayPush(list, &item);
 
   // mainfest
   item.name = pFile->pMainfest;
   item.type = ROCKSDB_MAINFEST_TYPE;
-  getFileSize(pFile->path, item.name, &item.size);
+  streamGetFileSize(pFile->path, item.name, &item.size);
   taosArrayPush(list, &item);
 
   // options
   item.name = pFile->pOptions;
   item.type = ROCKSDB_OPTIONS_TYPE;
-  getFileSize(pFile->path, item.name, &item.size);
+  streamGetFileSize(pFile->path, item.name, &item.size);
   taosArrayPush(list, &item);
   // sst
   for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) {
     char* sst = taosArrayGetP(pFile->pSst, i);
     item.name = sst;
     item.type = ROCKSDB_SST_TYPE;
-    getFileSize(pFile->path, item.name, &item.size);
+    streamGetFileSize(pFile->path, item.name, &item.size);
     taosArrayPush(list, &item);
   }
   // meta
   item.name = pFile->pCheckpointMeta;
   item.type = ROCKSDB_CHECKPOINT_META_TYPE;
-  getFileSize(pFile->path, item.name, &item.size);
-  taosArrayPush(list, &item);
+  if (streamGetFileSize(pFile->path, item.name, &item.size) == 0) {
+    taosArrayPush(list, &item);
+  }
 
   pHandle->pBackendFile = pFile;
 
   pHandle->currFileIdx = 0;
   pHandle->pFileList = list;
-
-  char  fullname[256] = {0};
-  char* file = ((SBackendFileItem*)taosArrayGet(pHandle->pFileList, pHandle->currFileIdx))->name;
-  STREAM_ROCKSDB_BUILD_FULLNAME(pFile->path, file, fullname);
-
-  pHandle->fd = taosOpenFile(fullname, TD_FILE_READ);
-  if (pHandle->fd == NULL) {
-    qError("stream-state failed to open %s, reason: %s", tdir, tstrerror(errno));
-    tdir = NULL;
-    goto _err;
-  }
-  qDebug("stream-state open file %s, current offset %" PRId64 "", file, (int64_t)0);
   pHandle->seraial = 0;
   pHandle->offset = 0;
   return 0;
@@ -275,29 +271,35 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si
 
   SBackendFileItem* item = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx);
 
-  qDebug("stream-state start to read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
-         (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
+  if (pHandle->fd == NULL) {
+    if (pHandle->currFileIdx >= taosArrayGetSize(pHandle->pFileList)) {
+      // finish
+      *ppData = NULL;
+      *size = 0;
+      return 0;
+    } else {
+      pHandle->fd = streamOpenFile(pFile->path, item->name, TD_FILE_READ);
+      qDebug("stream-state open file %s, current offset:%" PRId64 ", size:% " PRId64 ", file no.%d", item->name,
+             (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
+    }
+  }
 
+  qDebug("stream-state start to read file %s, current offset:%" PRId64 ", size:%" PRId64 ", file no.%d", item->name,
+         (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
   uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize);
   int64_t  nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset);
   if (nread == -1) {
     code = TAOS_SYSTEM_ERROR(terrno);
-    qError("stream-state snap failed to read snap, file name:%s, type:%d, reason:%s", item->name, item->type,
+    qError("stream-state snap failed to read snap, file name:%s, type:%d,reason:%s", item->name, item->type,
            tstrerror(code));
-    qDebug("stream-state failed to read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d",
-           item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
-    return code;
-    // handle later
     return -1;
   } else if (nread > 0 && nread <= kBlockSize) {
     // left bytes less than kBlockSize
-    qDebug("stream-state read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
+    qDebug("stream-state read file %s, current offset:%" PRId64 ",size:% " PRId64 ", file no.%d", item->name,
            (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
     pHandle->offset += nread;
     if (pHandle->offset >= item->size || nread < kBlockSize) {
       taosCloseFile(&pHandle->fd);
-      qDebug("stream-state close file no.%d, move to next file, next file no.%d", pHandle->currFileIdx,
-             pHandle->currFileIdx + 1);
       pHandle->offset = 0;
       pHandle->currFileIdx += 1;
     }
@@ -314,15 +316,13 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si
       return 0;
     }
     item = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx);
-    char fullname[256] = {0};
-    STREAM_ROCKSDB_BUILD_FULLNAME(pFile->path, item->name, fullname);
+    pHandle->fd = streamOpenFile(pFile->path, item->name, TD_FILE_READ);
 
-    pHandle->fd = taosOpenFile(fullname, TD_FILE_READ);
     nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset);
     pHandle->offset += nread;
 
-    qDebug("read file %s, current offset %" PRId64 ", size : % " PRId64 ", file no. %d", item->name,
-           (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
+    qDebug("stream-state open file and read file %s, current offset:%" PRId64 ", size:% " PRId64 ", file no.%d",
+           item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx);
   }
 
   SStreamSnapBlockHdr* pHdr = (SStreamSnapBlockHdr*)buf;
@@ -361,11 +361,8 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path
   pHandle->currFileIdx = 0;
   pHandle->offset = 0;
 
-  char  fullname[256] = {0};
-  char* name = ((SBackendFileItem*)taosArrayGet(pHandle->pFileList, pHandle->currFileIdx))->name;
-  STREAM_ROCKSDB_BUILD_FULLNAME(pFile->path, name, fullname);
-
-  pHandle->fd = taosOpenFile(fullname, TD_FILE_WRITE);
+  SBackendFileItem* pItem = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx);
+  pHandle->fd = streamOpenFile(pFile->path, pItem->name, TD_FILE_WRITE);
   *ppWriter = pWriter;
   return 0;
 }
@@ -394,11 +391,8 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa
     item.type = pHdr->type;
     taosArrayPush(pHandle->pFileList, &item);
 
-    char fullname[256] = {0};
-
-    char* name = ((SBackendFileItem*)taosArrayGet(pHandle->pFileList, pHandle->currFileIdx))->name;
-    STREAM_ROCKSDB_BUILD_FULLNAME(pFile->path, name, fullname);
-    pHandle->fd = taosOpenFile(fullname, TD_FILE_WRITE);
+    SBackendFileItem* pItem = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx);
+    pHandle->fd = streamOpenFile(pFile->path, pItem->name, TD_FILE_WRITE);
 
     taosPWriteFile(pHandle->fd, pHdr->data, pHdr->size, pHandle->offset);
     pHandle->offset += pHdr->size;

From c61393a7a1bc6d04156eb57579bc656132a5b434 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 26 Jul 2023 01:33:41 +0000
Subject: [PATCH 09/36] fix stream state transfer

---
 source/dnode/vnode/src/tq/tqStreamStateSnap.c | 2 +-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
index e0d58176ed..83fdd71c44 100644
--- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c
+++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
@@ -141,7 +141,7 @@ int32_t streamStateSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS
     goto _err;
   }
 
-  tqDebug("vgId:%d, vnode stream-state snapshot writer opened", TD_VID(pTq->pVnode));
+  tqDebug("vgId:%d, vnode stream-state snapshot writer opened, path:%s", TD_VID(pTq->pVnode), tdir);
   pWriter->pWriterImpl = pSnapWriter;
   return code;
 _err:
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 496a151c6e..16bd233807 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -517,7 +517,7 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
       code = streamTaskSnapWrite(pWriter->pStreamTaskWriter, pData, nData);
       if (code) goto _err;
     } break;
-    case SNAP_DATA_STREAM_STATE: {
+    case SNAP_DATA_STREAM_STATE_BACKEND: {
       if (pWriter->pStreamStateWriter == NULL) {
         code = streamStateSnapWriterOpen(pVnode->pTq, pWriter->sver, pWriter->ever, &pWriter->pStreamStateWriter);
         if (code) goto _err;

From 287088ae3fc3d0bedac9d8c947c67559392840a1 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 26 Jul 2023 10:16:06 +0000
Subject: [PATCH 10/36] fix stream state transfer

---
 source/dnode/vnode/src/tq/tqStreamStateSnap.c |  6 +++-
 source/libs/stream/src/streamSnapshot.c       | 30 ++++++++++++++-----
 2 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
index 83fdd71c44..79b633e9d7 100644
--- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c
+++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
@@ -135,7 +135,9 @@ int32_t streamStateSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS
   pWriter->sver = sver;
   pWriter->ever = ever;
 
-  sprintf(tdir, "%s%s%s", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM);
+  sprintf(tdir, "%s%s%s%s%s", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM, TD_DIRSEP, "received");
+  taosMkDir(tdir);
+
   SStreamSnapWriter* pSnapWriter = NULL;
   if (streamSnapWriterOpen(pTq, sver, ever, tdir, &pSnapWriter) < 0) {
     goto _err;
@@ -143,6 +145,8 @@ int32_t streamStateSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS
 
   tqDebug("vgId:%d, vnode stream-state snapshot writer opened, path:%s", TD_VID(pTq->pVnode), tdir);
   pWriter->pWriterImpl = pSnapWriter;
+
+  *ppWriter = pWriter;
   return code;
 _err:
   tqError("vgId:%d, vnode stream-state snapshot writer failed to open since %s", TD_VID(pTq->pVnode), tstrerror(code));
diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c
index 5ae0f6f30e..0bf029f574 100644
--- a/source/libs/stream/src/streamSnapshot.c
+++ b/source/libs/stream/src/streamSnapshot.c
@@ -361,8 +361,6 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path
   pHandle->currFileIdx = 0;
   pHandle->offset = 0;
 
-  SBackendFileItem* pItem = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx);
-  pHandle->fd = streamOpenFile(pFile->path, pItem->name, TD_FILE_WRITE);
   *ppWriter = pWriter;
   return 0;
 }
@@ -373,14 +371,25 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa
   SStreamSnapBlockHdr* pHdr = (SStreamSnapBlockHdr*)pData;
   SStreamSnapHandle*   pHandle = &pWriter->handle;
   SBanckendFile*       pFile = pHandle->pBackendFile;
-  SBackendFileItem*    pItem = taosArrayGetP(pHandle->pFileList, pHandle->currFileIdx);
-  if (strlen(pHdr->name) == strlen(pItem->name) && strcmp(pHdr->name, pItem->name) == 0) {
-    if (taosPWriteFile(pHandle->fd, pHdr->data, pHdr->size, pHandle->offset) != pHdr->size) {
+  SBackendFileItem*    pItem = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx);
+
+  if (pHandle->fd == NULL) {
+    pHandle->fd = streamOpenFile(pFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
+    if (pHandle->fd == NULL) {
       code = TAOS_SYSTEM_ERROR(terrno);
-      qError("stream snap failed to write snap, file name:%s, reason:%s", pHdr->name, tstrerror(code));
+      qError("stream-state failed to open file name:%s%s%s, reason:%s", pFile->path, TD_DIRSEP, pHdr->name,
+             tstrerror(code));
+    }
+  }
+
+  if (strlen(pHdr->name) == strlen(pItem->name) && strcmp(pHdr->name, pItem->name) == 0) {
+    int64_t bytes = taosPWriteFile(pHandle->fd, pHdr->data, pHdr->size, pHandle->offset);
+    if (bytes != pHdr->size) {
+      code = TAOS_SYSTEM_ERROR(terrno);
+      qError("stream-state failed to write snap, file name:%s, reason:%s", pHdr->name, tstrerror(code));
       return code;
     }
-    pHandle->offset += pHdr->size;
+    pHandle->offset += bytes;
   } else {
     taosCloseFile(&pHandle->fd);
     pHandle->offset = 0;
@@ -392,7 +401,12 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa
     taosArrayPush(pHandle->pFileList, &item);
 
     SBackendFileItem* pItem = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx);
-    pHandle->fd = streamOpenFile(pFile->path, pItem->name, TD_FILE_WRITE);
+    pHandle->fd = streamOpenFile(pFile->path, pItem->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
+    if (pHandle->fd == NULL) {
+      code = TAOS_SYSTEM_ERROR(terrno);
+      qError("stream-state failed to open file name:%s%s%s, reason:%s", pFile->path, TD_DIRSEP, pHdr->name,
+             tstrerror(code));
+    }
 
     taosPWriteFile(pHandle->fd, pHdr->data, pHdr->size, pHandle->offset);
     pHandle->offset += pHdr->size;

From 4aef18733f364d6b7b7dc3308df2669e5fe909ec Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 2 Aug 2023 03:16:24 +0000
Subject: [PATCH 11/36] fix state mem leak

---
 source/dnode/mnode/impl/src/mndStream.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c
index d0300c622f..881bdef152 100644
--- a/source/dnode/mnode/impl/src/mndStream.c
+++ b/source/dnode/mnode/impl/src/mndStream.c
@@ -33,6 +33,7 @@
 
 #define MND_STREAM_MAX_NUM 60
 
+#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"
 static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
 static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
 static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream);
@@ -1129,7 +1130,6 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
     }
   }
 
-  pStream->checkpointFreq = checkpointId;
   pStream->checkpointId = checkpointId;
   pStream->checkpointFreq = taosGetTimestampMs();
   atomic_store_64(&pStream->currentTick, 0);
@@ -1165,7 +1165,7 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
   SMStreamDoCheckpointMsg *pMsg = (SMStreamDoCheckpointMsg *)pReq->pCont;
   int64_t                  checkpointId = pMsg->checkpointId;
 
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, "stream-checkpoint");
+  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, MND_STREAM_CHECKPOINT_NAME);
   if (pTrans == NULL) {
     mError("failed to trigger checkpoint, reason: %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));
     return -1;

From f9d0874e4525a2adb1cc8dd17ea3a168746da85b Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 3 Aug 2023 08:24:50 +0000
Subject: [PATCH 12/36] support reopen stream state

---
 include/libs/stream/tstream.h                 |  5 +-
 source/dnode/vnode/src/inc/vnodeInt.h         |  8 +-
 source/dnode/vnode/src/tq/tqStreamStateSnap.c |  3 +
 source/libs/stream/src/streamBackendRocksdb.c | 67 ++++++++--------
 source/libs/stream/src/streamMeta.c           | 79 +++++++++++++------
 5 files changed, 102 insertions(+), 60 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index efc7901d4a..51a31f72ed 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -369,7 +369,7 @@ typedef struct SStreamMeta {
 
   int32_t chkptNotReadyTasks;
 
-  int64_t checkpointId;
+  int64_t  checkpointId;
   SArray*  checkpointSaved;
   SArray*  checkpointInUse;
   int32_t  checkpointCap;
@@ -601,6 +601,7 @@ int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, int32_t childId);
 void         streamMetaInit();
 void         streamMetaCleanup();
 SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId);
+
 void         streamMetaClose(SStreamMeta* streamMeta);
 int32_t      streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask);
 int32_t      streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask);
@@ -610,6 +611,8 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 void         streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
 
+int32_t streamStateRebuild(SStreamMeta* pMeta, char* path, int64_t chkpId);
+
 int32_t streamMetaBegin(SStreamMeta* pMeta);
 int32_t streamMetaCommit(SStreamMeta* pMeta);
 int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver);
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 2f37048d5c..cb7af681ee 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -93,7 +93,7 @@ typedef struct SQueryNode         SQueryNode;
 
 #define VNODE_BUFPOOL_SEGMENTS 3
 
-#define VND_INFO_FNAME "vnode.json"
+#define VND_INFO_FNAME     "vnode.json"
 #define VND_INFO_FNAME_TMP "vnode_tmp.json"
 
 // vnd.h
@@ -181,8 +181,8 @@ SArray*       metaGetSmaTbUids(SMeta* pMeta);
 void*         metaGetIdx(SMeta* pMeta);
 void*         metaGetIvtIdx(SMeta* pMeta);
 
-int64_t       metaGetTbNum(SMeta *pMeta);
-void          metaReaderDoInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
+int64_t metaGetTbNum(SMeta* pMeta);
+void    metaReaderDoInit(SMetaReader* pReader, SMeta* pMeta, int32_t flags);
 
 int32_t metaCreateTSma(SMeta* pMeta, int64_t version, SSmaCfg* pCfg);
 int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
@@ -329,6 +329,8 @@ int32_t streamStateSnapRead(SStreamStateReader* pReader, uint8_t** ppData);
 int32_t streamStateSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamStateWriter** ppWriter);
 int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback);
 int32_t streamStateSnapWrite(SStreamStateWriter* pWriter, uint8_t* pData, uint32_t nData);
+int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, char* path, int64_t chkpId);
+
 // SStreamTaskReader ======================================
 // SStreamStateWriter =====================================
 // SStreamStateReader =====================================
diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
index 79b633e9d7..3478928c4f 100644
--- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c
+++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
@@ -162,6 +162,9 @@ int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback)
   taosMemoryFree(pWriter);
   return code;
 }
+int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, char* path, int64_t chkpId) {
+  return streamStateRebuild(pWriter->pTq->pStreamMeta, path, chkpId);
+}
 
 int32_t streamStateSnapWrite(SStreamStateWriter* pWriter, uint8_t* pData, uint32_t nData) {
   int32_t code = 0;
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index f643e5b1e8..2e9032a47e 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -158,7 +158,6 @@ int32_t copyFiles(const char* src, const char* dst) {
   if (pDir == NULL) return 0;
 
   TdDirEntryPtr de = NULL;
-
   while ((de = taosReadDir(pDir)) != NULL) {
     char* name = taosGetDirEntryName(de);
     if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue;
@@ -186,7 +185,11 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) {
   // impl later
   int32_t code = 0;
 
-  // chkpId = 0;
+  /*param@1: checkpointId dir
+    param@2: state
+    copy checkpointdir's file to state dir
+    opt to set hard link to previous file
+  */
   char* state = taosMemoryCalloc(1, strlen(path) + 32);
   sprintf(state, "%s%s%s", path, TD_DIRSEP, "state");
   if (chkpId != 0) {
@@ -1469,10 +1472,11 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) {
   qDebug("streamStateClear_rocksdb");
 
   SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper;
-  char               sKeyStr[128] = {0};
-  char               eKeyStr[128] = {0};
-  SStateKey          sKey = {.key = {.ts = 0, .groupId = 0}, .opNum = pState->number};
-  SStateKey          eKey = {.key = {.ts = INT64_MAX, .groupId = UINT64_MAX}, .opNum = pState->number};
+
+  char      sKeyStr[128] = {0};
+  char      eKeyStr[128] = {0};
+  SStateKey sKey = {.key = {.ts = 0, .groupId = 0}, .opNum = pState->number};
+  SStateKey eKey = {.key = {.ts = INT64_MAX, .groupId = UINT64_MAX}, .opNum = pState->number};
 
   int sLen = stateKeyEncode(&sKey, sKeyStr);
   int eLen = stateKeyEncode(&eKey, eKeyStr);
@@ -1507,9 +1511,9 @@ int32_t streamStateGetFirst_rocksdb(SStreamState* pState, SWinKey* key) {
   qDebug("streamStateGetFirst_rocksdb");
   SWinKey tmp = {.ts = 0, .groupId = 0};
   streamStatePut_rocksdb(pState, &tmp, NULL, 0);
-  SStreamStateCur* pCur = streamStateSeekKeyNext_rocksdb(pState, &tmp);
 
-  int32_t code = streamStateGetKVByCur_rocksdb(pCur, key, NULL, 0);
+  SStreamStateCur* pCur = streamStateSeekKeyNext_rocksdb(pState, &tmp);
+  int32_t          code = streamStateGetKVByCur_rocksdb(pCur, key, NULL, 0);
   streamStateFreeCur(pCur);
   streamStateDel_rocksdb(pState, &tmp);
   return code;
@@ -1523,10 +1527,8 @@ int32_t streamStateGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey,
   uint64_t groupId = pKey->groupId;
 
   int32_t code = streamStateFillGetKVByCur_rocksdb(pCur, pKey, pVal, pVLen);
-  if (code == 0) {
-    if (pKey->groupId == groupId) {
-      return 0;
-    }
+  if (code == 0 && pKey->groupId == groupId) {
+    return 0;
   }
   return -1;
 }
@@ -1628,6 +1630,9 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinK
 
   const SStateKey maxStateKey = {.key = {.groupId = UINT64_MAX, .ts = INT64_MAX}, .opNum = INT64_MAX};
   STREAM_STATE_PUT_ROCKSDB(pState, "state", &maxStateKey, "", 0);
+  if (code != 0) {
+    return NULL;
+  }
 
   char    buf[128] = {0};
   int32_t klen = stateKeyEncode((void*)&maxStateKey, buf);
@@ -1664,9 +1669,10 @@ SStreamStateCur* streamStateSeekToLast_rocksdb(SStreamState* pState, const SWinK
 SStreamStateCur* streamStateGetCur_rocksdb(SStreamState* pState, const SWinKey* key) {
   qDebug("streamStateGetCur_rocksdb");
   SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper;
-  SStreamStateCur*   pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
 
+  SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
   if (pCur == NULL) return NULL;
+
   pCur->db = wrapper->rocksdb;
   pCur->iter = streamStateIterCreate(pState, "state", (rocksdb_snapshot_t**)&pCur->snapshot,
                                      (rocksdb_readoptions_t**)&pCur->readOpt);
@@ -1715,8 +1721,6 @@ int32_t streamStateSessionPut_rocksdb(SStreamState* pState, const SSessionKey* k
   int              code = 0;
   SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
   STREAM_STATE_PUT_ROCKSDB(pState, "sess", &sKey, value, vLen);
-  if (code == -1) {
-  }
   return code;
 }
 int32_t streamStateSessionGet_rocksdb(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) {
@@ -1726,21 +1730,21 @@ int32_t streamStateSessionGet_rocksdb(SStreamState* pState, SSessionKey* key, vo
   SSessionKey      resKey = *key;
   void*            tmp = NULL;
   int32_t          vLen = 0;
-  code = streamStateSessionGetKVByCur_rocksdb(pCur, &resKey, &tmp, &vLen);
-  if (code == 0) {
-    if (pVLen != NULL) *pVLen = vLen;
 
-    if (key->win.skey != resKey.win.skey) {
-      code = -1;
-    } else {
-      *key = resKey;
-      *pVal = taosMemoryCalloc(1, *pVLen);
-      memcpy(*pVal, tmp, *pVLen);
-    }
+  code = streamStateSessionGetKVByCur_rocksdb(pCur, &resKey, &tmp, &vLen);
+  if (code == 0 && key->win.skey != resKey.win.skey) {
+    *key = resKey;
+
+    if (pVal) {
+      *pVal = tmp;
+      tmp = NULL;
+    };
+    if (pVLen) *pVLen = vLen;
+  } else {
+    code = -1;
   }
+
   taosMemoryFree(tmp);
-  streamStateFreeCur(pCur);
-  // impl later
   return code;
 }
 
@@ -1786,8 +1790,6 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentPrev_rocksdb(SStreamState* pSta
 
   rocksdb_iter_prev(pCur->iter);
   if (!rocksdb_iter_valid(pCur->iter)) {
-    // qWarn("streamState failed to seek key prev
-    // %s", toString);
     streamStateFreeCur(pCur);
     return NULL;
   }
@@ -1805,10 +1807,10 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext_rocksdb(SStreamState* pSta
                                      (rocksdb_readoptions_t**)&pCur->readOpt);
   pCur->number = pState->number;
 
-  char buf[128] = {0};
-
+  char             buf[128] = {0};
   SStateSessionKey sKey = {.key = *key, .opNum = pState->number};
   int              len = stateSessionKeyEncode(&sKey, buf);
+
   if (!streamStateIterSeekAndValid(pCur->iter, buf, len)) {
     streamStateFreeCur(pCur);
     return NULL;
@@ -1856,6 +1858,7 @@ SStreamStateCur* streamStateSessionSeekKeyNext_rocksdb(SStreamState* pState, con
     streamStateFreeCur(pCur);
     return NULL;
   }
+
   size_t           klen;
   const char*      iKey = rocksdb_iter_key(pCur->iter, &klen);
   SStateSessionKey curKey = {0};
@@ -2391,8 +2394,8 @@ void    streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rock
 int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb_writebatch_t* pBatch, void* key,
                             void* val, int32_t vlen, int64_t ttl) {
   SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper;
-  int                i = streamStateGetCfIdx(pState, cfKeyName);
 
+  int i = streamStateGetCfIdx(pState, cfKeyName);
   if (i < 0) {
     qError("streamState failed to put to cf name:%s", cfKeyName);
     return -1;
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 4711f4af19..3cf967a219 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -36,6 +36,14 @@ void streamMetaCleanup() {
   taosCloseRef(streamBackendCfWrapperId);
 }
 
+int32_t streamStateRebuild(SStreamMeta* pMeta, char* path, int64_t chkpId) {
+  int32_t code = 0;
+
+  int32_t nTask = taosHashGetSize(pMeta->pTasks);
+  assert(nTask == 0);
+
+  return code;
+}
 SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId) {
   int32_t      code = -1;
   SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta));
@@ -44,22 +52,13 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
     return NULL;
   }
 
-  int32_t len = strlen(path) + 20;
-  char*   streamPath = taosMemoryCalloc(1, len);
-  sprintf(streamPath, "%s/%s", path, "stream");
-  pMeta->path = taosStrdup(streamPath);
+  char* tpath = taosMemoryCalloc(1, strlen(path) + 64);
+  sprintf(tpath, "%s%s%s", path, TD_DIRSEP, "stream");
+  pMeta->path = tpath;
+
   if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0) < 0) {
     goto _err;
   }
-
-  memset(streamPath, 0, len);
-  sprintf(streamPath, "%s/%s", pMeta->path, "checkpoints");
-  code = taosMulModeMkDir(streamPath, 0755);
-  if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(code);
-    goto _err;
-  }
-
   if (tdbTbOpen("task.db", sizeof(int32_t), -1, NULL, pMeta->db, &pMeta->pTaskDb, 0) < 0) {
     goto _err;
   }
@@ -90,14 +89,6 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
   pMeta->ahandle = ahandle;
   pMeta->expandFunc = expandFunc;
 
-  // memset(streamPath, 0, len);
-  // sprintf(streamPath, "%s/%s", pMeta->path, "state");
-  // code = taosMulModeMkDir(streamPath, 0755);
-  // if (code != 0) {
-  //   terrno = TAOS_SYSTEM_ERROR(code);
-  //   goto _err;
-  // }
-
   pMeta->pTaskBackendUnique =
       taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
   pMeta->checkpointSaved = taosArrayInit(4, sizeof(int64_t));
@@ -119,15 +110,12 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
     goto _err;
   }
 
-  taosMemoryFree(streamPath);
-
   taosInitRWLatch(&pMeta->lock);
   taosThreadMutexInit(&pMeta->backendMutex, NULL);
 
   return pMeta;
 
 _err:
-  taosMemoryFree(streamPath);
   taosMemoryFree(pMeta->path);
   if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks);
   if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList);
@@ -140,6 +128,49 @@ _err:
   return NULL;
 }
 
+void streamMetaReopen(SStreamMeta** ppMeta) {
+  SStreamMeta* pMeta = *ppMeta;
+
+  SStreamMeta* pNewMeta = taosMemoryCalloc(1, sizeof(SStreamMeta));
+  pNewMeta->path = taosStrdup(pMeta->path);
+  pNewMeta->vgId = pMeta->vgId;
+  pNewMeta->walScanCounter = 0;
+  pNewMeta->ahandle = pMeta->ahandle;
+  pNewMeta->expandFunc = pMeta->expandFunc;
+
+  *ppMeta = pNewMeta;
+
+  streamMetaClose(pMeta);
+
+  // tdbAbort(pMeta->db, pMeta->txn);
+  // tdbTbClose(pMeta->pTaskDb);
+  // tdbTbClose(pMeta->pCheckpointDb);
+  // tdbClose(pMeta->db);
+
+  // void* pIter = NULL;
+  // while (1) {
+  //   pIter = taosHashIterate(pMeta->pTasks, pIter);
+  //   if (pIter == NULL) {
+  //     break;
+  //   }
+
+  //   SStreamTask* pTask = *(SStreamTask**)pIter;
+  //   if (pTask->schedTimer) {
+  //     taosTmrStop(pTask->schedTimer);
+  //     pTask->schedTimer = NULL;
+  //   }
+
+  //   if (pTask->launchTaskTimer) {
+  //     taosTmrStop(pTask->launchTaskTimer);
+  //     pTask->launchTaskTimer = NULL;
+  //   }
+
+  //   tFreeStreamTask(pTask);
+  // }
+
+  // taosHashClear(pMeta->pTasks);
+  // taosRemoveRef(streamBackendId, pMeta->streamBackendRid);
+}
 void streamMetaClose(SStreamMeta* pMeta) {
   tdbAbort(pMeta->db, pMeta->txn);
   tdbTbClose(pMeta->pTaskDb);

From 7396799d90cd321a0c824be4f2ece2d4d45ac6db Mon Sep 17 00:00:00 2001
From: yihaoDeng <cyang@taosdata.com>
Date: Fri, 4 Aug 2023 19:06:16 +0800
Subject: [PATCH 13/36] fix invalid free

---
 source/libs/stream/src/streamBackendRocksdb.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index f643e5b1e8..6cd81058da 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -296,12 +296,14 @@ _EXIT:
 }
 void streamBackendCleanup(void* arg) {
   SBackendWrapper* pHandle = (SBackendWrapper*)arg;
-  RocksdbCfInst**  pIter = (RocksdbCfInst**)taosHashIterate(pHandle->cfInst, NULL);
+
+  void* pIter = taosHashIterate(pHandle->cfInst, NULL);
   while (pIter != NULL) {
-    RocksdbCfInst* inst = *pIter;
+    RocksdbCfInst* inst = *(RocksdbCfInst**)pIter;
     destroyRocksdbCfInst(inst);
-    taosHashIterate(pHandle->cfInst, pIter);
+    pIter = taosHashIterate(pHandle->cfInst, pIter);
   }
+
   taosHashCleanup(pHandle->cfInst);
 
   if (pHandle->db) {
@@ -1133,9 +1135,9 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t
       inst->pHandle[idx] = cfHandle[i];
     }
   }
-  void** pIter = taosHashIterate(handle->cfInst, NULL);
+  void* pIter = taosHashIterate(handle->cfInst, NULL);
   while (pIter) {
-    RocksdbCfInst* inst = *pIter;
+    RocksdbCfInst* inst = *(RocksdbCfInst**)pIter;
 
     for (int i = 0; i < cfLen; i++) {
       if (inst->cfOpt[i] == NULL) {

From 4ca39d0f3c4c2224a8b371381f1177d9af89dcd2 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 7 Aug 2023 01:45:16 +0000
Subject: [PATCH 14/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 2e9032a47e..f24186c673 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -299,11 +299,12 @@ _EXIT:
 }
 void streamBackendCleanup(void* arg) {
   SBackendWrapper* pHandle = (SBackendWrapper*)arg;
-  RocksdbCfInst**  pIter = (RocksdbCfInst**)taosHashIterate(pHandle->cfInst, NULL);
+
+  void* pIter = taosHashIterate(pHandle->cfInst, NULL);
   while (pIter != NULL) {
-    RocksdbCfInst* inst = *pIter;
+    RocksdbCfInst* inst = *(RocksdbCfInst**)pIter;
     destroyRocksdbCfInst(inst);
-    taosHashIterate(pHandle->cfInst, pIter);
+    pIter = taosHashIterate(pHandle->cfInst, pIter);
   }
   taosHashCleanup(pHandle->cfInst);
 
@@ -1103,7 +1104,8 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t
   static int32_t cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]);
   for (int i = 0; i < nCf; i++) {
     char* cf = cfs[i];
-    if (i == 0) continue;
+    if (i == 0) continue;  // skip default column family, not set opt
+
     char funcname[64] = {0};
     if (3 == sscanf(cf, "0x%" PRIx64 "-%d_%s", &streamId, &taskId, funcname)) {
       char idstr[128] = {0};
@@ -1125,7 +1127,6 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t
         inst->pCompares = taosMemoryCalloc(cfLen, sizeof(rocksdb_comparator_t*));
 
         inst->dbOpt = handle->dbOpt;
-        // rocksdb_writeoptions_disable_WAL(inst->wOpt, 1);
         taosHashPut(handle->cfInst, idstr, strlen(idstr) + 1, &inst, sizeof(void*));
       } else {
         inst = *pInst;
@@ -1136,9 +1137,9 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t
       inst->pHandle[idx] = cfHandle[i];
     }
   }
-  void** pIter = taosHashIterate(handle->cfInst, NULL);
+  void* pIter = taosHashIterate(handle->cfInst, NULL);
   while (pIter) {
-    RocksdbCfInst* inst = *pIter;
+    RocksdbCfInst* inst = *(RocksdbCfInst**)pIter;
 
     for (int i = 0; i < cfLen; i++) {
       if (inst->cfOpt[i] == NULL) {
@@ -1179,8 +1180,8 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
   taosAcquireRef(streamBackendId, pState->streamBackendRid);
   SBackendWrapper*   handle = backend;
   SBackendCfWrapper* pBackendCfWrapper = taosMemoryCalloc(1, sizeof(SBackendCfWrapper));
-  taosThreadMutexLock(&handle->cfMutex);
 
+  taosThreadMutexLock(&handle->cfMutex);
   RocksdbCfInst** ppInst = taosHashGet(handle->cfInst, pState->pTdbState->idstr, strlen(pState->pTdbState->idstr) + 1);
   if (ppInst != NULL && *ppInst != NULL) {
     RocksdbCfInst* inst = *ppInst;

From 7ef4df87526517993a83c9995a309228a08a8083 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 7 Aug 2023 07:20:05 +0000
Subject: [PATCH 15/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 54 +++++++++----------
 1 file changed, 24 insertions(+), 30 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index f24186c673..ba12c47bc0 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -408,6 +408,7 @@ int32_t getLatestCheckpoint(void* arg, int64_t* checkpoint) {
   int64_t tc = 0;
   int32_t sz = taosArrayGetSize(pMeta->checkpointSaved);
   if (sz <= 0) {
+    taosWUnLockLatch(&pMeta->checkpointDirLock);
     return -1;
   } else {
     tc = *(int64_t*)taosArrayGetLast(pMeta->checkpointSaved);
@@ -623,9 +624,9 @@ int defaultKeyComp(void* state, const char* aBuf, size_t aLen, const char* bBuf,
     return ret;
   }
 }
-int streamStateValueIsStale(char* vv) {
+int streamStateValueIsStale(char* v) {
   int64_t ts = 0;
-  taosDecodeFixedI64(vv, &ts);
+  taosDecodeFixedI64(v, &ts);
   return (ts != 0 && ts < taosGetTimestampMs()) ? 1 : 0;
 }
 int iterValueIsStale(rocksdb_iterator_t* iter) {
@@ -956,33 +957,23 @@ int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest) {
   SStreamValue key = {0};
   char*        p = value;
   if (streamStateValueIsStale(p)) {
-    if (dest != NULL) *dest = NULL;
-    return -1;
+    goto _EXCEPT;
   }
   p = taosDecodeFixedI64(p, &key.unixTimestamp);
   p = taosDecodeFixedI32(p, &key.len);
   if (vlen != (sizeof(int64_t) + sizeof(int32_t) + key.len)) {
-    if (dest != NULL) *dest = NULL;
     qError("vlen: %d, read len: %d", vlen, key.len);
-    return -1;
+    goto _EXCEPT;
   }
+  if (key.len != 0 && dest != NULL) p = taosDecodeBinary(p, (void**)dest, key.len);
 
-  if (key.len == 0) {
-    key.data = NULL;
-  } else {
-    p = taosDecodeBinary(p, (void**)&(key.data), key.len);
-  }
-
-  if (ttl != NULL) {
-    int64_t now = taosGetTimestampMs();
-    *ttl = key.unixTimestamp == 0 ? 0 : key.unixTimestamp - now;
-  }
-  if (dest != NULL) {
-    *dest = key.data;
-  } else {
-    taosMemoryFree(key.data);
-  }
+  if (ttl != NULL) *ttl = key.unixTimestamp == 0 ? 0 : key.unixTimestamp - taosGetTimestampMs();
   return key.len;
+
+_EXCEPT:
+  if (dest != NULL) *dest = NULL;
+  if (ttl != NULL) *ttl = 0;
+  return -1;
 }
 
 const char* compareDefaultName(void* arg) {
@@ -1096,9 +1087,10 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t
   } else {
     qDebug("succ to open rocksdb cf");
   }
-  // close default cf
+  // close default cf and destroy default cfOpts
   if (((rocksdb_column_family_handle_t**)cfHandle)[0] != 0) rocksdb_column_family_handle_destroy(cfHandle[0]);
   rocksdb_options_destroy(cfOpts[0]);
+
   handle->db = db;
 
   static int32_t cfLen = sizeof(ginitDict) / sizeof(ginitDict[0]);
@@ -2354,9 +2346,7 @@ void* streamDefaultIterCreate_rocksdb(SStreamState* pState) {
 }
 int32_t streamDefaultIterValid_rocksdb(void* iter) {
   SStreamStateCur* pCur = iter;
-  bool             val = rocksdb_iter_valid(pCur->iter);
-
-  return val ? 1 : 0;
+  return rocksdb_iter_valid(pCur->iter) ? 1 : 0;
 }
 void streamDefaultIterSeek_rocksdb(void* iter, const char* key) {
   SStreamStateCur* pCur = iter;
@@ -2372,13 +2362,16 @@ char* streamDefaultIterKey_rocksdb(void* iter, int32_t* len) {
 }
 char* streamDefaultIterVal_rocksdb(void* iter, int32_t* len) {
   SStreamStateCur* pCur = iter;
-  int32_t          vlen = 0;
-  char*            dst = NULL;
-  const char*      vval = rocksdb_iter_value(pCur->iter, (size_t*)&vlen);
-  if (decodeValueFunc((void*)vval, vlen, NULL, &dst) < 0) {
+  char*            ret = NULL;
+
+  int32_t     vlen = 0;
+  const char* val = rocksdb_iter_value(pCur->iter, (size_t*)&vlen);
+  *len = decodeValueFunc((void*)val, vlen, NULL, &ret);
+  if (*len < 0) {
     return NULL;
   }
-  return dst;
+
+  return ret;
 }
 // batch func
 void* streamStateCreateBatch() {
@@ -2433,6 +2426,7 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb
   if (tmpBuf == NULL) {
     taosMemoryFree(ttlV);
   }
+
   {
     char tbuf[256] = {0};
     ginitDict[cfIdx].toStrFunc((void*)key, tbuf);

From eeb97351e8712c1156ccf78814f9d36f08940fcc Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 7 Aug 2023 08:51:01 +0000
Subject: [PATCH 16/36] support reopen stream state

---
 include/libs/stream/tstream.h                 |  1 +
 source/dnode/vnode/src/inc/vnodeInt.h         |  3 +
 source/dnode/vnode/src/tq/tqStreamStateSnap.c | 11 ++-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c    |  6 ++
 source/libs/stream/src/streamBackendRocksdb.c |  8 +--
 source/libs/stream/src/streamMeta.c           | 69 ++++++++++---------
 6 files changed, 60 insertions(+), 38 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 51a31f72ed..9c01b40dce 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -612,6 +612,7 @@ void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 void         streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
 
 int32_t streamStateRebuild(SStreamMeta* pMeta, char* path, int64_t chkpId);
+int32_t streamStateReopen(SStreamMeta *pMeta, int64_t chkpId);
 
 int32_t streamMetaBegin(SStreamMeta* pMeta);
 int32_t streamMetaCommit(SStreamMeta* pMeta);
diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index cb7af681ee..18f8872e03 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -331,6 +331,9 @@ int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback)
 int32_t streamStateSnapWrite(SStreamStateWriter* pWriter, uint8_t* pData, uint32_t nData);
 int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, char* path, int64_t chkpId);
 
+int32_t streamStateLoadTasks(SStreamStateWriter* pWriter);
+
+
 // SStreamTaskReader ======================================
 // SStreamStateWriter =====================================
 // SStreamStateReader =====================================
diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
index 3478928c4f..f1bdc0d6de 100644
--- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c
+++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
@@ -163,7 +163,16 @@ int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback)
   return code;
 }
 int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, char* path, int64_t chkpId) {
-  return streamStateRebuild(pWriter->pTq->pStreamMeta, path, chkpId);
+  return streamStateReopen(pWriter->pTq->pStreamMeta, chkpId);
+}
+
+int32_t streamStateLoadTasksImpl(SStreamMeta* pMeta, int64_t ver) {
+  // impl later
+  return streamLoadTasks(pMeta, ver);
+}
+int32_t streamStateLoadTasks(SStreamStateWriter* pWriter) {
+  SWal* pWal = pWriter->pTq->pVnode->pWal;
+  return streamStateLoadTasksImpl(pWriter->pTq->pStreamMeta, walGetCommittedVer(pWal));
 }
 
 int32_t streamStateSnapWrite(SStreamStateWriter* pWriter, uint8_t* pData, uint32_t nData) {
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index 16bd233807..be5fb4785c 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -409,6 +409,12 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
   if (pWriter->pStreamStateWriter) {
     code = streamStateSnapWriterClose(pWriter->pStreamStateWriter, rollback);
     if (code) goto _exit;
+
+    code = streamStateRebuildFromSnap(pWriter->pStreamStateWriter, NULL, 0);
+    if (code) goto _exit;
+
+    code = streamStateLoadTasks(pWriter->pStreamStateWriter);
+    if (code) goto _exit;
   }
 
   if (pWriter->pRsmaSnapWriter) {
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index ba12c47bc0..c63942c8cc 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -732,8 +732,8 @@ int stateSessionKeyDBComp(void* state, const char* aBuf, size_t aLen, const char
 
   return stateSessionKeyCmpr(&w1, sizeof(w1), &w2, sizeof(w2));
 }
-int stateSessionKeyEncode(void* ses, char* buf) {
-  SStateSessionKey* sess = ses;
+int stateSessionKeyEncode(void* k, char* buf) {
+  SStateSessionKey* sess = k;
   int               len = 0;
   len += taosEncodeFixedI64((void**)&buf, sess->key.win.skey);
   len += taosEncodeFixedI64((void**)&buf, sess->key.win.ekey);
@@ -741,8 +741,8 @@ int stateSessionKeyEncode(void* ses, char* buf) {
   len += taosEncodeFixedI64((void**)&buf, sess->opNum);
   return len;
 }
-int stateSessionKeyDecode(void* ses, char* buf) {
-  SStateSessionKey* sess = ses;
+int stateSessionKeyDecode(void* k, char* buf) {
+  SStateSessionKey* sess = k;
   int               len = 0;
 
   char* p = buf;
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 3cf967a219..e93499ab89 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -128,48 +128,51 @@ _err:
   return NULL;
 }
 
-void streamMetaReopen(SStreamMeta** ppMeta) {
-  SStreamMeta* pMeta = *ppMeta;
+void streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId) {
+  // stop all running tasking and reopen later
+  void* pIter = NULL;
+  while (1) {
+    pIter = taosHashIterate(pMeta->pTasks, pIter);
+    if (pIter == NULL) {
+      break;
+    }
 
-  SStreamMeta* pNewMeta = taosMemoryCalloc(1, sizeof(SStreamMeta));
-  pNewMeta->path = taosStrdup(pMeta->path);
-  pNewMeta->vgId = pMeta->vgId;
-  pNewMeta->walScanCounter = 0;
-  pNewMeta->ahandle = pMeta->ahandle;
-  pNewMeta->expandFunc = pMeta->expandFunc;
+    SStreamTask* pTask = *(SStreamTask**)pIter;
+    if (pTask->schedTimer) {
+      taosTmrStop(pTask->schedTimer);
+      pTask->schedTimer = NULL;
+    }
 
-  *ppMeta = pNewMeta;
+    if (pTask->launchTaskTimer) {
+      taosTmrStop(pTask->launchTaskTimer);
+      pTask->launchTaskTimer = NULL;
+    }
 
-  streamMetaClose(pMeta);
+    tFreeStreamTask(pTask);
+  }
 
-  // tdbAbort(pMeta->db, pMeta->txn);
-  // tdbTbClose(pMeta->pTaskDb);
-  // tdbTbClose(pMeta->pCheckpointDb);
-  // tdbClose(pMeta->db);
+  // close stream backend
+  streamBackendCleanup(pMeta->streamBackend);
+  taosRemoveRef(streamBackendId, pMeta->streamBackendRid);
+  pMeta->streamBackendRid = -1;
+  pMeta->streamBackend = NULL;
 
-  // void* pIter = NULL;
-  // while (1) {
-  //   pIter = taosHashIterate(pMeta->pTasks, pIter);
-  //   if (pIter == NULL) {
-  //     break;
-  //   }
+  pMeta->streamBackend = streamBackendInit(pMeta->path, chkpId);
+  pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend);
 
-  //   SStreamTask* pTask = *(SStreamTask**)pIter;
-  //   if (pTask->schedTimer) {
-  //     taosTmrStop(pTask->schedTimer);
-  //     pTask->schedTimer = NULL;
-  //   }
+  taosHashClear(pMeta->pTasks);
 
-  //   if (pTask->launchTaskTimer) {
-  //     taosTmrStop(pTask->launchTaskTimer);
-  //     pTask->launchTaskTimer = NULL;
-  //   }
+  taosArrayClear(pMeta->pTaskList);
 
-  //   tFreeStreamTask(pTask);
-  // }
+  taosHashClear(pMeta->pTaskBackendUnique);
 
-  // taosHashClear(pMeta->pTasks);
-  // taosRemoveRef(streamBackendId, pMeta->streamBackendRid);
+  taosArrayClear(pMeta->checkpointSaved);
+
+  taosArrayClear(pMeta->checkpointInUse);
+
+  // if (streamLoadTasks(pMeta,int64_t ver))
+
+  return;
 }
 void streamMetaClose(SStreamMeta* pMeta) {
   tdbAbort(pMeta->db, pMeta->txn);

From b6c991f896129ff512212f18b5f8c14b6ffd7d41 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 7 Aug 2023 08:59:04 +0000
Subject: [PATCH 17/36] support reopen stream state

---
 include/libs/stream/tstream.h                 |  4 ++--
 source/dnode/vnode/src/tq/tqStreamStateSnap.c |  6 +++++-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c    |  3 ---
 source/libs/stream/src/streamMeta.c           | 16 ++++++++--------
 4 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 9c01b40dce..4c0005ece7 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -611,8 +611,8 @@ SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
 void         streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId);
 
-int32_t streamStateRebuild(SStreamMeta* pMeta, char* path, int64_t chkpId);
-int32_t streamStateReopen(SStreamMeta *pMeta, int64_t chkpId);
+// int32_t streamStateRebuild(SStreamMeta* pMeta, char* path, int64_t chkpId);
+int32_t streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId);
 
 int32_t streamMetaBegin(SStreamMeta* pMeta);
 int32_t streamMetaCommit(SStreamMeta* pMeta);
diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
index f1bdc0d6de..87d174715d 100644
--- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c
+++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
@@ -163,7 +163,11 @@ int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback)
   return code;
 }
 int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, char* path, int64_t chkpId) {
-  return streamStateReopen(pWriter->pTq->pStreamMeta, chkpId);
+  int32_t code = streamMetaReopen(pWriter->pTq->pStreamMeta, chkpId);
+  if (code == 0) {
+    code = streamStateLoadTasks(pWriter);
+  }
+  return code;
 }
 
 int32_t streamStateLoadTasksImpl(SStreamMeta* pMeta, int64_t ver) {
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index be5fb4785c..00f7150c0b 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -412,9 +412,6 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
 
     code = streamStateRebuildFromSnap(pWriter->pStreamStateWriter, NULL, 0);
     if (code) goto _exit;
-
-    code = streamStateLoadTasks(pWriter->pStreamStateWriter);
-    if (code) goto _exit;
   }
 
   if (pWriter->pRsmaSnapWriter) {
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index e93499ab89..5d0861624e 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -36,14 +36,14 @@ void streamMetaCleanup() {
   taosCloseRef(streamBackendCfWrapperId);
 }
 
-int32_t streamStateRebuild(SStreamMeta* pMeta, char* path, int64_t chkpId) {
-  int32_t code = 0;
+// int32_t streamStateRebuild(SStreamMeta* pMeta, char* path, int64_t chkpId) {
+//   int32_t code = 0;
 
-  int32_t nTask = taosHashGetSize(pMeta->pTasks);
-  assert(nTask == 0);
+//   int32_t nTask = taosHashGetSize(pMeta->pTasks);
+//   assert(nTask == 0);
 
-  return code;
-}
+//   return code;
+// }
 SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId) {
   int32_t      code = -1;
   SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta));
@@ -128,7 +128,7 @@ _err:
   return NULL;
 }
 
-void streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId) {
+int32_t streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId) {
   // stop all running tasking and reopen later
   void* pIter = NULL;
   while (1) {
@@ -172,7 +172,7 @@ void streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId) {
 
   // if (streamLoadTasks(pMeta,int64_t ver))
 
-  return;
+  return 0;
 }
 void streamMetaClose(SStreamMeta* pMeta) {
   tdbAbort(pMeta->db, pMeta->txn);

From 6c8c57554934f6b870654fecfd43105737f3f70f Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 7 Aug 2023 10:48:04 +0000
Subject: [PATCH 18/36] support reopen stream state

---
 source/libs/stream/src/streamMeta.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 5d0861624e..7e264128d9 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -157,7 +157,20 @@ int32_t streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId) {
   pMeta->streamBackendRid = -1;
   pMeta->streamBackend = NULL;
 
-  pMeta->streamBackend = streamBackendInit(pMeta->path, chkpId);
+  char* path1 = taosMemoryCalloc(1, strlen(pMeta->path) + 64);
+  sprintf(path1, "%s%s%s", pMeta->path, TD_DIRSEP, "state");
+  taosRemoveDir(path1);
+
+  char* path2 = taosMemoryCalloc(1, strlen(pMeta->path) + 64);
+  sprintf(path2, "%s%s%s", pMeta->path, TD_DIRSEP, "received");
+
+  if (taosRenameFile(path2, path1) < 0) {
+    taosMemoryFree(path1);
+    taosMemoryFree(path2);
+    return -1;
+  }
+
+  pMeta->streamBackend = streamBackendInit(pMeta->path, 0);
   pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend);
 
   taosHashClear(pMeta->pTasks);
@@ -170,8 +183,6 @@ int32_t streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId) {
 
   taosArrayClear(pMeta->checkpointInUse);
 
-  // if (streamLoadTasks(pMeta,int64_t ver))
-
   return 0;
 }
 void streamMetaClose(SStreamMeta* pMeta) {

From 6a69c56b32febe446631d86b57860e2559b83d2e Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 7 Aug 2023 10:55:42 +0000
Subject: [PATCH 19/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index c63942c8cc..54b0e8498e 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -487,15 +487,15 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) {
   int32_t      code = 0;
 
   int32_t len = strlen(pMeta->path) + 30;
-  char*   checkpointPath = taosMemoryCalloc(1, len);
-  sprintf(checkpointPath, "%s%s%s", pMeta->path, TD_DIRSEP, "checkpoints");
+  char*   chkpPath = taosMemoryCalloc(1, len);
+  sprintf(chkpPath, "%s%s%s", pMeta->path, TD_DIRSEP, "checkpoints");
 
-  if (!taosDirExist(checkpointPath)) {
+  if (!taosDirExist(chkpPath)) {
     // no checkpoint, nothing to load
     return 0;
   }
 
-  TdDirPtr pDir = taosOpenDir(checkpointPath);
+  TdDirPtr pDir = taosOpenDir(chkpPath);
   if (pDir == NULL) return 0;
 
   TdDirEntryPtr de = NULL;
@@ -525,7 +525,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) {
 
   taosArrayDestroy(suffix);
   taosCloseDir(&pDir);
-  taosMemoryFree(checkpointPath);
+  taosMemoryFree(chkpPath);
   return 0;
 }
 int32_t streamBackendDoCheckpoint(void* arg, uint64_t checkpointId) {

From 80e78e054d219e09f8fc55ccaa374cba4e99fb5e Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 7 Aug 2023 11:39:32 +0000
Subject: [PATCH 20/36] support reopen stream state

---
 source/libs/stream/src/streamMeta.c | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 7e264128d9..907ff2b48a 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -157,20 +157,23 @@ int32_t streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId) {
   pMeta->streamBackendRid = -1;
   pMeta->streamBackend = NULL;
 
-  char* path1 = taosMemoryCalloc(1, strlen(pMeta->path) + 64);
-  sprintf(path1, "%s%s%s", pMeta->path, TD_DIRSEP, "state");
-  taosRemoveDir(path1);
+  char* defaultPath = taosMemoryCalloc(1, strlen(pMeta->path) + 64);
+  sprintf(defaultPath, "%s%s%s", pMeta->path, TD_DIRSEP, "state");
+  taosRemoveDir(defaultPath);
 
-  char* path2 = taosMemoryCalloc(1, strlen(pMeta->path) + 64);
-  sprintf(path2, "%s%s%s", pMeta->path, TD_DIRSEP, "received");
+  char* newPath = taosMemoryCalloc(1, strlen(pMeta->path) + 64);
+  sprintf(newPath, "%s%s%s", pMeta->path, TD_DIRSEP, "received");
 
-  if (taosRenameFile(path2, path1) < 0) {
-    taosMemoryFree(path1);
-    taosMemoryFree(path2);
+  if (taosRenameFile(newPath, defaultPath) < 0) {
+    taosMemoryFree(defaultPath);
+    taosMemoryFree(newPath);
     return -1;
   }
 
   pMeta->streamBackend = streamBackendInit(pMeta->path, 0);
+  if (pMeta->streamBackend == NULL) {
+    return -1;
+  }
   pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend);
 
   taosHashClear(pMeta->pTasks);

From 6b73fc9c06dc74da04d48fd5ddc69894a770cb8c Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 7 Aug 2023 13:56:53 +0000
Subject: [PATCH 21/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 52 +++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 54b0e8498e..f926efb94d 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -20,6 +20,17 @@
 #include "tcommon.h"
 #include "tref.h"
 
+typedef struct {
+  char*   pCurrent;
+  char*   pManifest;
+  SArray* pSST;
+  int64_t preCkptId;
+  int64_t curChkpId;
+  char*   path;
+  char*   buf;
+  int32_t len;
+} SBackendManager;
+
 typedef struct SCompactFilteFactory {
   void* status;
 } SCompactFilteFactory;
@@ -127,6 +138,46 @@ void    destroyFunc(void* arg);
 int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest);
 int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest);
 
+SBackendManager* backendManagerCreate(char* path) {
+  SBackendManager* p = taosMemoryCalloc(1, sizeof(SBackendManager));
+  p->curChkpId = 0;
+  p->preCkptId = 0;
+  p->pSST = taosArrayInit(64, sizeof(void*));
+  p->path = taosStrdup(path);
+
+  p->len = strlen(path) + 128;
+  p->buf = taosMemoryCalloc(1, p->len);
+  return p;
+}
+
+int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list) {
+  memset(bm->buf, 0, bm->len);
+  sprintf(bm->buf, "%s%scheckpoint%" PRId64 "", bm->path, TD_DIRSEP, chkpId);
+
+  TdDirPtr      pDir = taosOpenDir(bm->buf);
+  TdDirEntryPtr de = NULL;
+
+  while ((de = taosReadDir(pDir)) != NULL) {
+    char* name = taosGetDirEntryName(de);
+    if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue;
+
+
+    // sprintf(absSrcPath, "%s%s%s", src, TD_DIRSEP, name);
+    // sprintf(absDstPath, "%s%s%s", dst, TD_DIRSEP, name);
+    // if (!taosDirEntryIsDir(de)) {
+    //    code = taosCopyFile(absSrcPath, absDstPath);
+    //    if (code == -1) {
+    //      goto _err;
+    //    }
+    //  }
+
+    // memset(absSrcPath, 0, sLen + 64);
+    // memset(absDstPath, 0, dLen + 64);
+  }
+
+  return 0;
+}
+
 SCfInit ginitDict[] = {
     {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName,
      destroyFunc, encodeValueFunc, decodeValueFunc},
@@ -219,6 +270,7 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) {
 
   return 0;
 }
+
 void* streamBackendInit(const char* streamPath, int64_t chkpId) {
   char*   backendPath = NULL;
   int32_t code = rebuildDirFromCheckpoint(streamPath, chkpId, &backendPath);

From ac11537fc8a1e36c044b4dea839a28c66cb97547 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Tue, 8 Aug 2023 07:21:47 +0000
Subject: [PATCH 22/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 97 +++++++++++++++----
 1 file changed, 76 insertions(+), 21 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index f926efb94d..d72901a2a3 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -21,14 +21,16 @@
 #include "tref.h"
 
 typedef struct {
-  char*   pCurrent;
-  char*   pManifest;
-  SArray* pSST;
-  int64_t preCkptId;
-  int64_t curChkpId;
-  char*   path;
-  char*   buf;
-  int32_t len;
+  int8_t    init;
+  char*     pCurrent;
+  char*     pManifest;
+  SArray*   pSST;
+  int64_t   preCkptId;
+  int64_t   curChkpId;
+  SHashObj* pSSTable;
+  char*     path;
+  char*     buf;
+  int32_t   len;
 } SBackendManager;
 
 typedef struct SCompactFilteFactory {
@@ -144,37 +146,90 @@ SBackendManager* backendManagerCreate(char* path) {
   p->preCkptId = 0;
   p->pSST = taosArrayInit(64, sizeof(void*));
   p->path = taosStrdup(path);
-
+  p->pSSTable = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
   p->len = strlen(path) + 128;
   p->buf = taosMemoryCalloc(1, p->len);
   return p;
 }
 
+int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) {
+  int32_t code = 0;
+  size_t  len = 0;
+  void*   pIter = taosHashIterate(p2, NULL);
+  while (pIter) {
+    char* name = taosHashGetKey(pIter, &len);
+    if (!taosHashGet(p1, name, len)) {
+      char* p = taosStrdup(name);
+      taosArrayPush(diff, &p);
+    }
+    pIter = taosHashIterate(p2, pIter);
+  }
+  return code;
+}
+int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) {
+  int32_t code = 0;
+
+  code = compareHashTableImpl(p1, p2, add);
+  code = compareHashTableImpl(p2, p1, del);
+
+  return code;
+}
 int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list) {
+  const char* pCurrent = "CURRENT";
+  int32_t     currLen = strlen(pCurrent);
+
+  const char* pManifest = "MANIFEST-";
+  int32_t     maniLen = strlen(pManifest);
+
+  const char* pSST = ".sst";
+  int32_t     sstLen = strlen(pSST);
+
   memset(bm->buf, 0, bm->len);
   sprintf(bm->buf, "%s%scheckpoint%" PRId64 "", bm->path, TD_DIRSEP, chkpId);
 
+  SHashObj* pTable = bm->init == 0
+                         ? bm->pSSTable
+                         : taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
+
   TdDirPtr      pDir = taosOpenDir(bm->buf);
   TdDirEntryPtr de = NULL;
-
+  int8_t        dummy = 0;
   while ((de = taosReadDir(pDir)) != NULL) {
     char* name = taosGetDirEntryName(de);
     if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue;
+    if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) {
+      taosMemoryFreeClear(bm->pCurrent);
+      bm->pCurrent = taosStrdup(name);
+      taosHashPut(pTable, name, strlen(name), &dummy, sizeof(dummy));
+      continue;
+    }
 
+    if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) {
+      taosMemoryFreeClear(bm->pManifest);
+      bm->pManifest = taosStrdup(name);
+      taosHashPut(pTable, name, strlen(name), &dummy, sizeof(dummy));
+      continue;
+    }
+    if (strlen(name) >= sstLen && strncmp(name + strlen(name) - 4, pSST, sstLen) == 0) {
+      char* p = taosStrdup(name);
 
-    // sprintf(absSrcPath, "%s%s%s", src, TD_DIRSEP, name);
-    // sprintf(absDstPath, "%s%s%s", dst, TD_DIRSEP, name);
-    // if (!taosDirEntryIsDir(de)) {
-    //    code = taosCopyFile(absSrcPath, absDstPath);
-    //    if (code == -1) {
-    //      goto _err;
-    //    }
-    //  }
-
-    // memset(absSrcPath, 0, sLen + 64);
-    // memset(absDstPath, 0, dLen + 64);
+      taosHashPut(pTable, name, strlen(name), &dummy, sizeof(dummy));
+      continue;
+    }
   }
+  if (bm->init == 0) {
+    bm->preCkptId = chkpId;
+    bm->curChkpId = chkpId;
+    bm->init = 1;
+  } else {
+    SArray* add = taosArrayInit(64, sizeof(void*));
+    SArray* del = taosArrayInit(64, sizeof(void*));
 
+    int32_t code = compareHashTable(bm->pSSTable, pTable, add, del);
+
+    bm->curChkpId = chkpId;
+    taosHashCleanup(pTable);
+  }
   return 0;
 }
 

From fb7dec00e1ed3039bc9e8a0c764f59c14be0f880 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 9 Aug 2023 01:50:49 +0000
Subject: [PATCH 23/36] support reopen stream state

---
 include/libs/transport/trpc.h                 | 2 +-
 source/client/src/clientEnv.c                 | 2 +-
 source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +-
 source/libs/transport/inc/transportInt.h      | 6 +++---
 source/libs/transport/src/trans.c             | 2 +-
 source/libs/transport/src/transCli.c          | 2 +-
 6 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h
index 93e4d72ad7..e5955aad54 100644
--- a/include/libs/transport/trpc.h
+++ b/include/libs/transport/trpc.h
@@ -89,7 +89,7 @@ typedef struct SRpcInit {
   int32_t retryMinInterval;  // retry init interval
   int32_t retryStepFactor;   // retry interval factor
   int32_t retryMaxInterval;  // retry max interval
-  int64_t retryMaxTimouet;
+  int64_t retryMaxTimeout;
 
   int32_t failFastThreshold;
   int32_t failFastInterval;
diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c
index 238b3613f5..40c27bf164 100644
--- a/source/client/src/clientEnv.c
+++ b/source/client/src/clientEnv.c
@@ -169,7 +169,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
   rpcInit.retryMinInterval = tsRedirectPeriod;
   rpcInit.retryStepFactor = tsRedirectFactor;
   rpcInit.retryMaxInterval = tsRedirectMaxPeriod;
-  rpcInit.retryMaxTimouet = tsMaxRetryWaitTime;
+  rpcInit.retryMaxTimeout = tsMaxRetryWaitTime;
 
   int32_t connLimitNum = tsNumOfRpcSessions / (tsNumOfRpcThreads * 3);
   connLimitNum = TMAX(connLimitNum, 10);
diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
index 5d6d16ccf8..c0194fe4a9 100644
--- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c
+++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c
@@ -298,7 +298,7 @@ int32_t dmInitClient(SDnode *pDnode) {
   rpcInit.retryMinInterval = tsRedirectPeriod;
   rpcInit.retryStepFactor = tsRedirectFactor;
   rpcInit.retryMaxInterval = tsRedirectMaxPeriod;
-  rpcInit.retryMaxTimouet = tsMaxRetryWaitTime;
+  rpcInit.retryMaxTimeout = tsMaxRetryWaitTime;
 
   rpcInit.failFastInterval = 5000;  // interval threshold(ms)
   rpcInit.failFastThreshold = 3;    // failed threshold
diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h
index ca48da690b..cc2c0d4e84 100644
--- a/source/libs/transport/inc/transportInt.h
+++ b/source/libs/transport/inc/transportInt.h
@@ -46,14 +46,14 @@ typedef struct {
   int8_t   connType;
   char     label[TSDB_LABEL_LEN];
   char     user[TSDB_UNI_LEN];  // meter ID
-  int32_t compatibilityVer;
+  int32_t  compatibilityVer;
   int32_t  compressSize;  // -1: no compress, 0 : all data compressed, size: compress data if larger than size
   int8_t   encryption;    // encrypt or not
-  
+
   int32_t retryMinInterval;  // retry init interval
   int32_t retryStepFactor;   // retry interval factor
   int32_t retryMaxInterval;  // retry max interval
-  int32_t retryMaxTimouet;
+  int32_t retryMaxTimeout;
 
   int32_t failFastThreshold;
   int32_t failFastInterval;
diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c
index 08b0451982..ed94521df0 100644
--- a/source/libs/transport/src/trans.c
+++ b/source/libs/transport/src/trans.c
@@ -55,7 +55,7 @@ void* rpcOpen(const SRpcInit* pInit) {
   pRpc->retryMinInterval = pInit->retryMinInterval;  // retry init interval
   pRpc->retryStepFactor = pInit->retryStepFactor;
   pRpc->retryMaxInterval = pInit->retryMaxInterval;
-  pRpc->retryMaxTimouet = pInit->retryMaxTimouet;
+  pRpc->retryMaxTimeout = pInit->retryMaxTimeout;
 
   pRpc->failFastThreshold = pInit->failFastThreshold;
   pRpc->failFastInterval = pInit->failFastInterval;
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index 8062a0618b..e7afda59ab 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -2256,7 +2256,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
     pCtx->retryMinInterval = pTransInst->retryMinInterval;
     pCtx->retryMaxInterval = pTransInst->retryMaxInterval;
     pCtx->retryStepFactor = pTransInst->retryStepFactor;
-    pCtx->retryMaxTimeout = pTransInst->retryMaxTimouet;
+    pCtx->retryMaxTimeout = pTransInst->retryMaxTimeout;
     pCtx->retryInitTimestamp = taosGetTimestampMs();
     pCtx->retryNextInterval = pCtx->retryMinInterval;
     pCtx->retryStep = 0;

From c43a6b272c59ee3100e6b593f82998988a57a2da Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 9 Aug 2023 02:09:50 +0000
Subject: [PATCH 24/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index d72901a2a3..8492410a46 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -221,6 +221,19 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
     bm->preCkptId = chkpId;
     bm->curChkpId = chkpId;
     bm->init = 1;
+
+    SArray* add = taosArrayInit(64, sizeof(void*));
+
+    void* pIter = taosHashIterate(pTable, NULL);
+    while (pIter) {
+      size_t len;
+      char*  name = taosHashGetKey(pIter, &len);
+      if (name != NULL && len != 0) {
+        taosArrayPush(add, &name);
+      }
+      pIter = taosHashIterate(pTable, pIter);
+    }
+
   } else {
     SArray* add = taosArrayInit(64, sizeof(void*));
     SArray* del = taosArrayInit(64, sizeof(void*));
@@ -233,6 +246,20 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
   return 0;
 }
 
+int32_t backendManagerDumpTo(SBackendManager* bm, char* name) {
+  int32_t code = 0;
+  char*   buf = taosMemoryCalloc(1, strlen(bm->path) + 64);
+  sprintf(buf, "%s%s%s", bm->path, TD_DIRSEP, name);
+
+  code = taosMkDir(buf);
+  if (code != 0) {
+    return code;
+  }
+
+  
+
+}
+
 SCfInit ginitDict[] = {
     {"default", 7, 0, defaultKeyComp, defaultKeyEncode, defaultKeyDecode, defaultKeyToString, compareDefaultName,
      destroyFunc, encodeValueFunc, decodeValueFunc},

From bbcfa9ab038f5e78db2b783ab8e729f6839fbe16 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 9 Aug 2023 06:10:40 +0000
Subject: [PATCH 25/36] support reopen stream state

---
 include/util/tarray.h                         |  7 +-
 source/libs/stream/src/streamBackendRocksdb.c | 67 +++++++++++++++++--
 source/util/src/tarray.c                      | 23 ++++++-
 3 files changed, 85 insertions(+), 12 deletions(-)

diff --git a/include/util/tarray.h b/include/util/tarray.h
index a93c695370..1b6b4587b7 100644
--- a/include/util/tarray.h
+++ b/include/util/tarray.h
@@ -200,13 +200,16 @@ void taosArrayClear(SArray* pArray);
  * @param pArray
  * @param fp
  */
+
 void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
 
+void taosArrayClearP(SArray* pArray, void (*fp)(void*));
+
 void* taosArrayDestroy(SArray* pArray);
 
-void  taosArrayDestroyP(SArray* pArray, FDelete fp);
+void taosArrayDestroyP(SArray* pArray, FDelete fp);
 
-void  taosArrayDestroyEx(SArray* pArray, FDelete fp);
+void taosArrayDestroyEx(SArray* pArray, FDelete fp);
 
 void taosArraySwap(SArray* a, SArray* b);
 
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 8492410a46..0804ad4ede 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -31,6 +31,9 @@ typedef struct {
   char*     path;
   char*     buf;
   int32_t   len;
+
+  SArray* pAdd;
+  SArray* pDel;
 } SBackendManager;
 
 typedef struct SCompactFilteFactory {
@@ -149,8 +152,25 @@ SBackendManager* backendManagerCreate(char* path) {
   p->pSSTable = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
   p->len = strlen(path) + 128;
   p->buf = taosMemoryCalloc(1, p->len);
+
+  p->pAdd = taosArrayInit(64, sizeof(void*));
+  p->pDel = taosArrayInit(64, sizeof(void*));
   return p;
 }
+void backendManagerDestroy(SBackendManager* bm) {
+  if (bm == NULL) return;
+
+  taosMemoryFree(bm->buf);
+  taosMemoryFree(bm->path);
+
+  taosHashCleanup(bm->pSSTable);
+
+  taosArrayDestroyP(bm->pSST, taosMemoryFree);
+  taosArrayDestroyP(bm->pAdd, taosMemoryFree);
+  taosArrayDestroyP(bm->pDel, taosMemoryFree);
+
+  taosMemoryFree(bm);
+}
 
 int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) {
   int32_t code = 0;
@@ -191,6 +211,9 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
                          ? bm->pSSTable
                          : taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
 
+  taosArrayClearP(bm->pAdd, taosMemoryFree);
+  taosArrayClearP(bm->pDel, taosMemoryFree);
+
   TdDirPtr      pDir = taosOpenDir(bm->buf);
   TdDirEntryPtr de = NULL;
   int8_t        dummy = 0;
@@ -222,23 +245,23 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
     bm->curChkpId = chkpId;
     bm->init = 1;
 
-    SArray* add = taosArrayInit(64, sizeof(void*));
+    // SArray* add = taosArrayInit(64, sizeof(void*));
 
     void* pIter = taosHashIterate(pTable, NULL);
     while (pIter) {
       size_t len;
       char*  name = taosHashGetKey(pIter, &len);
       if (name != NULL && len != 0) {
-        taosArrayPush(add, &name);
+        taosArrayPush(bm->pAdd, &name);
       }
       pIter = taosHashIterate(pTable, pIter);
     }
 
   } else {
-    SArray* add = taosArrayInit(64, sizeof(void*));
-    SArray* del = taosArrayInit(64, sizeof(void*));
+    // SArray* add = taosArrayInit(64, sizeof(void*));
+    // SArray* del = taosArrayInit(64, sizeof(void*));
 
-    int32_t code = compareHashTable(bm->pSSTable, pTable, add, del);
+    int32_t code = compareHashTable(bm->pSSTable, pTable, bm->pAdd, bm->pDel);
 
     bm->curChkpId = chkpId;
     taosHashCleanup(pTable);
@@ -248,7 +271,8 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
 
 int32_t backendManagerDumpTo(SBackendManager* bm, char* name) {
   int32_t code = 0;
-  char*   buf = taosMemoryCalloc(1, strlen(bm->path) + 64);
+  int32_t len = bm->len + 64;
+  char*   buf = taosMemoryCalloc(1, len);
   sprintf(buf, "%s%s%s", bm->path, TD_DIRSEP, name);
 
   code = taosMkDir(buf);
@@ -256,8 +280,37 @@ int32_t backendManagerDumpTo(SBackendManager* bm, char* name) {
     return code;
   }
 
-  
+  // clear current file
+  memset(buf, 0, len);
+  sprintf(buf, "%s%s%s%s%s", bm->path, TD_DIRSEP, name, TD_DIRSEP, bm->pCurrent);
+  taosRemoveFile(buf);
 
+  memset(buf, 0, len);
+  sprintf(buf, "%s%s%s%s%s", bm->path, TD_DIRSEP, name, TD_DIRSEP, bm->pManifest);
+  taosRemoveFile(buf);
+
+  for (int i = 0; i < taosArrayGetSize(bm->pAdd); i++) {
+    memset(buf, 0, len);
+
+    char* filename = taosArrayGetP(bm->pAdd, i);
+    sprintf(buf, "%s%s%s%s%s", bm->path, TD_DIRSEP, name, TD_DIRSEP, filename);
+
+    char* src = taosMemoryCalloc(1, len);
+    sprintf(src, "%s%s%s%" PRId64 "%s%s", bm->path, TD_DIRSEP, "checkpoint", bm->curChkpId, TD_DIRSEP, filename);
+    taosCopyFile(src, buf);
+  }
+
+  for (int i = 0; i < taosArrayGetSize(bm->pDel); i++) {
+    memset(buf, 0, len);
+
+    char* filename = taosArrayGetP(bm->pDel, i);
+    sprintf(buf, "%s%s%s%s%s", bm->path, TD_DIRSEP, name, TD_DIRSEP, filename);
+    taosRemoveFile(buf);
+  }
+  // clear delta data
+  taosArrayClearP(bm->pAdd, taosMemoryFree);
+  taosArrayClearP(bm->pDel, taosMemoryFree);
+  return code;
 }
 
 SCfInit ginitDict[] = {
diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index 8906391a9a..06beba0655 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -191,7 +191,7 @@ void* taosArrayGet(const SArray* pArray, size_t index) {
   }
 
   if (index >= pArray->size) {
-    uError("index is out of range, current:%"PRIzu" max:%d", index, pArray->capacity);
+    uError("index is out of range, current:%" PRIzu " max:%d", index, pArray->capacity);
     return NULL;
   }
 
@@ -319,7 +319,7 @@ SArray* taosArrayDup(const SArray* pSrc, __array_item_dup_fn_t fn) {
   if (NULL == pSrc) {
     return NULL;
   }
-  
+
   if (pSrc->size == 0) {  // empty array list
     return taosArrayInit(8, pSrc->elemSize);
   }
@@ -360,6 +360,23 @@ void taosArrayClearEx(SArray* pArray, void (*fp)(void*)) {
 
   pArray->size = 0;
 }
+void taosArrayClearP(SArray* pArray, void (*fp)(void*)) {
+  // if (pArray == NULL) return;
+  // if (fp == NULL) {
+  //   pArray->size = 0;
+  //   return;
+  // }
+
+  // for (int32_t i = 0; i < pArray->size; ++i) {
+  //   fp(TARRAY_GET_ELEM(pArray, i));
+  // }
+  if (pArray) {
+    for (int32_t i = 0; i < pArray->size; i++) {
+      fp(*(void**)TARRAY_GET_ELEM(pArray, i));
+    }
+  }
+  taosArrayClear(pArray);
+}
 
 void* taosArrayDestroy(SArray* pArray) {
   if (pArray) {
@@ -492,7 +509,7 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t
 // order array<type *>
 void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) {
   taosqsort(pArray->pData, pArray->size, pArray->elemSize, param, fn);
-//  taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param);
+  //  taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param);
 }
 
 void taosArraySwap(SArray* a, SArray* b) {

From 92b247aae842d2c92fab6f963a813d16d1a0b1b6 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 9 Aug 2023 07:36:00 +0000
Subject: [PATCH 26/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 0804ad4ede..024d53adae 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -245,8 +245,6 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
     bm->curChkpId = chkpId;
     bm->init = 1;
 
-    // SArray* add = taosArrayInit(64, sizeof(void*));
-
     void* pIter = taosHashIterate(pTable, NULL);
     while (pIter) {
       size_t len;
@@ -258,9 +256,6 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
     }
 
   } else {
-    // SArray* add = taosArrayInit(64, sizeof(void*));
-    // SArray* del = taosArrayInit(64, sizeof(void*));
-
     int32_t code = compareHashTable(bm->pSSTable, pTable, bm->pAdd, bm->pDel);
 
     bm->curChkpId = chkpId;

From a619b8f5be035feb0cfb286caa7cff4bff297d58 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 9 Aug 2023 08:58:49 +0000
Subject: [PATCH 27/36] rm duplicate para

---
 source/libs/stream/src/streamBackendRocksdb.c | 68 +++++++++++++------
 1 file changed, 47 insertions(+), 21 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 024d53adae..529b29153d 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -264,47 +264,73 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
   return 0;
 }
 
-int32_t backendManagerDumpTo(SBackendManager* bm, char* name) {
+int32_t backendManagerDumpTo(SBackendManager* bm, char* dname) {
   int32_t code = 0;
-  int32_t len = bm->len + 64;
-  char*   buf = taosMemoryCalloc(1, len);
-  sprintf(buf, "%s%s%s", bm->path, TD_DIRSEP, name);
+  int32_t len = bm->len + 128;
 
-  code = taosMkDir(buf);
+  char* dstBuf = taosMemoryCalloc(1, len);
+  char* srcBuf = taosMemoryCalloc(1, len);
+
+  char* srcDir = taosMemoryCalloc(1, len);
+  char* dstDir = taosMemoryCalloc(1, len);
+
+  sprintf(srcDir, "%s%s%s%" PRId64 "", bm->path, TD_DIRSEP, "checkpoint", bm->curChkpId);
+  sprintf(dstDir, "%s%s%s", bm->path, TD_DIRSEP, dname);
+
+  code = taosMkDir(dstDir);
   if (code != 0) {
     return code;
   }
 
   // clear current file
-  memset(buf, 0, len);
-  sprintf(buf, "%s%s%s%s%s", bm->path, TD_DIRSEP, name, TD_DIRSEP, bm->pCurrent);
-  taosRemoveFile(buf);
+  memset(dstBuf, 0, len);
+  sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pCurrent);
+  taosRemoveFile(dstBuf);
 
-  memset(buf, 0, len);
-  sprintf(buf, "%s%s%s%s%s", bm->path, TD_DIRSEP, name, TD_DIRSEP, bm->pManifest);
-  taosRemoveFile(buf);
+  memset(dstBuf, 0, len);
+  sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pManifest);
+  taosRemoveFile(dstBuf);
 
+  // add file to $name dir
   for (int i = 0; i < taosArrayGetSize(bm->pAdd); i++) {
-    memset(buf, 0, len);
+    memset(dstBuf, 0, len);
 
     char* filename = taosArrayGetP(bm->pAdd, i);
-    sprintf(buf, "%s%s%s%s%s", bm->path, TD_DIRSEP, name, TD_DIRSEP, filename);
+    sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, filename);
+    sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename);
 
-    char* src = taosMemoryCalloc(1, len);
-    sprintf(src, "%s%s%s%" PRId64 "%s%s", bm->path, TD_DIRSEP, "checkpoint", bm->curChkpId, TD_DIRSEP, filename);
-    taosCopyFile(src, buf);
+    taosCopyFile(srcBuf, dstBuf);
   }
-
+  // del file in $name
   for (int i = 0; i < taosArrayGetSize(bm->pDel); i++) {
-    memset(buf, 0, len);
-
+    memset(dstBuf, 0, len);
     char* filename = taosArrayGetP(bm->pDel, i);
-    sprintf(buf, "%s%s%s%s%s", bm->path, TD_DIRSEP, name, TD_DIRSEP, filename);
-    taosRemoveFile(buf);
+    sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename);
+    taosRemoveFile(dstBuf);
   }
+
+  // copy current file to dst dir
+  memset(srcBuf, 0, len);
+  memset(dstBuf, 0, len);
+  sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, bm->pCurrent);
+  sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pCurrent);
+  taosCopyFile(srcBuf, dstBuf);
+
+  // copy manifest file to dst dir
+  memset(srcBuf, 0, len);
+  memset(dstBuf, 0, len);
+  sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, bm->pManifest);
+  sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pManifest);
+  taosCopyFile(srcBuf, dstBuf);
+
   // clear delta data
   taosArrayClearP(bm->pAdd, taosMemoryFree);
   taosArrayClearP(bm->pDel, taosMemoryFree);
+
+  taosMemoryFree(srcBuf);
+  taosMemoryFree(dstBuf);
+  taosMemoryFree(srcDir);
+  taosMemoryFree(dstDir);
   return code;
 }
 

From a336a7b1a4d3e9da279c0533f96bbd1532b0a437 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 9 Aug 2023 09:06:22 +0000
Subject: [PATCH 28/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 529b29153d..a724e90866 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -277,6 +277,10 @@ int32_t backendManagerDumpTo(SBackendManager* bm, char* dname) {
   sprintf(srcDir, "%s%s%s%" PRId64 "", bm->path, TD_DIRSEP, "checkpoint", bm->curChkpId);
   sprintf(dstDir, "%s%s%s", bm->path, TD_DIRSEP, dname);
 
+  if (!taosDirExist(srcDir)) {
+    return 0;
+  }
+
   code = taosMkDir(dstDir);
   if (code != 0) {
     return code;

From f3cf907d5af99ae8ffac5aacc63042c359bb1698 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 9 Aug 2023 09:07:13 +0000
Subject: [PATCH 29/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index a724e90866..8f8f268d9a 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -308,6 +308,7 @@ int32_t backendManagerDumpTo(SBackendManager* bm, char* dname) {
   // del file in $name
   for (int i = 0; i < taosArrayGetSize(bm->pDel); i++) {
     memset(dstBuf, 0, len);
+
     char* filename = taosArrayGetP(bm->pDel, i);
     sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename);
     taosRemoveFile(dstBuf);

From 6fdcd82a330e3c48f51563b0b670eb1f72dced32 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 9 Aug 2023 09:19:03 +0000
Subject: [PATCH 30/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 8f8f268d9a..d336c4235a 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -34,6 +34,8 @@ typedef struct {
 
   SArray* pAdd;
   SArray* pDel;
+
+  int8_t update;
 } SBackendManager;
 
 typedef struct SCompactFilteFactory {
@@ -155,6 +157,7 @@ SBackendManager* backendManagerCreate(char* path) {
 
   p->pAdd = taosArrayInit(64, sizeof(void*));
   p->pDel = taosArrayInit(64, sizeof(void*));
+  p->update = 0;
   return p;
 }
 void backendManagerDestroy(SBackendManager* bm) {
@@ -254,12 +257,16 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
       }
       pIter = taosHashIterate(pTable, pIter);
     }
+    bm->update = 1;
 
   } else {
     int32_t code = compareHashTable(bm->pSSTable, pTable, bm->pAdd, bm->pDel);
 
     bm->curChkpId = chkpId;
     taosHashCleanup(pTable);
+    if (taosArrayGetSize(bm->pAdd) == 0 && taosArrayGetSize(bm->pDel) == 0) {
+      bm->update = 0;
+    }
   }
   return 0;
 }

From 1db15da4e18d10c8390b8a81239eef0e61f6c86a Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 9 Aug 2023 09:50:49 +0000
Subject: [PATCH 31/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 39 +++++++++++--------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index d336c4235a..f6439d7499 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -32,10 +32,11 @@ typedef struct {
   char*     buf;
   int32_t   len;
 
-  SArray* pAdd;
-  SArray* pDel;
-
-  int8_t update;
+  SHashObj* pSstTbl[2];
+  SArray*   pAdd;
+  SArray*   pDel;
+  int8_t    idx;
+  int8_t    update;
 } SBackendManager;
 
 typedef struct SCompactFilteFactory {
@@ -155,6 +156,10 @@ SBackendManager* backendManagerCreate(char* path) {
   p->len = strlen(path) + 128;
   p->buf = taosMemoryCalloc(1, p->len);
 
+  p->idx = 0;
+  p->pSstTbl[0] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
+  p->pSstTbl[1] = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
+
   p->pAdd = taosArrayInit(64, sizeof(void*));
   p->pDel = taosArrayInit(64, sizeof(void*));
   p->update = 0;
@@ -172,6 +177,8 @@ void backendManagerDestroy(SBackendManager* bm) {
   taosArrayDestroyP(bm->pAdd, taosMemoryFree);
   taosArrayDestroyP(bm->pDel, taosMemoryFree);
 
+  taosHashCleanup(bm->pSstTbl[0]);
+  taosHashCleanup(bm->pSstTbl[1]);
   taosMemoryFree(bm);
 }
 
@@ -210,10 +217,6 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
   memset(bm->buf, 0, bm->len);
   sprintf(bm->buf, "%s%scheckpoint%" PRId64 "", bm->path, TD_DIRSEP, chkpId);
 
-  SHashObj* pTable = bm->init == 0
-                         ? bm->pSSTable
-                         : taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
-
   taosArrayClearP(bm->pAdd, taosMemoryFree);
   taosArrayClearP(bm->pDel, taosMemoryFree);
 
@@ -226,48 +229,50 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
     if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) {
       taosMemoryFreeClear(bm->pCurrent);
       bm->pCurrent = taosStrdup(name);
-      taosHashPut(pTable, name, strlen(name), &dummy, sizeof(dummy));
+      taosHashPut(bm->pSstTbl[1 - bm->idx], name, strlen(name), &dummy, sizeof(dummy));
       continue;
     }
 
     if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) {
       taosMemoryFreeClear(bm->pManifest);
       bm->pManifest = taosStrdup(name);
-      taosHashPut(pTable, name, strlen(name), &dummy, sizeof(dummy));
+      taosHashPut(bm->pSstTbl[1 - bm->idx], name, strlen(name), &dummy, sizeof(dummy));
       continue;
     }
     if (strlen(name) >= sstLen && strncmp(name + strlen(name) - 4, pSST, sstLen) == 0) {
       char* p = taosStrdup(name);
-
-      taosHashPut(pTable, name, strlen(name), &dummy, sizeof(dummy));
+      taosHashPut(bm->pSstTbl[1 - bm->idx], name, strlen(name), &dummy, sizeof(dummy));
       continue;
     }
   }
   if (bm->init == 0) {
-    bm->preCkptId = chkpId;
+    bm->preCkptId = -1;
     bm->curChkpId = chkpId;
     bm->init = 1;
 
-    void* pIter = taosHashIterate(pTable, NULL);
+    void* pIter = taosHashIterate(bm->pSstTbl[1 - bm->idx], NULL);
     while (pIter) {
       size_t len;
       char*  name = taosHashGetKey(pIter, &len);
       if (name != NULL && len != 0) {
         taosArrayPush(bm->pAdd, &name);
       }
-      pIter = taosHashIterate(pTable, pIter);
+      pIter = taosHashIterate(bm->pSstTbl[1 - bm->idx], pIter);
     }
     bm->update = 1;
 
   } else {
-    int32_t code = compareHashTable(bm->pSSTable, pTable, bm->pAdd, bm->pDel);
+    int32_t code = compareHashTable(bm->pSstTbl[bm->idx], bm->pSstTbl[1 - bm->idx], bm->pAdd, bm->pDel);
 
+    bm->preCkptId = bm->curChkpId;
     bm->curChkpId = chkpId;
-    taosHashCleanup(pTable);
     if (taosArrayGetSize(bm->pAdd) == 0 && taosArrayGetSize(bm->pDel) == 0) {
       bm->update = 0;
     }
   }
+  taosHashClear(bm->pSstTbl[bm->idx]);
+  bm->idx = 1 - bm->idx;
+
   return 0;
 }
 

From 19ac9054d060e11cb210feb5f67c78ebfb1a9884 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 9 Aug 2023 12:47:49 +0000
Subject: [PATCH 32/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 35 ++++++++-----------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index f6439d7499..f34b7dc3cb 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -21,16 +21,15 @@
 #include "tref.h"
 
 typedef struct {
-  int8_t    init;
-  char*     pCurrent;
-  char*     pManifest;
-  SArray*   pSST;
-  int64_t   preCkptId;
-  int64_t   curChkpId;
-  SHashObj* pSSTable;
-  char*     path;
-  char*     buf;
-  int32_t   len;
+  int8_t  init;
+  char*   pCurrent;
+  char*   pManifest;
+  SArray* pSST;
+  int64_t preCkptId;
+  int64_t curChkpId;
+  char*   path;
+  char*   buf;
+  int32_t len;
 
   SHashObj* pSstTbl[2];
   SArray*   pAdd;
@@ -146,13 +145,12 @@ void    destroyFunc(void* arg);
 int32_t encodeValueFunc(void* value, int32_t vlen, int64_t ttl, char** dest);
 int32_t decodeValueFunc(void* value, int32_t vlen, int64_t* ttl, char** dest);
 
-SBackendManager* backendManagerCreate(char* path) {
+SBackendManager* bkdMgtCreate(char* path) {
   SBackendManager* p = taosMemoryCalloc(1, sizeof(SBackendManager));
   p->curChkpId = 0;
   p->preCkptId = 0;
   p->pSST = taosArrayInit(64, sizeof(void*));
   p->path = taosStrdup(path);
-  p->pSSTable = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
   p->len = strlen(path) + 128;
   p->buf = taosMemoryCalloc(1, p->len);
 
@@ -165,14 +163,12 @@ SBackendManager* backendManagerCreate(char* path) {
   p->update = 0;
   return p;
 }
-void backendManagerDestroy(SBackendManager* bm) {
+void bkdMgtDestroy(SBackendManager* bm) {
   if (bm == NULL) return;
 
   taosMemoryFree(bm->buf);
   taosMemoryFree(bm->path);
 
-  taosHashCleanup(bm->pSSTable);
-
   taosArrayDestroyP(bm->pSST, taosMemoryFree);
   taosArrayDestroyP(bm->pAdd, taosMemoryFree);
   taosArrayDestroyP(bm->pDel, taosMemoryFree);
@@ -204,7 +200,7 @@ int32_t compareHashTable(SHashObj* p1, SHashObj* p2, SArray* add, SArray* del) {
 
   return code;
 }
-int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list) {
+int32_t bkdMgtGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list) {
   const char* pCurrent = "CURRENT";
   int32_t     currLen = strlen(pCurrent);
 
@@ -259,8 +255,7 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
       }
       pIter = taosHashIterate(bm->pSstTbl[1 - bm->idx], pIter);
     }
-    bm->update = 1;
-
+    if (taosArrayGetSize(bm->pAdd) > 0) bm->update = 1;
   } else {
     int32_t code = compareHashTable(bm->pSstTbl[bm->idx], bm->pSstTbl[1 - bm->idx], bm->pAdd, bm->pDel);
 
@@ -276,7 +271,7 @@ int32_t backendManagerGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list
   return 0;
 }
 
-int32_t backendManagerDumpTo(SBackendManager* bm, char* dname) {
+int32_t bkdMgtDumpTo(SBackendManager* bm, char* dname) {
   int32_t code = 0;
   int32_t len = bm->len + 128;
 
@@ -340,7 +335,7 @@ int32_t backendManagerDumpTo(SBackendManager* bm, char* dname) {
   sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, bm->pManifest);
   taosCopyFile(srcBuf, dstBuf);
 
-  // clear delta data
+  // clear delta data buf
   taosArrayClearP(bm->pAdd, taosMemoryFree);
   taosArrayClearP(bm->pDel, taosMemoryFree);
 

From a20b299f9b2ca13d4096145adfb3653c8fd45b18 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Wed, 9 Aug 2023 13:06:10 +0000
Subject: [PATCH 33/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index f34b7dc3cb..85abe203d4 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -305,6 +305,7 @@ int32_t bkdMgtDumpTo(SBackendManager* bm, char* dname) {
   // add file to $name dir
   for (int i = 0; i < taosArrayGetSize(bm->pAdd); i++) {
     memset(dstBuf, 0, len);
+    memset(srcBuf, 0, len);
 
     char* filename = taosArrayGetP(bm->pAdd, i);
     sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, filename);
@@ -315,6 +316,7 @@ int32_t bkdMgtDumpTo(SBackendManager* bm, char* dname) {
   // del file in $name
   for (int i = 0; i < taosArrayGetSize(bm->pDel); i++) {
     memset(dstBuf, 0, len);
+    memset(srcBuf, 0, len);
 
     char* filename = taosArrayGetP(bm->pDel, i);
     sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename);

From ef247cdb1d1d0559b6d328482b4256c79f3e08db Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 10 Aug 2023 01:49:26 +0000
Subject: [PATCH 34/36] support reopen stream state

---
 include/libs/stream/tstream.h                 | 10 +--
 source/dnode/vnode/src/tq/tqStreamStateSnap.c |  4 +-
 source/libs/stream/src/streamBackendRocksdb.c | 84 +++++++++----------
 source/libs/stream/src/streamMeta.c           | 18 ++--
 4 files changed, 58 insertions(+), 58 deletions(-)

diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h
index 4c0005ece7..16c47024ef 100644
--- a/include/libs/stream/tstream.h
+++ b/include/libs/stream/tstream.h
@@ -369,11 +369,11 @@ typedef struct SStreamMeta {
 
   int32_t chkptNotReadyTasks;
 
-  int64_t  checkpointId;
-  SArray*  checkpointSaved;
-  SArray*  checkpointInUse;
-  int32_t  checkpointCap;
-  SRWLatch checkpointDirLock;
+  int64_t  chkpId;
+  SArray*  chkpSaved;
+  SArray*  chkpInUse;
+  int32_t  chkpCap;
+  SRWLatch chkpDirLock;
 } SStreamMeta;
 
 int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
index 87d174715d..5f77ea50e6 100644
--- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c
+++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
@@ -47,11 +47,11 @@ int32_t streamStateSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS
   pReader->sver = sver;
   pReader->ever = ever;
 
-  int64_t checkpointId = meta ? meta->checkpointId : 0;
+  int64_t chkpId = meta ? meta->chkpId : 0;
 
   SStreamSnapReader* pSnapReader = NULL;
 
-  if (streamSnapReaderOpen(pTq, sver, checkpointId, pTq->path, &pSnapReader) == 0) {
+  if (streamSnapReaderOpen(pTq, sver, chkpId, pTq->path, &pSnapReader) == 0) {
     pReader->complete = 1;
   } else {
     code = -1;
diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 85abe203d4..14c94a7996 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -372,8 +372,8 @@ int32_t copyFiles(const char* src, const char* dst) {
   // opt later, just hard link
   int32_t sLen = strlen(src);
   int32_t dLen = strlen(dst);
-  char*   absSrcPath = taosMemoryCalloc(1, sLen + 64);
-  char*   absDstPath = taosMemoryCalloc(1, dLen + 64);
+  char*   srcName = taosMemoryCalloc(1, sLen + 64);
+  char*   dstName = taosMemoryCalloc(1, dLen + 64);
 
   TdDirPtr pDir = taosOpenDir(src);
   if (pDir == NULL) return 0;
@@ -383,22 +383,22 @@ int32_t copyFiles(const char* src, const char* dst) {
     char* name = taosGetDirEntryName(de);
     if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue;
 
-    sprintf(absSrcPath, "%s%s%s", src, TD_DIRSEP, name);
-    sprintf(absDstPath, "%s%s%s", dst, TD_DIRSEP, name);
+    sprintf(srcName, "%s%s%s", src, TD_DIRSEP, name);
+    sprintf(dstName, "%s%s%s", dst, TD_DIRSEP, name);
     if (!taosDirEntryIsDir(de)) {
-      code = taosCopyFile(absSrcPath, absDstPath);
+      code = taosCopyFile(srcName, dstName);
       if (code == -1) {
         goto _err;
       }
     }
 
-    memset(absSrcPath, 0, sLen + 64);
-    memset(absDstPath, 0, dLen + 64);
+    memset(srcName, 0, sLen + 64);
+    memset(dstName, 0, dLen + 64);
   }
 
 _err:
-  taosMemoryFreeClear(absSrcPath);
-  taosMemoryFreeClear(absDstPath);
+  taosMemoryFreeClear(srcName);
+  taosMemoryFreeClear(dstName);
   taosCloseDir(&pDir);
   return code >= 0 ? 0 : -1;
 }
@@ -626,75 +626,75 @@ void streamBackendHandleCleanup(void* arg) {
 
 int32_t getLatestCheckpoint(void* arg, int64_t* checkpoint) {
   SStreamMeta* pMeta = arg;
-  taosWLockLatch(&pMeta->checkpointDirLock);
+  taosWLockLatch(&pMeta->chkpDirLock);
   int64_t tc = 0;
-  int32_t sz = taosArrayGetSize(pMeta->checkpointSaved);
+  int32_t sz = taosArrayGetSize(pMeta->chkpSaved);
   if (sz <= 0) {
-    taosWUnLockLatch(&pMeta->checkpointDirLock);
+    taosWUnLockLatch(&pMeta->chkpDirLock);
     return -1;
   } else {
-    tc = *(int64_t*)taosArrayGetLast(pMeta->checkpointSaved);
+    tc = *(int64_t*)taosArrayGetLast(pMeta->chkpSaved);
   }
 
-  taosArrayPush(pMeta->checkpointInUse, &tc);
+  taosArrayPush(pMeta->chkpInUse, &tc);
 
   *checkpoint = tc;
-  taosWUnLockLatch(&pMeta->checkpointDirLock);
+  taosWUnLockLatch(&pMeta->chkpDirLock);
   return 0;
 }
 /*
  *  checkpointSave |--cp1--|--cp2--|--cp3--|--cp4--|--cp5--|
- *  checkpointInUse: |--cp2--|--cp4--|
- *  checkpointInUse is doing translation, cannot del until
+ *  chkpInUse: |--cp2--|--cp4--|
+ *  chkpInUse is doing translation, cannot del until
  *  replication is finished
  */
 int32_t delObsoleteCheckpoint(void* arg, const char* path) {
   SStreamMeta* pMeta = arg;
 
-  taosWLockLatch(&pMeta->checkpointDirLock);
+  taosWLockLatch(&pMeta->chkpDirLock);
 
-  SArray* checkpointDel = taosArrayInit(10, sizeof(int64_t));
-  SArray* checkpointDup = taosArrayInit(10, sizeof(int64_t));
+  SArray* chkpDel = taosArrayInit(10, sizeof(int64_t));
+  SArray* chkpDup = taosArrayInit(10, sizeof(int64_t));
 
   int64_t minId = 0;
-  if (taosArrayGetSize(pMeta->checkpointInUse) >= 1) {
-    minId = *(int64_t*)taosArrayGet(pMeta->checkpointInUse, 0);
+  if (taosArrayGetSize(pMeta->chkpInUse) >= 1) {
+    minId = *(int64_t*)taosArrayGet(pMeta->chkpInUse, 0);
 
-    for (int i = 0; i < taosArrayGetSize(pMeta->checkpointSaved); i++) {
-      int64_t id = *(int64_t*)taosArrayGet(pMeta->checkpointSaved, i);
+    for (int i = 0; i < taosArrayGetSize(pMeta->chkpSaved); i++) {
+      int64_t id = *(int64_t*)taosArrayGet(pMeta->chkpSaved, i);
       if (id >= minId) {
-        taosArrayPush(checkpointDup, &id);
+        taosArrayPush(chkpDup, &id);
       } else {
-        taosArrayPush(checkpointDel, &id);
+        taosArrayPush(chkpDel, &id);
       }
     }
   } else {
-    int32_t sz = taosArrayGetSize(pMeta->checkpointSaved);
-    int32_t dsz = sz - pMeta->checkpointCap;  // del size
+    int32_t sz = taosArrayGetSize(pMeta->chkpSaved);
+    int32_t dsz = sz - pMeta->chkpCap;  // del size
 
     for (int i = 0; i < dsz; i++) {
-      int64_t id = *(int64_t*)taosArrayGet(pMeta->checkpointSaved, i);
-      taosArrayPush(checkpointDel, &id);
+      int64_t id = *(int64_t*)taosArrayGet(pMeta->chkpSaved, i);
+      taosArrayPush(chkpDel, &id);
     }
     for (int i = dsz < 0 ? 0 : dsz; i < sz; i++) {
-      int64_t id = *(int64_t*)taosArrayGet(pMeta->checkpointSaved, i);
-      taosArrayPush(checkpointDup, &id);
+      int64_t id = *(int64_t*)taosArrayGet(pMeta->chkpSaved, i);
+      taosArrayPush(chkpDup, &id);
     }
   }
-  taosArrayDestroy(pMeta->checkpointSaved);
-  pMeta->checkpointSaved = checkpointDup;
+  taosArrayDestroy(pMeta->chkpSaved);
+  pMeta->chkpSaved = chkpDup;
 
-  taosWUnLockLatch(&pMeta->checkpointDirLock);
+  taosWUnLockLatch(&pMeta->chkpDirLock);
 
-  for (int i = 0; i < taosArrayGetSize(checkpointDel); i++) {
-    int64_t id = *(int64_t*)taosArrayGet(checkpointDel, i);
+  for (int i = 0; i < taosArrayGetSize(chkpDel); i++) {
+    int64_t id = *(int64_t*)taosArrayGet(chkpDel, i);
     char    tbuf[256] = {0};
     sprintf(tbuf, "%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, id);
     if (taosIsDir(tbuf)) {
       taosRemoveDir(tbuf);
     }
   }
-  taosArrayDestroy(checkpointDel);
+  taosArrayDestroy(chkpDel);
   return 0;
 }
 
@@ -742,7 +742,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) {
 
   for (int i = 0; i < taosArrayGetSize(suffix); i++) {
     int64_t id = *(int64_t*)taosArrayGet(suffix, i);
-    taosArrayPush(pMeta->checkpointSaved, &id);
+    taosArrayPush(pMeta->chkpSaved, &id);
   }
 
   taosArrayDestroy(suffix);
@@ -794,9 +794,9 @@ int32_t streamBackendDoCheckpoint(void* arg, uint64_t checkpointId) {
     }
     rocksdb_checkpoint_object_destroy(cp);
   }
-  taosWLockLatch(&pMeta->checkpointDirLock);
-  taosArrayPush(pMeta->checkpointSaved, &checkpointId);
-  taosWUnLockLatch(&pMeta->checkpointDirLock);
+  taosWLockLatch(&pMeta->chkpDirLock);
+  taosArrayPush(pMeta->chkpSaved, &checkpointId);
+  taosWUnLockLatch(&pMeta->chkpDirLock);
 
   delObsoleteCheckpoint(arg, path);
 
diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c
index 907ff2b48a..20a62963cc 100644
--- a/source/libs/stream/src/streamMeta.c
+++ b/source/libs/stream/src/streamMeta.c
@@ -91,12 +91,13 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
 
   pMeta->pTaskBackendUnique =
       taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
-  pMeta->checkpointSaved = taosArrayInit(4, sizeof(int64_t));
-  pMeta->checkpointInUse = taosArrayInit(4, sizeof(int64_t));
-  pMeta->checkpointCap = 8;
-  taosInitRWLatch(&pMeta->checkpointDirLock);
+  pMeta->chkpSaved = taosArrayInit(4, sizeof(int64_t));
+  pMeta->chkpInUse = taosArrayInit(4, sizeof(int64_t));
+  pMeta->chkpCap = 8;
+  taosInitRWLatch(&pMeta->chkpDirLock);
 
   int64_t chkpId = streamGetLatestCheckpointId(pMeta);
+  pMeta->chkpId = chkpId;
 
   pMeta->streamBackend = streamBackendInit(pMeta->path, chkpId);
   if (pMeta->streamBackend == NULL) {
@@ -109,7 +110,6 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
     terrno = TAOS_SYSTEM_ERROR(code);
     goto _err;
   }
-
   taosInitRWLatch(&pMeta->lock);
   taosThreadMutexInit(&pMeta->backendMutex, NULL);
 
@@ -182,9 +182,9 @@ int32_t streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId) {
 
   taosHashClear(pMeta->pTaskBackendUnique);
 
-  taosArrayClear(pMeta->checkpointSaved);
+  taosArrayClear(pMeta->chkpSaved);
 
-  taosArrayClear(pMeta->checkpointInUse);
+  taosArrayClear(pMeta->chkpInUse);
 
   return 0;
 }
@@ -222,8 +222,8 @@ void streamMetaClose(SStreamMeta* pMeta) {
   taosThreadMutexDestroy(&pMeta->backendMutex);
   taosHashCleanup(pMeta->pTaskBackendUnique);
 
-  taosArrayDestroy(pMeta->checkpointSaved);
-  taosArrayDestroy(pMeta->checkpointInUse);
+  taosArrayDestroy(pMeta->chkpSaved);
+  taosArrayDestroy(pMeta->chkpInUse);
 
   taosMemoryFree(pMeta);
 }

From 100d2240c35f5e68d47d4e21cc3c5e8075c28fda Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 10 Aug 2023 06:35:25 +0000
Subject: [PATCH 35/36] support reopen stream state

---
 source/libs/stream/src/streamBackendRocksdb.c | 25 ++++++++++++++++---
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c
index 14c94a7996..69ba44b612 100644
--- a/source/libs/stream/src/streamBackendRocksdb.c
+++ b/source/libs/stream/src/streamBackendRocksdb.c
@@ -28,14 +28,17 @@ typedef struct {
   int64_t preCkptId;
   int64_t curChkpId;
   char*   path;
+
   char*   buf;
   int32_t len;
 
+  // ping-pong buf
   SHashObj* pSstTbl[2];
-  SArray*   pAdd;
-  SArray*   pDel;
   int8_t    idx;
-  int8_t    update;
+
+  SArray* pAdd;
+  SArray* pDel;
+  int8_t  update;
 } SBackendManager;
 
 typedef struct SCompactFilteFactory {
@@ -258,6 +261,15 @@ int32_t bkdMgtGetDelta(SBackendManager* bm, int64_t chkpId, SArray* list) {
     if (taosArrayGetSize(bm->pAdd) > 0) bm->update = 1;
   } else {
     int32_t code = compareHashTable(bm->pSstTbl[bm->idx], bm->pSstTbl[1 - bm->idx], bm->pAdd, bm->pDel);
+    if (code != 0) {
+      // dead code
+      taosArrayClearP(bm->pAdd, taosMemoryFree);
+      taosArrayClearP(bm->pDel, taosMemoryFree);
+      taosHashClear(bm->pSstTbl[1 - bm->idx]);
+      bm->update = 0;
+
+      return code;
+    }
 
     bm->preCkptId = bm->curChkpId;
     bm->curChkpId = chkpId;
@@ -714,11 +726,16 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) {
 
   if (!taosDirExist(chkpPath)) {
     // no checkpoint, nothing to load
+    taosMemoryFree(chkpPath);
     return 0;
   }
 
   TdDirPtr pDir = taosOpenDir(chkpPath);
-  if (pDir == NULL) return 0;
+
+  if (pDir == NULL) {
+    taosMemoryFree(chkpPath);
+    return 0;
+  }
 
   TdDirEntryPtr de = NULL;
   SArray*       suffix = taosArrayInit(4, sizeof(int64_t));

From 246efd83c6ec5d3e75d6676e7c5209ef929fe166 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Mon, 14 Aug 2023 02:03:41 +0000
Subject: [PATCH 36/36] refactor chpt transfer

---
 source/dnode/vnode/src/inc/vnodeInt.h         | 3 +--
 source/dnode/vnode/src/tq/tqStreamStateSnap.c | 2 +-
 source/dnode/vnode/src/vnd/vnodeSnapshot.c    | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h
index 402464100f..3112e59985 100644
--- a/source/dnode/vnode/src/inc/vnodeInt.h
+++ b/source/dnode/vnode/src/inc/vnodeInt.h
@@ -331,11 +331,10 @@ int32_t streamStateSnapRead(SStreamStateReader* pReader, uint8_t** ppData);
 int32_t streamStateSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamStateWriter** ppWriter);
 int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback);
 int32_t streamStateSnapWrite(SStreamStateWriter* pWriter, uint8_t* pData, uint32_t nData);
-int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, char* path, int64_t chkpId);
+int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, int64_t chkpId);
 
 int32_t streamStateLoadTasks(SStreamStateWriter* pWriter);
 
-
 // SStreamTaskReader ======================================
 // SStreamStateWriter =====================================
 // SStreamStateReader =====================================
diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
index ba2c104409..2c3a12b50b 100644
--- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c
+++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c
@@ -162,7 +162,7 @@ int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback)
   taosMemoryFree(pWriter);
   return code;
 }
-int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, char* path, int64_t chkpId) {
+int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, int64_t chkpId) {
   int32_t code = streamMetaReopen(pWriter->pTq->pStreamMeta, chkpId);
   if (code == 0) {
     code = streamStateLoadTasks(pWriter);
diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
index f8872a2822..abd6ea00a7 100644
--- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c
+++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c
@@ -419,7 +419,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
     code = streamStateSnapWriterClose(pWriter->pStreamStateWriter, rollback);
     if (code) goto _exit;
 
-    code = streamStateRebuildFromSnap(pWriter->pStreamStateWriter, NULL, 0);
+    code = streamStateRebuildFromSnap(pWriter->pStreamStateWriter, 0);
     if (code) goto _exit;
   }