From 9e7f860d7c38ed7898891a9279403a03a8c1a1ff Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 12 Jun 2023 16:40:24 +0800
Subject: [PATCH 01/17] enh(tdb/recycle): first round implemenation of page
 recycling

---
 source/libs/tdb/src/db/tdbBtree.c           |  11 +
 source/libs/tdb/src/db/tdbDb.c              |   6 +
 source/libs/tdb/src/db/tdbPager.c           |  56 +-
 source/libs/tdb/src/inc/tdbInt.h            |   3 +
 source/libs/tdb/test/CMakeLists.txt         |   4 +
 source/libs/tdb/test/tdbPageRecycleTest.cpp | 674 ++++++++++++++++++++
 6 files changed, 753 insertions(+), 1 deletion(-)
 create mode 100644 source/libs/tdb/test/tdbPageRecycleTest.cpp

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index c49b5726b6..bb02db8bb8 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -860,6 +860,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
 
     if (!TDB_BTREE_PAGE_IS_LEAF(pNews[0])) {
       ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno;
+    } else {
+      // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pNews[0]));
+      tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pNews[0]), pTxn);
     }
   }
 
@@ -870,9 +873,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
   }
 
   for (pageIdx = 0; pageIdx < nOlds; ++pageIdx) {
+    // printf("tdb/balance: btree balance old pgno: %d.\n", TDB_PAGE_PGNO(pOlds[pageIdx]));
+    if (pageIdx >= nNews) {
+      // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pOlds[pageIdx]));
+      tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pOlds[pageIdx]), pTxn);
+    }
     tdbPagerReturnPage(pBt->pPager, pOlds[pageIdx], pTxn);
   }
   for (; pageIdx < nNews; ++pageIdx) {
+    // printf("tdb/balance: btree balance new pgno: %d.\n", TDB_PAGE_PGNO(pNews[pageIdx]));
     tdbPagerReturnPage(pBt->pPager, pNews[pageIdx], pTxn);
   }
 
@@ -2113,6 +2122,8 @@ int tdbBtcDelete(SBTC *pBtc) {
         return -1;
       }
 
+      // printf("tdb/btc-delete: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pBtc->pPage));
+
       ret = tdbBtreeBalance(pBtc);
       if (ret < 0) {
         tdbError("tdb/btc-delete: btree balance failed with ret: %d.", ret);
diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c
index 952c49db73..fe9d51dc82 100644
--- a/source/libs/tdb/src/db/tdbDb.c
+++ b/source/libs/tdb/src/db/tdbDb.c
@@ -70,6 +70,11 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, i
   if (ret < 0) {
     return -1;
   }
+
+  ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback);
+  if (ret < 0) {
+    return -1;
+  }
 #endif
 
   *ppDb = pDb;
@@ -82,6 +87,7 @@ int tdbClose(TDB *pDb) {
   if (pDb) {
 #ifdef USE_MAINDB
     if (pDb->pMainDb) tdbTbClose(pDb->pMainDb);
+    if (pDb->pFreeDb) tdbTbClose(pDb->pFreeDb);
 #endif
 
     for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) {
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 5ea9be63db..5f187d339e 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -316,6 +316,10 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
       return -1;
     }
 
+    if (!TDB_PAGE_TOTAL_CELLS(pPage) && TDB_PAGE_PGNO(pPage) > 1) {
+      tdbDebug("pager/commit: %p, %d/%d, txnId:%" PRId64, pPager, pPager->dbOrigSize, pPager->dbFileSize, pTxn->txnId);
+    }
+
     ret = tdbPagerPWritePageToDB(pPager, pPage);
     if (ret < 0) {
       tdbError("failed to write page to db since %s", tstrerror(terrno));
@@ -695,9 +699,59 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) {
   //        TDB_PAGE_PGNO(pPage), pPage);
 }
 
+int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) {
+  int code = 0;
+
+  code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn);
+  if (code < 0) {
+    return -1;
+  }
+
+  return code;
+}
+
+static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) {
+  int  code = 0;
+  TBC *pCur;
+
+  if (!pPager->pEnv->pFreeDb) {
+    return 0;
+  }
+
+  code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, NULL);
+  if (code < 0) {
+    return 0;
+  }
+
+  code = tdbTbcMoveToFirst(pCur);
+  if (code) {
+    tdbTbcClose(pCur);
+    return 0;
+  }
+
+  void *pKey = NULL;
+  int   nKey = 0;
+
+  code = tdbTbcGet(pCur, (const void **)&pKey, &nKey, NULL, NULL);
+  if (code < 0) {
+    tdbTbcClose(pCur);
+    return 0;
+  }
+
+  *pPgno = *(SPgno *)pKey;
+
+  code = tdbTbcDelete(pCur);
+  if (code < 0) {
+    tdbTbcClose(pCur);
+    return 0;
+  }
+  tdbTbcClose(pCur);
+  return 0;
+}
+
 static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) {
   // TODO: Allocate a page from the free list
-  return 0;
+  return tdbPagerRemoveFreePage(pPager, ppgno);
 }
 
 static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) {
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index 7a0bcc00a4..e65edb4afe 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -198,6 +198,7 @@ int  tdbPagerAbort(SPager *pPager, TXN *pTxn);
 int  tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg,
                        TXN *pTxn);
 void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
+int  tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn);
 int  tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
 int  tdbPagerRestoreJournals(SPager *pPager);
 int  tdbPagerRollback(SPager *pPager);
@@ -373,6 +374,7 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) {
 
 #ifdef USE_MAINDB
 #define TDB_MAINDB_NAME "main.tdb"
+#define TDB_FREEDB_NAME "_free.db"
 #endif
 
 struct STDB {
@@ -386,6 +388,7 @@ struct STDB {
   SPager **pgrHash;
 #ifdef USE_MAINDB
   TTB *pMainDb;
+  TTB *pFreeDb;
 #endif
   int64_t txnId;
 };
diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt
index fd4d7c101d..4715ccbd41 100644
--- a/source/libs/tdb/test/CMakeLists.txt
+++ b/source/libs/tdb/test/CMakeLists.txt
@@ -14,3 +14,7 @@ target_link_libraries(tdbExOVFLTest tdb gtest gtest_main)
 add_executable(tdbPageDefragmentTest "tdbPageDefragmentTest.cpp")
 target_link_libraries(tdbPageDefragmentTest tdb gtest gtest_main)
 
+# page recycling testing
+add_executable(tdbPageRecycleTest "tdbPageRecycleTest.cpp")
+target_link_libraries(tdbPageRecycleTest tdb gtest gtest_main)
+
diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp
new file mode 100644
index 0000000000..39e89aaf3d
--- /dev/null
+++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp
@@ -0,0 +1,674 @@
+#include <gtest/gtest.h>
+
+#define ALLOW_FORBID_FUNC
+#include "os.h"
+#include "tdb.h"
+
+#include <shared_mutex>
+#include <string>
+#include <thread>
+#include <vector>
+#include "tlog.h"
+
+typedef struct SPoolMem {
+  int64_t          size;
+  struct SPoolMem *prev;
+  struct SPoolMem *next;
+} SPoolMem;
+
+static SPoolMem *openPool() {
+  SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool));
+
+  pPool->prev = pPool->next = pPool;
+  pPool->size = 0;
+
+  return pPool;
+}
+
+static void clearPool(SPoolMem *pPool) {
+  SPoolMem *pMem;
+
+  do {
+    pMem = pPool->next;
+
+    if (pMem == pPool) break;
+
+    pMem->next->prev = pMem->prev;
+    pMem->prev->next = pMem->next;
+    pPool->size -= pMem->size;
+
+    taosMemoryFree(pMem);
+  } while (1);
+
+  assert(pPool->size == 0);
+}
+
+static void closePool(SPoolMem *pPool) {
+  clearPool(pPool);
+  taosMemoryFree(pPool);
+}
+
+static void *poolMalloc(void *arg, size_t size) {
+  void     *ptr = NULL;
+  SPoolMem *pPool = (SPoolMem *)arg;
+  SPoolMem *pMem;
+
+  pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size);
+  if (pMem == NULL) {
+    assert(0);
+  }
+
+  pMem->size = sizeof(*pMem) + size;
+  pMem->next = pPool->next;
+  pMem->prev = pPool;
+
+  pPool->next->prev = pMem;
+  pPool->next = pMem;
+  pPool->size += pMem->size;
+
+  ptr = (void *)(&pMem[1]);
+  return ptr;
+}
+
+static void poolFree(void *arg, void *ptr) {
+  SPoolMem *pPool = (SPoolMem *)arg;
+  SPoolMem *pMem;
+
+  pMem = &(((SPoolMem *)ptr)[-1]);
+
+  pMem->next->prev = pMem->prev;
+  pMem->prev->next = pMem->next;
+  pPool->size -= pMem->size;
+
+  taosMemoryFree(pMem);
+}
+
+static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
+  int k1, k2;
+
+  std::string s1((char *)pKey1 + 3, kLen1 - 3);
+  std::string s2((char *)pKey2 + 3, kLen2 - 3);
+  k1 = stoi(s1);
+  k2 = stoi(s2);
+
+  if (k1 < k2) {
+    return -1;
+  } else if (k1 > k2) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) {
+  int mlen;
+  int cret;
+
+  ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL);
+
+  mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2;
+  cret = memcmp(pKey1, pKey2, mlen);
+  if (cret == 0) {
+    if (keyLen1 < keyLen2) {
+      cret = -1;
+    } else if (keyLen1 > keyLen2) {
+      cret = 1;
+    } else {
+      cret = 0;
+    }
+  }
+  return cret;
+}
+
+static void generateBigVal(char *val, int valLen) {
+  for (int i = 0; i < valLen; ++i) {
+    char c = char(i & 0xff);
+    if (c == 0) {
+      c = 1;
+    }
+    val[i] = c;
+  }
+}
+
+static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) {
+  TDB *pEnv = NULL;
+
+  int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0);
+  if (ret) {
+    pEnv = NULL;
+  }
+
+  return pEnv;
+}
+
+static void insertOfp(void) {
+  int ret = 0;
+
+  taosRemoveDir("tdb");
+
+  // open Env
+  int const pageSize = 4096;
+  int const pageNum = 64;
+  TDB      *pEnv = openEnv("tdb", pageSize, pageNum);
+  GTEST_ASSERT_NE(pEnv, nullptr);
+
+  // open db
+  TTB          *pDb = NULL;
+  tdb_cmpr_fn_t compFunc = tKeyCmpr;
+  // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0);
+  ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // open the pool
+  SPoolMem *pPool = openPool();
+
+  // start a transaction
+  TXN *txn = NULL;
+
+  tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+
+  // generate value payload
+  // char val[((4083 - 4 - 3 - 2) + 1) * 100];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+  char val[32605];
+  int  valLen = sizeof(val) / sizeof(val[0]);
+  generateBigVal(val, valLen);
+
+  // insert the generated big data
+  // char const *key = "key1";
+  char const *key = "key123456789";
+  ret = tdbTbInsert(pDb, key, strlen(key), val, valLen, txn);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // commit current transaction
+  tdbCommit(pEnv, txn);
+  tdbPostCommit(pEnv, txn);
+}
+
+TEST(TdbPageRecycleTest, DISABLED_TbInsertTest) {
+  // TEST(TdbPageRecycleTest, TbInsertTest) {
+  // ofp inserting
+  insertOfp();
+}
+
+TEST(TdbPageRecycleTest, DISABLED_TbGetTest) {
+  // TEST(TdbPageRecycleTest, TbGetTest) {
+  insertOfp();
+
+  // open Env
+  int const pageSize = 4096;
+  int const pageNum = 64;
+  TDB      *pEnv = openEnv("tdb", pageSize, pageNum);
+  GTEST_ASSERT_NE(pEnv, nullptr);
+
+  // open db
+  TTB          *pDb = NULL;
+  tdb_cmpr_fn_t compFunc = tKeyCmpr;
+  // int           ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0);
+  int ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // generate value payload
+  // char val[((4083 - 4 - 3 - 2) + 1) * 100];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+  char val[32605];
+  int  valLen = sizeof(val) / sizeof(val[0]);
+  generateBigVal(val, valLen);
+
+  {  // Query the data
+    void *pVal = NULL;
+    int   vLen;
+
+    // char const *key = "key1";
+    char const *key = "key123456789";
+    ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen);
+    ASSERT(ret == 0);
+    GTEST_ASSERT_EQ(ret, 0);
+
+    GTEST_ASSERT_EQ(vLen, valLen);
+    GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0);
+
+    tdbFree(pVal);
+  }
+}
+
+TEST(TdbPageRecycleTest, DISABLED_TbDeleteTest) {
+  // TEST(TdbPageRecycleTest, TbDeleteTest) {
+  int ret = 0;
+
+  taosRemoveDir("tdb");
+
+  // open Env
+  int const pageSize = 4096;
+  int const pageNum = 64;
+  TDB      *pEnv = openEnv("tdb", pageSize, pageNum);
+  GTEST_ASSERT_NE(pEnv, nullptr);
+
+  // open db
+  TTB          *pDb = NULL;
+  tdb_cmpr_fn_t compFunc = tKeyCmpr;
+  ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // open the pool
+  SPoolMem *pPool = openPool();
+
+  // start a transaction
+  TXN *txn;
+
+  tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+
+  // generate value payload
+  // char val[((4083 - 4 - 3 - 2) + 1) * 100];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+  char val[((4083 - 4 - 3 - 2) + 1) * 2];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+  int  valLen = sizeof(val) / sizeof(val[0]);
+  generateBigVal(val, valLen);
+
+  {  // insert the generated big data
+    ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn);
+    GTEST_ASSERT_EQ(ret, 0);
+  }
+
+  {  // query the data
+    void *pVal = NULL;
+    int   vLen;
+
+    ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen);
+    ASSERT(ret == 0);
+    GTEST_ASSERT_EQ(ret, 0);
+
+    GTEST_ASSERT_EQ(vLen, valLen);
+    GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0);
+
+    tdbFree(pVal);
+  }
+  /* open to debug committed file
+tdbCommit(pEnv, &txn);
+tdbTxnClose(&txn);
+
+++txnid;
+tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+tdbBegin(pEnv, &txn);
+  */
+  {  // upsert the data
+    ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), txn);
+    GTEST_ASSERT_EQ(ret, 0);
+  }
+
+  {  // query the upserted data
+    void *pVal = NULL;
+    int   vLen;
+
+    ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen);
+    ASSERT(ret == 0);
+    GTEST_ASSERT_EQ(ret, 0);
+
+    GTEST_ASSERT_EQ(vLen, strlen("value1"));
+    GTEST_ASSERT_EQ(memcmp("value1", pVal, vLen), 0);
+
+    tdbFree(pVal);
+  }
+
+  {  // delete the data
+    ret = tdbTbDelete(pDb, "key1", strlen("key1"), txn);
+    GTEST_ASSERT_EQ(ret, 0);
+  }
+
+  {  // query the deleted data
+    void *pVal = NULL;
+    int   vLen = -1;
+
+    ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen);
+    ASSERT(ret == -1);
+    GTEST_ASSERT_EQ(ret, -1);
+
+    GTEST_ASSERT_EQ(vLen, -1);
+    GTEST_ASSERT_EQ(pVal, nullptr);
+
+    tdbFree(pVal);
+  }
+
+  // commit current transaction
+  tdbCommit(pEnv, txn);
+  tdbPostCommit(pEnv, txn);
+}
+
+TEST(TdbPageRecycleTest, DISABLED_simple_insert1) {
+  // TEST(TdbPageRecycleTest, simple_insert1) {
+  int           ret;
+  TDB          *pEnv;
+  TTB          *pDb;
+  tdb_cmpr_fn_t compFunc;
+  int           nData = 1;
+  TXN          *txn;
+  int const     pageSize = 4096;
+
+  taosRemoveDir("tdb");
+
+  // Open Env
+  ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // Create a database
+  compFunc = tKeyCmpr;
+  ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  {
+    char key[64];
+    // char      val[(4083 - 4 - 3 - 2)]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+    char      val[(4083 - 4 - 3 - 2) + 1];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+    int64_t   poolLimit = 4096;             // 1M pool limit
+    SPoolMem *pPool;
+
+    // open the pool
+    pPool = openPool();
+
+    // start a transaction
+    tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+
+    for (int iData = 1; iData <= nData; iData++) {
+      sprintf(key, "key0");
+      sprintf(val, "value%d", iData);
+
+      // ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
+      // GTEST_ASSERT_EQ(ret, 0);
+
+      // generate value payload
+      int valLen = sizeof(val) / sizeof(val[0]);
+      for (int i = 6; i < valLen; ++i) {
+        char c = char(i & 0xff);
+        if (c == 0) {
+          c = 1;
+        }
+        val[i] = c;
+      }
+
+      ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn);
+      GTEST_ASSERT_EQ(ret, 0);
+
+      // if pool is full, commit the transaction and start a new one
+      if (pPool->size >= poolLimit) {
+        // commit current transaction
+        tdbCommit(pEnv, txn);
+        tdbPostCommit(pEnv, txn);
+
+        // start a new transaction
+        clearPool(pPool);
+
+        tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+      }
+    }
+
+    // commit the transaction
+    tdbCommit(pEnv, txn);
+    tdbPostCommit(pEnv, txn);
+
+    {  // Query the data
+      void *pVal = NULL;
+      int   vLen;
+
+      for (int i = 1; i <= nData; i++) {
+        sprintf(key, "key%d", i);
+        // sprintf(val, "value%d", i);
+
+        ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen);
+        ASSERT(ret == 0);
+        GTEST_ASSERT_EQ(ret, 0);
+
+        GTEST_ASSERT_EQ(vLen, sizeof(val) / sizeof(val[0]));
+        GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0);
+      }
+
+      tdbFree(pVal);
+    }
+
+    {  // Iterate to query the DB data
+      TBC  *pDBC;
+      void *pKey = NULL;
+      void *pVal = NULL;
+      int   vLen, kLen;
+      int   count = 0;
+
+      ret = tdbTbcOpen(pDb, &pDBC, NULL);
+      GTEST_ASSERT_EQ(ret, 0);
+
+      tdbTbcMoveToFirst(pDBC);
+
+      for (;;) {
+        ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen);
+        if (ret < 0) break;
+
+        // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " ";
+        // std::cout.write((char *)pVal, vLen) /* << " " << vLen */;
+        // std::cout << std::endl;
+
+        count++;
+      }
+
+      GTEST_ASSERT_EQ(count, nData);
+
+      tdbTbcClose(pDBC);
+
+      tdbFree(pKey);
+      tdbFree(pVal);
+    }
+  }
+
+  ret = tdbTbDrop(pDb);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // Close a database
+  tdbTbClose(pDb);
+
+  // Close Env
+  ret = tdbClose(pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
+}
+
+// TEST(TdbPageRecycleTest, DISABLED_seq_insert) {
+TEST(TdbPageRecycleTest, seq_insert) {
+  int           ret = 0;
+  TDB          *pEnv = NULL;
+  TTB          *pDb = NULL;
+  tdb_cmpr_fn_t compFunc;
+  int           nData = 256;
+  TXN          *txn = NULL;
+  int const     pageSize = 4 * 1024;
+
+  taosRemoveDir("tdb");
+
+  // Open Env
+  ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // Create a database
+  compFunc = tKeyCmpr;
+  ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // 1, insert nData kv
+  {
+    char      key[64];
+    char      val[(4083 - 4 - 3 - 2) + 1];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+    int64_t   poolLimit = 4096;             // 1M pool limit
+    SPoolMem *pPool;
+
+    // open the pool
+    pPool = openPool();
+
+    // start a transaction
+    tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+
+    for (int iData = 0; iData < nData; ++iData) {
+      sprintf(key, "key%03d", iData);
+      sprintf(val, "value%03d", iData);
+
+      ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn);
+      GTEST_ASSERT_EQ(ret, 0);
+      // if pool is full, commit the transaction and start a new one
+      if (pPool->size >= poolLimit) {
+        // commit current transaction
+        tdbCommit(pEnv, txn);
+        tdbPostCommit(pEnv, txn);
+
+        // start a new transaction
+        clearPool(pPool);
+
+        tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+      }
+    }
+
+    // commit the transaction
+    tdbCommit(pEnv, txn);
+    tdbPostCommit(pEnv, txn);
+
+    // 2, delete nData/2 records
+
+    closePool(pPool);
+  }
+
+  // Close a database
+  tdbTbClose(pDb);
+
+  // Close Env
+  ret = tdbClose(pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  system("ls -l ./tdb");
+}
+
+// TEST(TdbPageRecycleTest, DISABLED_seq_delete) {
+TEST(TdbPageRecycleTest, seq_delete) {
+  int           ret = 0;
+  TDB          *pEnv = NULL;
+  TTB          *pDb = NULL;
+  tdb_cmpr_fn_t compFunc;
+  int           nData = 256;
+  TXN          *txn = NULL;
+  int const     pageSize = 4 * 1024;
+
+  // Open Env
+  ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // Create a database
+  compFunc = tKeyCmpr;
+  ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // 2, delete nData/2 records
+  {
+    char      key[64];
+    char      val[(4083 - 4 - 3 - 2) + 1];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+    int64_t   poolLimit = 4096;             // 1M pool limit
+    SPoolMem *pPool;
+
+    // open the pool
+    pPool = openPool();
+
+    // start a transaction
+    tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+
+    for (int iData = 0; iData < nData; iData++) {
+      // if (iData % 2 == 0) continue;
+
+      sprintf(key, "key%03d", iData);
+      sprintf(val, "value%03d", iData);
+
+      {  // delete the data
+        ret = tdbTbDelete(pDb, key, strlen(key), txn);
+        GTEST_ASSERT_EQ(ret, 0);
+      }
+      // if pool is full, commit the transaction and start a new one
+      if (pPool->size >= poolLimit) {
+        // commit current transaction
+        tdbCommit(pEnv, txn);
+        tdbPostCommit(pEnv, txn);
+
+        // start a new transaction
+        clearPool(pPool);
+
+        tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+      }
+    }
+
+    // commit the transaction
+    tdbCommit(pEnv, txn);
+    tdbPostCommit(pEnv, txn);
+
+    closePool(pPool);
+  }
+
+  // Close a database
+  tdbTbClose(pDb);
+
+  // Close Env
+  ret = tdbClose(pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  system("ls -l ./tdb");
+}
+
+// TEST(TdbPageRecycleTest, DISABLED_recycly_insert) {
+TEST(TdbPageRecycleTest, recycly_insert) {
+  int           ret = 0;
+  TDB          *pEnv = NULL;
+  TTB          *pDb = NULL;
+  tdb_cmpr_fn_t compFunc = tKeyCmpr;
+  int           nData = 256;
+  TXN          *txn = NULL;
+  int const     pageSize = 4 * 1024;
+
+  // Open Env
+  ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // Create a database
+  ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // 3, insert 32k records
+  {
+    char      key[64];
+    char      val[(4083 - 4 - 3 - 2) + 1];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+    int64_t   poolLimit = 4096;
+    SPoolMem *pPool;
+
+    // open the pool
+    pPool = openPool();
+
+    // start a transaction
+    tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+
+    for (int iData = nData; iData < nData + nData; iData++) {
+      sprintf(key, "key%03d", iData);
+      sprintf(val, "value%03d", iData);
+
+      ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn);
+      GTEST_ASSERT_EQ(ret, 0);
+
+      if (pPool->size >= poolLimit) {
+        tdbCommit(pEnv, txn);
+        tdbPostCommit(pEnv, txn);
+
+        // start a new transaction
+        clearPool(pPool);
+
+        tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+      }
+    }
+
+    tdbCommit(pEnv, txn);
+    tdbPostCommit(pEnv, txn);
+
+    closePool(pPool);
+  }
+
+  // Close a database
+  tdbTbClose(pDb);
+
+  // Close Env
+  ret = tdbClose(pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  system("ls -l ./tdb");
+}

From 621d4b20c0361efb5d24261c700db89f70f8c76e Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 12 Jun 2023 16:43:49 +0800
Subject: [PATCH 02/17] tdb/pager: remove debug log

---
 source/libs/tdb/src/db/tdbPager.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 5f187d339e..8984de6476 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -316,10 +316,6 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
       return -1;
     }
 
-    if (!TDB_PAGE_TOTAL_CELLS(pPage) && TDB_PAGE_PGNO(pPage) > 1) {
-      tdbDebug("pager/commit: %p, %d/%d, txnId:%" PRId64, pPager, pPager->dbOrigSize, pPager->dbFileSize, pTxn->txnId);
-    }
-
     ret = tdbPagerPWritePageToDB(pPager, pPage);
     if (ret < 0) {
       tdbError("failed to write page to db since %s", tstrerror(terrno));

From 8fee813de633881fa4c998139ec3b9bf86f08d1c Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 13 Jun 2023 14:35:49 +0800
Subject: [PATCH 03/17] tdb/alloc-page: new param pTxn to fix memory leaking

---
 source/libs/tdb/src/db/tdbPager.c | 24 ++++++++++++++++--------
 source/libs/tdb/src/inc/tdbInt.h  |  6 +++---
 2 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 8984de6476..a1d57db8d3 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -338,10 +338,13 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
     if (pTxn->jPageSet) {
       hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage)));
     }
+
+    tdbTrace("tdb/pager-commit: remove page: %p %d from dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt);
+
     tdbPCacheRelease(pPager->pCache, pPage, pTxn);
   }
 
-  tdbTrace("pager/commit reset dirty tree: %p", &pPager->rbt);
+  tdbTrace("tdb/pager-commit reset dirty tree: %p", &pPager->rbt);
   tRBTreeCreate(&pPager->rbt, pageCmpFn);
 
   // sync the db file
@@ -629,6 +632,8 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) {
   return 0;
 }
 
+static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno, TXN *pTxn);
+
 int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg,
                       TXN *pTxn) {
   SPage *pPage;
@@ -643,7 +648,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa
   // alloc new page
   if (pgno == 0) {
     loadPage = 0;
-    ret = tdbPagerAllocPage(pPager, &pgno);
+    ret = tdbPagerAllocPage(pPager, &pgno, pTxn);
     if (ret < 0) {
       tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno);
       return -1;
@@ -706,7 +711,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) {
   return code;
 }
 
-static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) {
+static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) {
   int  code = 0;
   TBC *pCur;
 
@@ -714,13 +719,14 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) {
     return 0;
   }
 
-  code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, NULL);
+  code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn);
   if (code < 0) {
     return 0;
   }
 
   code = tdbTbcMoveToFirst(pCur);
   if (code) {
+    tdbError("tdb/remove-free-page: moveto first failed with ret: %d.", code);
     tdbTbcClose(pCur);
     return 0;
   }
@@ -730,6 +736,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) {
 
   code = tdbTbcGet(pCur, (const void **)&pKey, &nKey, NULL, NULL);
   if (code < 0) {
+    tdbError("tdb/remove-free-page: tbc get failed with ret: %d.", code);
     tdbTbcClose(pCur);
     return 0;
   }
@@ -738,6 +745,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) {
 
   code = tdbTbcDelete(pCur);
   if (code < 0) {
+    tdbError("tdb/remove-free-page: tbc delete failed with ret: %d.", code);
     tdbTbcClose(pCur);
     return 0;
   }
@@ -745,9 +753,9 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno) {
   return 0;
 }
 
-static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) {
+static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno, TXN *pTxn) {
   // TODO: Allocate a page from the free list
-  return tdbPagerRemoveFreePage(pPager, ppgno);
+  return tdbPagerRemoveFreePage(pPager, ppgno, pTxn);
 }
 
 static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) {
@@ -755,13 +763,13 @@ static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) {
   return 0;
 }
 
-int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) {
+static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno, TXN *pTxn) {
   int ret;
 
   *ppgno = 0;
 
   // Try to allocate from the free list of the pager
-  ret = tdbPagerAllocFreePage(pPager, ppgno);
+  ret = tdbPagerAllocFreePage(pPager, ppgno, pTxn);
   if (ret < 0) {
     return -1;
   }
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index e65edb4afe..bd680da09e 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -199,9 +199,9 @@ int  tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initP
                        TXN *pTxn);
 void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
 int  tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn);
-int  tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
-int  tdbPagerRestoreJournals(SPager *pPager);
-int  tdbPagerRollback(SPager *pPager);
+// int  tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
+int tdbPagerRestoreJournals(SPager *pPager);
+int tdbPagerRollback(SPager *pPager);
 
 // tdbPCache.c ====================================
 #define TDB_PCACHE_PAGE    \

From 40b741dfee02b21267c42fd242ac2f60a2bc7206 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 20 Jun 2023 07:52:45 +0800
Subject: [PATCH 04/17] tdb/pager: comment out error log

---
 source/libs/tdb/src/db/tdbPager.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index a1d57db8d3..9c00a82826 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -736,7 +736,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) {
 
   code = tdbTbcGet(pCur, (const void **)&pKey, &nKey, NULL, NULL);
   if (code < 0) {
-    tdbError("tdb/remove-free-page: tbc get failed with ret: %d.", code);
+    // tdbError("tdb/remove-free-page: tbc get failed with ret: %d.", code);
     tdbTbcClose(pCur);
     return 0;
   }
@@ -754,7 +754,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) {
 }
 
 static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno, TXN *pTxn) {
-  // TODO: Allocate a page from the free list
+  // Allocate a page from the free list
   return tdbPagerRemoveFreePage(pPager, ppgno, pTxn);
 }
 

From 4e3df6606bb3c03370722c6afc04cd4fab3b5108 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 20 Jun 2023 15:06:17 +0800
Subject: [PATCH 05/17] tdb/btree: recyle pNews 0

---
 source/libs/tdb/src/db/tdbBtree.c | 8 ++++----
 source/libs/tdb/src/db/tdbPager.c | 4 ++++
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index bb02db8bb8..3afdb9a84f 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -860,10 +860,10 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
 
     if (!TDB_BTREE_PAGE_IS_LEAF(pNews[0])) {
       ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno;
-    } else {
-      // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pNews[0]));
-      tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pNews[0]), pTxn);
-    }
+    }  // else {
+    // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pNews[0]));
+    tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pNews[0]), pTxn);
+    //}
   }
 
   for (int i = 0; i < 3; i++) {
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 9c00a82826..4e29ca45ca 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -703,8 +703,10 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) {
 int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) {
   int code = 0;
 
+  // tdbError("tdb/insert-free-page: tbc get page: %d.", pgno);
   code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn);
   if (code < 0) {
+    tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code);
     return -1;
   }
 
@@ -742,6 +744,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) {
   }
 
   *pPgno = *(SPgno *)pKey;
+  // tdbError("tdb/remove-free-page: tbc get page: %d.", *pPgno);
 
   code = tdbTbcDelete(pCur);
   if (code < 0) {
@@ -760,6 +763,7 @@ static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno, TXN *pTxn) {
 
 static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) {
   *ppgno = ++pPager->dbFileSize;
+  // tdbError("tdb/alloc-new-page: %d.", *ppgno);
   return 0;
 }
 

From 76a734c53a38cbb710ffe7519ec1e4fee4a22b55 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Tue, 20 Jun 2023 15:07:49 +0800
Subject: [PATCH 06/17] tdb/test: fix recycle testing cases

---
 source/libs/tdb/test/tdbPageRecycleTest.cpp | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp
index 39e89aaf3d..e4787fcc70 100644
--- a/source/libs/tdb/test/tdbPageRecycleTest.cpp
+++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp
@@ -464,13 +464,15 @@ TEST(TdbPageRecycleTest, DISABLED_simple_insert1) {
   GTEST_ASSERT_EQ(ret, 0);
 }
 
+static const int nDataConst = 256 * 19;
+
 // TEST(TdbPageRecycleTest, DISABLED_seq_insert) {
 TEST(TdbPageRecycleTest, seq_insert) {
   int           ret = 0;
   TDB          *pEnv = NULL;
   TTB          *pDb = NULL;
   tdb_cmpr_fn_t compFunc;
-  int           nData = 256;
+  int           nData = nDataConst;
   TXN          *txn = NULL;
   int const     pageSize = 4 * 1024;
 
@@ -480,11 +482,13 @@ TEST(TdbPageRecycleTest, seq_insert) {
   ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
   GTEST_ASSERT_EQ(ret, 0);
 
+  printf("tdb opened\n");
   // Create a database
   compFunc = tKeyCmpr;
   ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0);
   GTEST_ASSERT_EQ(ret, 0);
 
+  printf("tb opened\n");
   // 1, insert nData kv
   {
     char      key[64];
@@ -542,7 +546,7 @@ TEST(TdbPageRecycleTest, seq_delete) {
   TDB          *pEnv = NULL;
   TTB          *pDb = NULL;
   tdb_cmpr_fn_t compFunc;
-  int           nData = 256;
+  int           nData = nDataConst;
   TXN          *txn = NULL;
   int const     pageSize = 4 * 1024;
 
@@ -614,7 +618,7 @@ TEST(TdbPageRecycleTest, recycly_insert) {
   TDB          *pEnv = NULL;
   TTB          *pDb = NULL;
   tdb_cmpr_fn_t compFunc = tKeyCmpr;
-  int           nData = 256;
+  int           nData = nDataConst;
   TXN          *txn = NULL;
   int const     pageSize = 4 * 1024;
 
@@ -639,7 +643,8 @@ TEST(TdbPageRecycleTest, recycly_insert) {
     // start a transaction
     tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
 
-    for (int iData = nData; iData < nData + nData; iData++) {
+    // for (int iData = nData; iData < nData + nData; iData++) {
+    for (int iData = 0; iData < nData; iData++) {
       sprintf(key, "key%03d", iData);
       sprintf(val, "value%03d", iData);
 

From fe197ccf9c734ef20a5669a822129401903dba25 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Sun, 25 Jun 2023 10:46:28 +0800
Subject: [PATCH 07/17] tdb/ofp-test: fix memory leaks

---
 source/libs/tdb/test/tdbExOVFLTest.cpp | 27 ++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp
index b16bc643d3..325703c946 100644
--- a/source/libs/tdb/test/tdbExOVFLTest.cpp
+++ b/source/libs/tdb/test/tdbExOVFLTest.cpp
@@ -190,6 +190,15 @@ static void insertOfp(void) {
   // commit current transaction
   tdbCommit(pEnv, txn);
   tdbPostCommit(pEnv, txn);
+
+  closePool(pPool);
+
+  // Close a database
+  tdbTbClose(pDb);
+
+  // Close Env
+  ret = tdbClose(pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
 }
 
 // TEST(TdbOVFLPagesTest, DISABLED_TbInsertTest) {
@@ -233,6 +242,13 @@ TEST(TdbOVFLPagesTest, TbGetTest) {
 
     tdbFree(pVal);
   }
+
+  // Close a database
+  tdbTbClose(pDb);
+
+  // Close Env
+  ret = tdbClose(pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
 }
 
 // TEST(TdbOVFLPagesTest, DISABLED_TbDeleteTest) {
@@ -334,6 +350,15 @@ tdbBegin(pEnv, &txn);
   // commit current transaction
   tdbCommit(pEnv, txn);
   tdbPostCommit(pEnv, txn);
+
+  closePool(pPool);
+
+  // Close a database
+  tdbTbClose(pDb);
+
+  // Close Env
+  ret = tdbClose(pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
 }
 
 // TEST(tdb_test, DISABLED_simple_insert1) {
@@ -407,6 +432,8 @@ TEST(tdb_test, simple_insert1) {
     tdbCommit(pEnv, txn);
     tdbPostCommit(pEnv, txn);
 
+    closePool(pPool);
+
     {  // Query the data
       void *pVal = NULL;
       int   vLen;

From 204999d57ecc91f28890a9371a5d42b10f3be587 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Mon, 26 Jun 2023 08:10:45 +0800
Subject: [PATCH 08/17] tdb/test: refactor page recycling test cases

---
 source/libs/tdb/src/db/tdbBtree.c           |  8 +-
 source/libs/tdb/test/tdbPageRecycleTest.cpp | 88 ++++-----------------
 2 files changed, 16 insertions(+), 80 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 3afdb9a84f..64ae8d1c3f 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -860,10 +860,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
 
     if (!TDB_BTREE_PAGE_IS_LEAF(pNews[0])) {
       ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno;
-    }  // else {
-    // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pNews[0]));
+    }
+
     tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pNews[0]), pTxn);
-    //}
   }
 
   for (int i = 0; i < 3; i++) {
@@ -873,15 +872,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
   }
 
   for (pageIdx = 0; pageIdx < nOlds; ++pageIdx) {
-    // printf("tdb/balance: btree balance old pgno: %d.\n", TDB_PAGE_PGNO(pOlds[pageIdx]));
     if (pageIdx >= nNews) {
-      // printf("tdb/balance: btree balance delete pgno: %d.\n", TDB_PAGE_PGNO(pOlds[pageIdx]));
       tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pOlds[pageIdx]), pTxn);
     }
     tdbPagerReturnPage(pBt->pPager, pOlds[pageIdx], pTxn);
   }
   for (; pageIdx < nNews; ++pageIdx) {
-    // printf("tdb/balance: btree balance new pgno: %d.\n", TDB_PAGE_PGNO(pNews[pageIdx]));
     tdbPagerReturnPage(pBt->pPager, pNews[pageIdx], pTxn);
   }
 
diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp
index e4787fcc70..05b19fc4eb 100644
--- a/source/libs/tdb/test/tdbPageRecycleTest.cpp
+++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp
@@ -464,31 +464,25 @@ TEST(TdbPageRecycleTest, DISABLED_simple_insert1) {
   GTEST_ASSERT_EQ(ret, 0);
 }
 
-static const int nDataConst = 256 * 19;
+static void clearDb(char const *db) { taosRemoveDir(db); }
 
-// TEST(TdbPageRecycleTest, DISABLED_seq_insert) {
-TEST(TdbPageRecycleTest, seq_insert) {
+static void insertDb(int nData) {
   int           ret = 0;
   TDB          *pEnv = NULL;
   TTB          *pDb = NULL;
   tdb_cmpr_fn_t compFunc;
-  int           nData = nDataConst;
   TXN          *txn = NULL;
   int const     pageSize = 4 * 1024;
 
-  taosRemoveDir("tdb");
-
   // Open Env
   ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
   GTEST_ASSERT_EQ(ret, 0);
 
-  printf("tdb opened\n");
   // Create a database
   compFunc = tKeyCmpr;
   ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0);
   GTEST_ASSERT_EQ(ret, 0);
 
-  printf("tb opened\n");
   // 1, insert nData kv
   {
     char      key[64];
@@ -540,13 +534,11 @@ TEST(TdbPageRecycleTest, seq_insert) {
   system("ls -l ./tdb");
 }
 
-// TEST(TdbPageRecycleTest, DISABLED_seq_delete) {
-TEST(TdbPageRecycleTest, seq_delete) {
+static void deleteDb(int nData) {
   int           ret = 0;
   TDB          *pEnv = NULL;
   TTB          *pDb = NULL;
   tdb_cmpr_fn_t compFunc;
-  int           nData = nDataConst;
   TXN          *txn = NULL;
   int const     pageSize = 4 * 1024;
 
@@ -612,68 +604,16 @@ TEST(TdbPageRecycleTest, seq_delete) {
   system("ls -l ./tdb");
 }
 
-// TEST(TdbPageRecycleTest, DISABLED_recycly_insert) {
-TEST(TdbPageRecycleTest, recycly_insert) {
-  int           ret = 0;
-  TDB          *pEnv = NULL;
-  TTB          *pDb = NULL;
-  tdb_cmpr_fn_t compFunc = tKeyCmpr;
-  int           nData = nDataConst;
-  TXN          *txn = NULL;
-  int const     pageSize = 4 * 1024;
+static const int nDataConst = 256 * 19;
 
-  // Open Env
-  ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0);
-  GTEST_ASSERT_EQ(ret, 0);
-
-  // Create a database
-  ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0);
-  GTEST_ASSERT_EQ(ret, 0);
-
-  // 3, insert 32k records
-  {
-    char      key[64];
-    char      val[(4083 - 4 - 3 - 2) + 1];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
-    int64_t   poolLimit = 4096;
-    SPoolMem *pPool;
-
-    // open the pool
-    pPool = openPool();
-
-    // start a transaction
-    tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-
-    // for (int iData = nData; iData < nData + nData; iData++) {
-    for (int iData = 0; iData < nData; iData++) {
-      sprintf(key, "key%03d", iData);
-      sprintf(val, "value%03d", iData);
-
-      ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn);
-      GTEST_ASSERT_EQ(ret, 0);
-
-      if (pPool->size >= poolLimit) {
-        tdbCommit(pEnv, txn);
-        tdbPostCommit(pEnv, txn);
-
-        // start a new transaction
-        clearPool(pPool);
-
-        tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-      }
-    }
-
-    tdbCommit(pEnv, txn);
-    tdbPostCommit(pEnv, txn);
-
-    closePool(pPool);
-  }
-
-  // Close a database
-  tdbTbClose(pDb);
-
-  // Close Env
-  ret = tdbClose(pEnv);
-  GTEST_ASSERT_EQ(ret, 0);
-
-  system("ls -l ./tdb");
+// TEST(TdbPageRecycleTest, DISABLED_seq_insert) {
+TEST(TdbPageRecycleTest, seq_insert) {
+  clearDb("tdb");
+  insertDb(nDataConst);
 }
+
+// TEST(TdbPageRecycleTest, DISABLED_seq_delete) {
+TEST(TdbPageRecycleTest, seq_delete) { deleteDb(nDataConst); }
+
+// TEST(TdbPageRecycleTest, DISABLED_recycly_insert) {
+TEST(TdbPageRecycleTest, recycly_insert) { insertDb(nDataConst); }

From f8921199e78f00cd5fa666fb33294eb39ec1c377 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Wed, 28 Jun 2023 09:37:26 +0800
Subject: [PATCH 09/17] tdb/test: cases for ofp recycling

---
 source/libs/tdb/test/tdbPageRecycleTest.cpp | 105 +++++++++++++++++---
 1 file changed, 89 insertions(+), 16 deletions(-)

diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp
index 05b19fc4eb..2f2dd0659b 100644
--- a/source/libs/tdb/test/tdbPageRecycleTest.cpp
+++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp
@@ -120,16 +120,6 @@ static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, in
   return cret;
 }
 
-static void generateBigVal(char *val, int valLen) {
-  for (int i = 0; i < valLen; ++i) {
-    char c = char(i & 0xff);
-    if (c == 0) {
-      c = 1;
-    }
-    val[i] = c;
-  }
-}
-
 static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) {
   TDB *pEnv = NULL;
 
@@ -141,11 +131,19 @@ static TDB *openEnv(char const *envName, int const pageSize, int const pageNum)
   return pEnv;
 }
 
+static void generateBigVal(char *val, int valLen) {
+  for (int i = 0; i < valLen; ++i) {
+    char c = char(i & 0xff);
+    if (c == 0) {
+      c = 1;
+    }
+    val[i] = c;
+  }
+}
+
 static void insertOfp(void) {
   int ret = 0;
 
-  taosRemoveDir("tdb");
-
   // open Env
   int const pageSize = 4096;
   int const pageNum = 64;
@@ -156,7 +154,7 @@ static void insertOfp(void) {
   TTB          *pDb = NULL;
   tdb_cmpr_fn_t compFunc = tKeyCmpr;
   // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0);
-  ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0);
+  ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0);
   GTEST_ASSERT_EQ(ret, 0);
 
   // open the pool
@@ -176,22 +174,35 @@ static void insertOfp(void) {
   // insert the generated big data
   // char const *key = "key1";
   char const *key = "key123456789";
-  ret = tdbTbInsert(pDb, key, strlen(key), val, valLen, txn);
+  ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn);
   GTEST_ASSERT_EQ(ret, 0);
 
   // commit current transaction
   tdbCommit(pEnv, txn);
   tdbPostCommit(pEnv, txn);
+
+  closePool(pPool);
+
+  // Close a database
+  tdbTbClose(pDb);
+
+  // Close Env
+  ret = tdbClose(pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
 }
 
+static void clearDb(char const *db) { taosRemoveDir(db); }
+
 TEST(TdbPageRecycleTest, DISABLED_TbInsertTest) {
   // TEST(TdbPageRecycleTest, TbInsertTest) {
   // ofp inserting
+  clearDb("tdb");
   insertOfp();
 }
 
 TEST(TdbPageRecycleTest, DISABLED_TbGetTest) {
   // TEST(TdbPageRecycleTest, TbGetTest) {
+  clearDb("tdb");
   insertOfp();
 
   // open Env
@@ -464,8 +475,6 @@ TEST(TdbPageRecycleTest, DISABLED_simple_insert1) {
   GTEST_ASSERT_EQ(ret, 0);
 }
 
-static void clearDb(char const *db) { taosRemoveDir(db); }
-
 static void insertDb(int nData) {
   int           ret = 0;
   TDB          *pEnv = NULL;
@@ -617,3 +626,67 @@ TEST(TdbPageRecycleTest, seq_delete) { deleteDb(nDataConst); }
 
 // TEST(TdbPageRecycleTest, DISABLED_recycly_insert) {
 TEST(TdbPageRecycleTest, recycly_insert) { insertDb(nDataConst); }
+
+// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp) {
+TEST(TdbPageRecycleTest, recycly_seq_insert_ofp) {
+  clearDb("tdb");
+  insertOfp();
+  system("ls -l ./tdb");
+}
+
+static void deleteOfp(void) {
+  // open Env
+  int       ret = 0;
+  int const pageSize = 4096;
+  int const pageNum = 64;
+  TDB      *pEnv = openEnv("tdb", pageSize, pageNum);
+  GTEST_ASSERT_NE(pEnv, nullptr);
+
+  // open db
+  TTB          *pDb = NULL;
+  tdb_cmpr_fn_t compFunc = tKeyCmpr;
+  ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // open the pool
+  SPoolMem *pPool = openPool();
+
+  // start a transaction
+  TXN *txn;
+
+  tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+
+  {  // delete the data
+    char const *key = "key123456789";
+    ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn);
+    GTEST_ASSERT_EQ(ret, 0);
+  }
+
+  // commit current transaction
+  tdbCommit(pEnv, txn);
+  tdbPostCommit(pEnv, txn);
+
+  closePool(pPool);
+
+  ret = tdbTbDrop(pDb);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // Close a database
+  tdbTbClose(pDb);
+
+  // Close Env
+  ret = tdbClose(pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
+}
+
+// TEST(TdbPageRecycleTest, DISABLED_seq_delete_ofp) {
+TEST(TdbPageRecycleTest, seq_delete_ofp) {
+  deleteOfp();
+  system("ls -l ./tdb");
+}
+
+// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp_again) {
+TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_again) {
+  insertOfp();
+  system("ls -l ./tdb");
+}

From a3c9b17212a1a58aebd00f95905226a88396f339 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Wed, 28 Jun 2023 10:46:01 +0800
Subject: [PATCH 10/17] tdb/ofp: recycl ofps

---
 source/libs/tdb/src/db/tdbBtree.c | 17 +++++++++++++++++
 source/libs/tdb/src/inc/tdbInt.h  | 15 ++++++++-------
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 64ae8d1c3f..65d1c30328 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -1317,6 +1317,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader,
           return -1;
         }
 
+        if (!pDecoder->ofps) {
+          pDecoder->ofps = taosArrayInit(8, sizeof(SPgno));
+        }
+        taosArrayPush(pDecoder->ofps, &pgno);
+
         ofpCell = tdbPageGetCell(ofp, 0);
 
         if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
@@ -2075,6 +2080,14 @@ int tdbBtcDelete(SBTC *pBtc) {
 
   tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt);
 
+  // recycle ofps if any
+  if (pBtc->coder.ofps) {
+    for (int i = 0; i < TARRAY_SIZE(pBtc->coder.ofps); ++i) {
+      SPgno *pgno = taosArrayGet(pBtc->coder.ofps, i);
+      tdbPagerInsertFreePage(pBtc->pBt->pPager, *pgno, pBtc->pTxn);
+    }
+  }
+
   // update interior page or do balance
   if (idx == nCells - 1) {
     if (idx) {
@@ -2370,6 +2383,10 @@ int tdbBtcClose(SBTC *pBtc) {
     tdbTxnClose(pBtc->pTxn);
   }
 
+  if (pBtc->coder.ofps) {
+    taosArrayDestroy(pBtc->coder.ofps);
+  }
+
   return 0;
 }
 
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index bd680da09e..7b08da4ca8 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -131,13 +131,14 @@ typedef struct SBtInfo {
 #define TDB_CELLDECODER_FREE_VAL(pCellDecoder) ((pCellDecoder)->freeKV & TDB_CELLD_F_VAL)
 
 typedef struct {
-  int   kLen;
-  u8   *pKey;
-  int   vLen;
-  u8   *pVal;
-  SPgno pgno;
-  u8   *pBuf;
-  u8    freeKV;
+  int     kLen;
+  u8     *pKey;
+  int     vLen;
+  u8     *pVal;
+  SPgno   pgno;
+  u8     *pBuf;
+  u8      freeKV;
+  SArray *ofps;
 } SCellDecoder;
 
 struct SBTC {

From b2c0bcb1e0480d968e45b5cac1c24ce34a4e6959 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Wed, 28 Jun 2023 13:35:14 +0800
Subject: [PATCH 11/17] tdb/ofp-recycle: fix mem leaks

---
 source/libs/tdb/src/db/tdbBtree.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 65d1c30328..ef9aaa4571 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -233,6 +233,7 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) {
   int  ret;
 
   tdbBtcOpen(&btc, pBt, pTxn);
+  btc.coder.ofps = taosArrayInit(8, sizeof(SPgno));
 
   tdbTrace("tdb delete, btc: %p, pTxn: %p", &btc, pTxn);
 
@@ -1317,10 +1318,9 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader,
           return -1;
         }
 
-        if (!pDecoder->ofps) {
-          pDecoder->ofps = taosArrayInit(8, sizeof(SPgno));
+        if (pDecoder->ofps) {
+          taosArrayPush(pDecoder->ofps, &pgno);
         }
-        taosArrayPush(pDecoder->ofps, &pgno);
 
         ofpCell = tdbPageGetCell(ofp, 0);
 

From fc79074e499c864951f08dae8d648cff389e3a60 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Wed, 28 Jun 2023 15:51:33 +0800
Subject: [PATCH 12/17] tdb/ofp: turn ofp recycle off for ci

---
 source/libs/tdb/src/db/tdbBtree.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index ef9aaa4571..7cbca72e71 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -564,6 +564,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
       }
     }
     // copy the parent key out if child pages are not leaf page
+    // childNotLeaf = !(TDB_BTREE_PAGE_IS_LEAF(pOlds[0]) || TDB_BTREE_PAGE_IS_OVFL(pOlds[0]));
     childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(pOlds[0]);
     if (childNotLeaf) {
       for (int i = 0; i < nOlds; i++) {
@@ -2084,7 +2085,7 @@ int tdbBtcDelete(SBTC *pBtc) {
   if (pBtc->coder.ofps) {
     for (int i = 0; i < TARRAY_SIZE(pBtc->coder.ofps); ++i) {
       SPgno *pgno = taosArrayGet(pBtc->coder.ofps, i);
-      tdbPagerInsertFreePage(pBtc->pBt->pPager, *pgno, pBtc->pTxn);
+      // tdbPagerInsertFreePage(pBtc->pBt->pPager, *pgno, pBtc->pTxn);
     }
   }
 

From 0ec80ff47fcb71701954741be4eee4b5c4d84422 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 29 Jun 2023 13:14:45 +0800
Subject: [PATCH 13/17] tdb/ofp-recycle: recycle ofps when dropOfp

---
 source/libs/tdb/src/db/tdbBtree.c           | 51 ++++++-----
 source/libs/tdb/src/db/tdbPager.c           | 23 ++++-
 source/libs/tdb/src/inc/tdbInt.h            |  2 +-
 source/libs/tdb/test/tdbPageRecycleTest.cpp | 97 +++++++++++++++++++++
 4 files changed, 149 insertions(+), 24 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 7cbca72e71..6921a26f19 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -233,8 +233,9 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) {
   int  ret;
 
   tdbBtcOpen(&btc, pBt, pTxn);
+  /*
   btc.coder.ofps = taosArrayInit(8, sizeof(SPgno));
-
+  */
   tdbTrace("tdb delete, btc: %p, pTxn: %p", &btc, pTxn);
 
   // move the cursor
@@ -864,7 +865,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
       ((SIntHdr *)(pParent->pData))->pgno = ((SIntHdr *)(pNews[0]->pData))->pgno;
     }
 
-    tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pNews[0]), pTxn);
+    tdbPagerInsertFreePage(pBt->pPager, pNews[0], pTxn);
   }
 
   for (int i = 0; i < 3; i++) {
@@ -875,7 +876,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
 
   for (pageIdx = 0; pageIdx < nOlds; ++pageIdx) {
     if (pageIdx >= nNews) {
-      tdbPagerInsertFreePage(pBt->pPager, TDB_PAGE_PGNO(pOlds[pageIdx]), pTxn);
+      tdbPagerInsertFreePage(pBt->pPager, pOlds[pageIdx], pTxn);
     }
     tdbPagerReturnPage(pBt->pPager, pOlds[pageIdx], pTxn);
   }
@@ -1319,10 +1320,6 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader,
           return -1;
         }
 
-        if (pDecoder->ofps) {
-          taosArrayPush(pDecoder->ofps, &pgno);
-        }
-
         ofpCell = tdbPageGetCell(ofp, 0);
 
         if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
@@ -1529,8 +1526,8 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
 
   if (pPage->vLen == TDB_VARIANT_LEN) {
     if (!leaf) {
-      tdbError("tdb/btree-cell-size: not a leaf page.");
-      return -1;
+      tdbError("tdb/btree-cell-size: not a leaf page:%p, pgno:%" PRIu32 ".", pPage, TDB_PAGE_PGNO(pPage));
+      // return -1;
     }
     nHeader += tdbGetVarInt(pCell + nHeader, &vLen);
   } else if (leaf) {
@@ -1570,8 +1567,27 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
           bytes = ofp->maxLocal - sizeof(SPgno);
         }
 
+        SPgno origPgno = pgno;
         memcpy(&pgno, ofpCell + bytes, sizeof(pgno));
 
+        ret = tdbPagerWrite(pBt->pPager, ofp);
+        if (ret < 0) {
+          tdbError("failed to write page since %s", terrstr());
+          return -1;
+        }
+        // tdbPageDropCell(ofp, 0, pTxn, pBt);
+        // tdbPageZero(ofp, sizeof(SLeafHdr), tdbBtreeCellSize);
+        // tdbPageZero(ofp, sizeof(SIntHdr), tdbBtreeCellSize);
+        // SIntHdr *pIntHdr = (SIntHdr *)(ofp->pData);
+        //  pIntHdr->flags = TDB_FLAG_ADD(0, TDB_BTREE_OVFL);
+        // pIntHdr->pgno = 0;
+        // ofp->pPager = NULL;
+
+        tdbPagerInsertFreePage(pBt->pPager, ofp, pTxn);
+
+        // printf("tdb recycle, pTxn: %p, pgno:%u\n", pTxn, pgno);
+        tdbTrace("tdb recycle, pTxn: %p, pgno:%u", pTxn, origPgno);
+
         tdbPagerReturnPage(pPage->pPager, ofp, pTxn);
 
         nLeft -= bytes;
@@ -1991,6 +2007,11 @@ static int tdbBtcMoveDownward(SBTC *pBtc) {
     return -1;
   }
 
+  if (TDB_BTREE_PAGE_IS_OVFL(pBtc->pPage)) {
+    tdbError("tdb/btc-move-downward: should not be a ovfl page here.");
+    return -1;
+  }
+
   if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) {
     pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx);
     pgno = ((SPgno *)pCell)[0];
@@ -2081,14 +2102,6 @@ int tdbBtcDelete(SBTC *pBtc) {
 
   tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt);
 
-  // recycle ofps if any
-  if (pBtc->coder.ofps) {
-    for (int i = 0; i < TARRAY_SIZE(pBtc->coder.ofps); ++i) {
-      SPgno *pgno = taosArrayGet(pBtc->coder.ofps, i);
-      // tdbPagerInsertFreePage(pBtc->pBt->pPager, *pgno, pBtc->pTxn);
-    }
-  }
-
   // update interior page or do balance
   if (idx == nCells - 1) {
     if (idx) {
@@ -2384,10 +2397,6 @@ int tdbBtcClose(SBTC *pBtc) {
     tdbTxnClose(pBtc->pTxn);
   }
 
-  if (pBtc->coder.ofps) {
-    taosArrayDestroy(pBtc->coder.ofps);
-  }
-
   return 0;
 }
 
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 4e29ca45ca..62702cbf40 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -292,7 +292,23 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) {
   */
   return 0;
 }
+/*
+int tdbPagerCancelDirty(SPager *pPager, SPage *pPage, TXN *pTxn) {
+  SRBTreeNode *pNode = tRBTreeGet(&pPager->rbt, (SRBTreeNode *)pPage);
+  if (pNode) {
+    pPage->isDirty = 0;
 
+    tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
+    if (pTxn->jPageSet) {
+      hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage)));
+    }
+
+    tdbPCacheRelease(pPager->pCache, pPage, pTxn);
+  }
+
+  return 0;
+}
+*/
 int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
   SPage *pPage;
   int    ret;
@@ -700,8 +716,9 @@ void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) {
   //        TDB_PAGE_PGNO(pPage), pPage);
 }
 
-int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) {
-  int code = 0;
+int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
+  int   code = 0;
+  SPgno pgno = TDB_PAGE_PGNO(pPage);
 
   // tdbError("tdb/insert-free-page: tbc get page: %d.", pgno);
   code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn);
@@ -710,6 +727,8 @@ int tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn) {
     return -1;
   }
 
+  pPage->pPager = NULL;
+
   return code;
 }
 
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index 7b08da4ca8..879e6a3a49 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -199,7 +199,7 @@ int  tdbPagerAbort(SPager *pPager, TXN *pTxn);
 int  tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg,
                        TXN *pTxn);
 void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
-int  tdbPagerInsertFreePage(SPager *pPager, SPgno pgno, TXN *pTxn);
+int  tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn);
 // int  tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
 int tdbPagerRestoreJournals(SPager *pPager);
 int tdbPagerRollback(SPager *pPager);
diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp
index 2f2dd0659b..b4391c4a8c 100644
--- a/source/libs/tdb/test/tdbPageRecycleTest.cpp
+++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp
@@ -690,3 +690,100 @@ TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_again) {
   insertOfp();
   system("ls -l ./tdb");
 }
+
+// TEST(TdbPageRecycleTest, DISABLED_recycly_seq_insert_ofp_nocommit) {
+TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) {
+  clearDb("tdb");
+  insertOfp();
+  system("ls -l ./tdb");
+
+  // open Env
+  int       ret = 0;
+  int const pageSize = 4096;
+  int const pageNum = 64;
+  TDB      *pEnv = openEnv("tdb", pageSize, pageNum);
+  GTEST_ASSERT_NE(pEnv, nullptr);
+
+  // open db
+  TTB          *pDb = NULL;
+  tdb_cmpr_fn_t compFunc = tKeyCmpr;
+  ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // open the pool
+  SPoolMem *pPool = openPool();
+
+  // start a transaction
+  TXN *txn;
+
+  tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+
+  {  // delete the data
+    char const *key = "key123456789";
+    ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn);
+    GTEST_ASSERT_EQ(ret, 0);
+  }
+
+  // 1, insert nData kv
+  {
+    int     nData = nDataConst;
+    char    key[64];
+    char    val[(4083 - 4 - 3 - 2) + 1];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+    int64_t poolLimit = 4096;             // 1M pool limit
+    /*
+    SPoolMem *pPool;
+
+    // open the pool
+    pPool = openPool();
+
+    // start a transaction
+    tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+    */
+    for (int iData = 0; iData < nData; ++iData) {
+      sprintf(key, "key%03d", iData);
+      sprintf(val, "value%03d", iData);
+
+      ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn);
+      GTEST_ASSERT_EQ(ret, 0);
+      // if pool is full, commit the transaction and start a new one
+      if (pPool->size >= poolLimit) {
+        // commit current transaction
+        tdbCommit(pEnv, txn);
+        tdbPostCommit(pEnv, txn);
+
+        // start a new transaction
+        clearPool(pPool);
+
+        tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+      }
+    }
+  }
+
+  /*
+  // generate value payload
+  // char val[((4083 - 4 - 3 - 2) + 1) * 100];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
+  char val[32605];
+  int  valLen = sizeof(val) / sizeof(val[0]);
+  generateBigVal(val, valLen);
+
+  // insert the generated big data
+  // char const *key = "key1";
+  char const *key = "key123456789";
+  ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn);
+  GTEST_ASSERT_EQ(ret, 0);
+  */
+  // commit current transaction
+  tdbCommit(pEnv, txn);
+  tdbPostCommit(pEnv, txn);
+
+  closePool(pPool);
+
+  // Close a database
+  tdbTbClose(pDb);
+
+  // Close Env
+  ret = tdbClose(pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  system("ls -l ./tdb");
+}

From 8e491c307fec09033b0ceb7b0cda68f7be58de00 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 30 Jun 2023 08:32:39 +0800
Subject: [PATCH 14/17] tdb/recyle-ofp: nullize ofp's pager to mark
 uninitailized

---
 source/libs/tdb/src/db/tdbBtree.c | 37 ++++++++++++++++++++++---------
 source/libs/tdb/src/db/tdbPager.c |  5 +++--
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 6921a26f19..382c25bfd5 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -1320,6 +1320,10 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader,
           return -1;
         }
 
+        if (pDecoder->ofps) {
+          taosArrayPush(pDecoder->ofps, &ofp);
+        }
+
         ofpCell = tdbPageGetCell(ofp, 0);
 
         if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
@@ -1354,11 +1358,16 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader,
       int lastKeyPageSpace = 0;
       // load left key & val to ovpages
       while (pgno != 0) {
+        tdbTrace("tdb decode-ofp, pTxn: %p, pgno:%u by cell:%p", pTxn, pgno, pCell);
         ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
         if (ret < 0) {
           return -1;
         }
 
+        if (pDecoder->ofps) {
+          taosArrayPush(pDecoder->ofps, &ofp);
+        }
+
         ofpCell = tdbPageGetCell(ofp, 0);
 
         int lastKeyPage = 0;
@@ -1567,27 +1576,21 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
           bytes = ofp->maxLocal - sizeof(SPgno);
         }
 
-        SPgno origPgno = pgno;
+        // SPgno origPgno = pgno;
         memcpy(&pgno, ofpCell + bytes, sizeof(pgno));
-
+        /*
         ret = tdbPagerWrite(pBt->pPager, ofp);
         if (ret < 0) {
           tdbError("failed to write page since %s", terrstr());
           return -1;
         }
-        // tdbPageDropCell(ofp, 0, pTxn, pBt);
-        // tdbPageZero(ofp, sizeof(SLeafHdr), tdbBtreeCellSize);
-        // tdbPageZero(ofp, sizeof(SIntHdr), tdbBtreeCellSize);
+        tdbPageDropCell(ofp, 0, pTxn, pBt);
+        */
         // SIntHdr *pIntHdr = (SIntHdr *)(ofp->pData);
         //  pIntHdr->flags = TDB_FLAG_ADD(0, TDB_BTREE_OVFL);
         // pIntHdr->pgno = 0;
         // ofp->pPager = NULL;
 
-        tdbPagerInsertFreePage(pBt->pPager, ofp, pTxn);
-
-        // printf("tdb recycle, pTxn: %p, pgno:%u\n", pTxn, pgno);
-        tdbTrace("tdb recycle, pTxn: %p, pgno:%u", pTxn, origPgno);
-
         tdbPagerReturnPage(pPage->pPager, ofp, pTxn);
 
         nLeft -= bytes;
@@ -2100,6 +2103,9 @@ int tdbBtcDelete(SBTC *pBtc) {
     return -1;
   }
 
+  // btc.coder.ofps = taosArrayInit(8, sizeof(SPgno));
+  pBtc->coder.ofps = taosArrayInit(8, sizeof(SPage *));
+
   tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt);
 
   // update interior page or do balance
@@ -2155,6 +2161,17 @@ int tdbBtcDelete(SBTC *pBtc) {
     }
   }
 
+  SArray *ofps = pBtc->coder.ofps;
+  if (ofps) {
+    for (int i = 0; i < TARRAY_SIZE(ofps); ++i) {
+      SPage *ofp = *(SPage **)taosArrayGet(ofps, i);
+      // tdbPagerInsertFreePage(pBtc->pBt->pPager, ofp, pBtc->pTxn);
+    }
+
+    taosArrayDestroy(ofps);
+    pBtc->coder.ofps = NULL;
+  }
+
   return 0;
 }
 
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 62702cbf40..5bfcdfa344 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -720,7 +720,8 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
   int   code = 0;
   SPgno pgno = TDB_PAGE_PGNO(pPage);
 
-  // tdbError("tdb/insert-free-page: tbc get page: %d.", pgno);
+  // memset(pPage->pData, 0, pPage->pageSize);
+  tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno);
   code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn);
   if (code < 0) {
     tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code);
@@ -763,7 +764,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) {
   }
 
   *pPgno = *(SPgno *)pKey;
-  // tdbError("tdb/remove-free-page: tbc get page: %d.", *pPgno);
+  tdbTrace("tdb/remove-free-page: tbc get page: %d.", *pPgno);
 
   code = tdbTbcDelete(pCur);
   if (code < 0) {

From f89b43b64ca16e966f05b4da8e99d00c798de9af Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Fri, 30 Jun 2023 10:05:17 +0800
Subject: [PATCH 15/17] tdb/ofp-recycle: new ofps list with pager

---
 source/libs/tdb/src/db/tdbBtree.c           | 40 +++++++++++++++------
 source/libs/tdb/src/db/tdbPager.c           |  2 ++
 source/libs/tdb/src/inc/tdbInt.h            |  1 +
 source/libs/tdb/test/tdbPageRecycleTest.cpp | 21 -----------
 4 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 382c25bfd5..8ffb5cd43e 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -234,7 +234,9 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) {
 
   tdbBtcOpen(&btc, pBt, pTxn);
   /*
-  btc.coder.ofps = taosArrayInit(8, sizeof(SPgno));
+  btc.coder.ofps = taosArrayInit(8, sizeof(SPage *));
+  // btc.coder.ofps = taosArrayInit(8, sizeof(SPgno));
+  //pBtc->coder.ofps = taosArrayInit(8, sizeof(SPage *));
   */
   tdbTrace("tdb delete, btc: %p, pTxn: %p", &btc, pTxn);
 
@@ -256,7 +258,18 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) {
     tdbBtcClose(&btc);
     return -1;
   }
+  /*
+  SArray *ofps = btc.coder.ofps;
+  if (ofps) {
+    for (int i = 0; i < TARRAY_SIZE(ofps); ++i) {
+      SPage *ofp = *(SPage **)taosArrayGet(ofps, i);
+      tdbPagerInsertFreePage(btc.pBt->pPager, ofp, btc.pTxn);
+    }
 
+    taosArrayDestroy(ofps);
+    btc.coder.ofps = NULL;
+  }
+  */
   tdbBtcClose(&btc);
   return 0;
 }
@@ -1319,11 +1332,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader,
         if (ret < 0) {
           return -1;
         }
-
+        /*
         if (pDecoder->ofps) {
           taosArrayPush(pDecoder->ofps, &ofp);
         }
-
+        */
         ofpCell = tdbPageGetCell(ofp, 0);
 
         if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
@@ -1363,11 +1376,11 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader,
         if (ret < 0) {
           return -1;
         }
-
+        /*
         if (pDecoder->ofps) {
           taosArrayPush(pDecoder->ofps, &ofp);
         }
-
+        */
         ofpCell = tdbPageGetCell(ofp, 0);
 
         int lastKeyPage = 0;
@@ -1578,12 +1591,13 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
 
         // SPgno origPgno = pgno;
         memcpy(&pgno, ofpCell + bytes, sizeof(pgno));
-        /*
+
         ret = tdbPagerWrite(pBt->pPager, ofp);
         if (ret < 0) {
           tdbError("failed to write page since %s", terrstr());
           return -1;
         }
+        /*
         tdbPageDropCell(ofp, 0, pTxn, pBt);
         */
         // SIntHdr *pIntHdr = (SIntHdr *)(ofp->pData);
@@ -1591,6 +1605,11 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
         // pIntHdr->pgno = 0;
         // ofp->pPager = NULL;
 
+        SArray *ofps = pPage->pPager->ofps;
+        if (ofps) {
+          taosArrayPush(ofps, &ofp);
+        }
+
         tdbPagerReturnPage(pPage->pPager, ofp, pTxn);
 
         nLeft -= bytes;
@@ -2103,8 +2122,7 @@ int tdbBtcDelete(SBTC *pBtc) {
     return -1;
   }
 
-  // btc.coder.ofps = taosArrayInit(8, sizeof(SPgno));
-  pBtc->coder.ofps = taosArrayInit(8, sizeof(SPage *));
+  pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *));
 
   tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt);
 
@@ -2161,15 +2179,15 @@ int tdbBtcDelete(SBTC *pBtc) {
     }
   }
 
-  SArray *ofps = pBtc->coder.ofps;
+  SArray *ofps = pBtc->pPage->pPager->ofps;
   if (ofps) {
     for (int i = 0; i < TARRAY_SIZE(ofps); ++i) {
       SPage *ofp = *(SPage **)taosArrayGet(ofps, i);
-      // tdbPagerInsertFreePage(pBtc->pBt->pPager, ofp, pBtc->pTxn);
+      tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn);
     }
 
     taosArrayDestroy(ofps);
-    pBtc->coder.ofps = NULL;
+    pBtc->pPage->pPager->ofps = NULL;
   }
 
   return 0;
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 5bfcdfa344..469416cd1b 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -722,6 +722,7 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
 
   // memset(pPage->pData, 0, pPage->pageSize);
   tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno);
+  // printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno);
   code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn);
   if (code < 0) {
     tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code);
@@ -765,6 +766,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) {
 
   *pPgno = *(SPgno *)pKey;
   tdbTrace("tdb/remove-free-page: tbc get page: %d.", *pPgno);
+  // printf("tdb/remove-free-page: tbc get page: %d.\n", *pPgno);
 
   code = tdbTbcDelete(pCur);
   if (code < 0) {
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index 879e6a3a49..8defe54868 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -407,6 +407,7 @@ struct SPager {
   SRBTree rbt;
   // u8        inTran;
   TXN    *pActiveTxn;
+  SArray *ofps;
   SPager *pNext;      // used by TDB
   SPager *pHashNext;  // used by TDB
 #ifdef USE_MAINDB
diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp
index b4391c4a8c..40208f5070 100644
--- a/source/libs/tdb/test/tdbPageRecycleTest.cpp
+++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp
@@ -730,15 +730,7 @@ TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) {
     char    key[64];
     char    val[(4083 - 4 - 3 - 2) + 1];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
     int64_t poolLimit = 4096;             // 1M pool limit
-    /*
-    SPoolMem *pPool;
 
-    // open the pool
-    pPool = openPool();
-
-    // start a transaction
-    tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
-    */
     for (int iData = 0; iData < nData; ++iData) {
       sprintf(key, "key%03d", iData);
       sprintf(val, "value%03d", iData);
@@ -759,19 +751,6 @@ TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) {
     }
   }
 
-  /*
-  // generate value payload
-  // char val[((4083 - 4 - 3 - 2) + 1) * 100];  // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4)
-  char val[32605];
-  int  valLen = sizeof(val) / sizeof(val[0]);
-  generateBigVal(val, valLen);
-
-  // insert the generated big data
-  // char const *key = "key1";
-  char const *key = "key123456789";
-  ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn);
-  GTEST_ASSERT_EQ(ret, 0);
-  */
   // commit current transaction
   tdbCommit(pEnv, txn);
   tdbPostCommit(pEnv, txn);

From c66524d87bc7f6f4bc7bee29ae4b5fe7ef42ecbc Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Wed, 5 Jul 2023 15:33:37 +0800
Subject: [PATCH 16/17] tdb/ofp: recycle ofp cell on parent page

---
 source/libs/tdb/src/db/tdbBtree.c           | 75 +++++++++++++++++----
 source/libs/tdb/test/tdbPageRecycleTest.cpp | 67 ++++++++++++++++++
 2 files changed, 129 insertions(+), 13 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 8ffb5cd43e..08e61c2272 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -608,7 +608,30 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
     for (int i = 0; i < nOlds; i++) {
       nCells = TDB_PAGE_TOTAL_CELLS(pParent);
       if (sIdx < nCells) {
+        bool destroyOfps = false;
+        if (!childNotLeaf) {
+          if (!pParent->pPager->ofps) {
+            pParent->pPager->ofps = taosArrayInit(8, sizeof(SPage *));
+            destroyOfps = true;
+          }
+        }
+
         tdbPageDropCell(pParent, sIdx, pTxn, pBt);
+
+        if (!childNotLeaf) {
+          SArray *ofps = pParent->pPager->ofps;
+          if (ofps) {
+            for (int i = 0; i < TARRAY_SIZE(ofps); ++i) {
+              SPage *ofp = *(SPage **)taosArrayGet(ofps, i);
+              tdbPagerInsertFreePage(pParent->pPager, ofp, pTxn);
+            }
+
+            if (destroyOfps) {
+              taosArrayDestroy(ofps);
+              pParent->pPager->ofps = NULL;
+            }
+          }
+        }
       } else {
         ((SIntHdr *)pParent->pData)->pgno = 0;
       }
@@ -1372,6 +1395,7 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader,
       // load left key & val to ovpages
       while (pgno != 0) {
         tdbTrace("tdb decode-ofp, pTxn: %p, pgno:%u by cell:%p", pTxn, pgno, pCell);
+        // printf("tdb decode-ofp, pTxn: %p, pgno:%u by cell:%p\n", pTxn, pgno, pCell);
         ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
         if (ret < 0) {
           return -1;
@@ -2122,10 +2146,27 @@ int tdbBtcDelete(SBTC *pBtc) {
     return -1;
   }
 
-  pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *));
+  bool destroyOfps = false;
+  if (!pBtc->pPage->pPager->ofps) {
+    pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *));
+    destroyOfps = true;
+  }
 
   tdbPageDropCell(pBtc->pPage, idx, pBtc->pTxn, pBtc->pBt);
 
+  SArray *ofps = pBtc->pPage->pPager->ofps;
+  if (ofps) {
+    for (int i = 0; i < TARRAY_SIZE(ofps); ++i) {
+      SPage *ofp = *(SPage **)taosArrayGet(ofps, i);
+      tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn);
+    }
+
+    if (destroyOfps) {
+      taosArrayDestroy(ofps);
+      pBtc->pPage->pPager->ofps = NULL;
+    }
+  }
+
   // update interior page or do balance
   if (idx == nCells - 1) {
     if (idx) {
@@ -2179,17 +2220,6 @@ int tdbBtcDelete(SBTC *pBtc) {
     }
   }
 
-  SArray *ofps = pBtc->pPage->pPager->ofps;
-  if (ofps) {
-    for (int i = 0; i < TARRAY_SIZE(ofps); ++i) {
-      SPage *ofp = *(SPage **)taosArrayGet(ofps, i);
-      tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn);
-    }
-
-    taosArrayDestroy(ofps);
-    pBtc->pPage->pPager->ofps = NULL;
-  }
-
   return 0;
 }
 
@@ -2250,7 +2280,13 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int
     tdbError("tdb/btc-upsert: page insert/update cell failed with ret: %d.", ret);
     return -1;
   }
-
+  /*
+  bool destroyOfps = false;
+  if (!pBtc->pPage->pPager->ofps) {
+    pBtc->pPage->pPager->ofps = taosArrayInit(8, sizeof(SPage *));
+    destroyOfps = true;
+  }
+  */
   // check balance
   if (pBtc->pPage->nOverflow > 0) {
     ret = tdbBtreeBalance(pBtc);
@@ -2259,7 +2295,20 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int
       return -1;
     }
   }
+  /*
+  SArray *ofps = pBtc->pPage->pPager->ofps;
+  if (ofps) {
+    for (int i = 0; i < TARRAY_SIZE(ofps); ++i) {
+      SPage *ofp = *(SPage **)taosArrayGet(ofps, i);
+      tdbPagerInsertFreePage(pBtc->pPage->pPager, ofp, pBtc->pTxn);
+    }
 
+    if (destroyOfps) {
+      taosArrayDestroy(ofps);
+      pBtc->pPage->pPager->ofps = NULL;
+    }
+  }
+  */
   return 0;
 }
 
diff --git a/source/libs/tdb/test/tdbPageRecycleTest.cpp b/source/libs/tdb/test/tdbPageRecycleTest.cpp
index 40208f5070..4d7b314917 100644
--- a/source/libs/tdb/test/tdbPageRecycleTest.cpp
+++ b/source/libs/tdb/test/tdbPageRecycleTest.cpp
@@ -766,3 +766,70 @@ TEST(TdbPageRecycleTest, recycly_seq_insert_ofp_nocommit) {
 
   system("ls -l ./tdb");
 }
+
+// TEST(TdbPageRecycleTest, DISABLED_recycly_delete_interior_ofp_nocommit) {
+TEST(TdbPageRecycleTest, recycly_delete_interior_ofp_nocommit) {
+  clearDb("tdb");
+
+  // open Env
+  int       ret = 0;
+  int const pageSize = 4096;
+  int const pageNum = 64;
+  TDB      *pEnv = openEnv("tdb", pageSize, pageNum);
+  GTEST_ASSERT_NE(pEnv, nullptr);
+
+  // open db
+  TTB          *pDb = NULL;
+  tdb_cmpr_fn_t compFunc = NULL;  // tKeyCmpr;
+  ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  // open the pool
+  SPoolMem *pPool = openPool();
+
+  // start a transaction
+  TXN *txn;
+
+  tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+
+  char key[1024] = {0};
+  int  count = sizeof(key) / sizeof(key[0]);
+  for (int i = 0; i < count - 1; ++i) {
+    key[i] = 'a';
+  }
+
+  // insert n ofp keys to form 2-layer btree
+  {
+    for (int i = 0; i < 7; ++i) {
+      // sprintf(&key[count - 2], "%c", i);
+      key[count - 2] = '0' + i;
+
+      ret = tdbTbInsert(pDb, key, count, NULL, NULL, txn);
+      GTEST_ASSERT_EQ(ret, 0);
+    }
+  }
+  /*
+  // delete one interior key
+  {
+    sprintf(&key[count - 2], "%c", 2);
+    key[count - 2] = '0' + 2;
+
+    ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn);
+    GTEST_ASSERT_EQ(ret, 0);
+  }
+  */
+  // commit current transaction
+  tdbCommit(pEnv, txn);
+  tdbPostCommit(pEnv, txn);
+
+  closePool(pPool);
+
+  // Close a database
+  tdbTbClose(pDb);
+
+  // Close Env
+  ret = tdbClose(pEnv);
+  GTEST_ASSERT_EQ(ret, 0);
+
+  system("ls -l ./tdb");
+}

From 4f29e14963e4b6cc55e0a05d2d38a0ddb7e5ddc5 Mon Sep 17 00:00:00 2001
From: Minglei Jin <mljin@taosdata.com>
Date: Thu, 6 Jul 2023 08:11:46 +0800
Subject: [PATCH 17/17] tsim/sma/drop_sma: sleep 1s before creating db to avoid
 in dropping

---
 tests/script/tsim/sma/drop_sma.sim | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/script/tsim/sma/drop_sma.sim b/tests/script/tsim/sma/drop_sma.sim
index 0d2712f8db..8fd8ebdcfd 100644
--- a/tests/script/tsim/sma/drop_sma.sim
+++ b/tests/script/tsim/sma/drop_sma.sim
@@ -129,6 +129,7 @@ sql DROP INDEX sma_index_3 ;
 
 print ========== step8
 sql drop database if exists db;
+sleep 2000
 sql create database db duration 300;
 sql use db;
 sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);