From 08ed99b0692cef6d791887dd2351f6bb8f1d1523 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Sat, 2 Apr 2022 02:56:09 +0000
Subject: [PATCH 01/16] more TDB

---
 source/libs/tdb/src/btree/tdbBtreeBalance.c | 14 ------
 source/libs/tdb/src/btree/tdbBtreeCommon.c  | 14 ------
 source/libs/tdb/src/btree/tdbBtreeDelete.c  | 14 ------
 source/libs/tdb/src/btree/tdbBtreeInsert.c  | 14 ------
 source/libs/tdb/src/btree/tdbBtreeInt.h     | 27 ------------
 source/libs/tdb/src/btree/tdbBtreeOpen.c    | 14 ------
 source/libs/tdb/src/db/tdbPCache.c          | 47 ++++++++++++---------
 source/libs/tdb/src/inc/tdbPCache.h         |  1 +
 source/libs/tdb/src/inc/tdbTxn.h            |  1 +
 9 files changed, 29 insertions(+), 117 deletions(-)
 delete mode 100644 source/libs/tdb/src/btree/tdbBtreeBalance.c
 delete mode 100644 source/libs/tdb/src/btree/tdbBtreeCommon.c
 delete mode 100644 source/libs/tdb/src/btree/tdbBtreeDelete.c
 delete mode 100644 source/libs/tdb/src/btree/tdbBtreeInsert.c
 delete mode 100644 source/libs/tdb/src/btree/tdbBtreeInt.h
 delete mode 100644 source/libs/tdb/src/btree/tdbBtreeOpen.c

diff --git a/source/libs/tdb/src/btree/tdbBtreeBalance.c b/source/libs/tdb/src/btree/tdbBtreeBalance.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/libs/tdb/src/btree/tdbBtreeBalance.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/libs/tdb/src/btree/tdbBtreeCommon.c b/source/libs/tdb/src/btree/tdbBtreeCommon.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/libs/tdb/src/btree/tdbBtreeCommon.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/libs/tdb/src/btree/tdbBtreeDelete.c b/source/libs/tdb/src/btree/tdbBtreeDelete.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/libs/tdb/src/btree/tdbBtreeDelete.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/libs/tdb/src/btree/tdbBtreeInsert.c b/source/libs/tdb/src/btree/tdbBtreeInsert.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/libs/tdb/src/btree/tdbBtreeInsert.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/libs/tdb/src/btree/tdbBtreeInt.h b/source/libs/tdb/src/btree/tdbBtreeInt.h
deleted file mode 100644
index b8a935a614..0000000000
--- a/source/libs/tdb/src/btree/tdbBtreeInt.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _TDB_BTREE_INT_H_
-#define _TDB_BTREE_INT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /*_TDB_BTREE_INT_H_*/
\ No newline at end of file
diff --git a/source/libs/tdb/src/btree/tdbBtreeOpen.c b/source/libs/tdb/src/btree/tdbBtreeOpen.c
deleted file mode 100644
index 6dea4a4e57..0000000000
--- a/source/libs/tdb/src/btree/tdbBtreeOpen.c
+++ /dev/null
@@ -1,14 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
\ No newline at end of file
diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c
index d886cfd889..024143c8b2 100644
--- a/source/libs/tdb/src/db/tdbPCache.c
+++ b/source/libs/tdb/src/db/tdbPCache.c
@@ -18,6 +18,7 @@ struct SPCache {
   int         pageSize;
   int         cacheSize;
   tdb_mutex_t mutex;
+  SPage      *pList;
   int         nFree;
   SPage      *pFree;
   int         nPage;
@@ -35,16 +36,17 @@ struct SPCache {
 #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL)
 
 static int    tdbPCacheOpenImpl(SPCache *pCache);
-static void   tdbPCacheInitLock(SPCache *pCache);
-static void   tdbPCacheClearLock(SPCache *pCache);
-static void   tdbPCacheLock(SPCache *pCache);
-static void   tdbPCacheUnlock(SPCache *pCache);
-static bool   tdbPCacheLocked(SPCache *pCache);
 static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage);
 static void   tdbPCachePinPage(SPCache *pCache, SPage *pPage);
 static void   tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage);
 static void   tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage);
 static void   tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage);
+static int    tdbPCacheCloseImpl(SPCache *pCache);
+
+static void tdbPCacheInitLock(SPCache *pCache) { tdbMutexInit(&(pCache->mutex), NULL); }
+static void tdbPCacheDestroyLock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mutex)); }
+static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); }
+static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); }
 
 int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
   SPCache *pCache;
@@ -69,7 +71,10 @@ int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache) {
 }
 
 int tdbPCacheClose(SPCache *pCache) {
-  /* TODO */
+  if (pCache) {
+    tdbPCacheCloseImpl(pCache);
+    tdbOsFree(pCache);
+  }
   return 0;
 }
 
@@ -99,19 +104,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage) {
   }
 }
 
-static void tdbPCacheInitLock(SPCache *pCache) { tdbMutexInit(&(pCache->mutex), NULL); }
-
-static void tdbPCacheClearLock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mutex)); }
-
-static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); }
-
-static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); }
-
-static bool tdbPCacheLocked(SPCache *pCache) {
-  assert(0);
-  // TODO
-  return true;
-}
+int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; }
 
 static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) {
   SPage *pPage;
@@ -268,4 +261,18 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
   return 0;
 }
 
-int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; }
+static int tdbPCacheDestroyPage(SPage *pPage) {
+  // TODO
+  return 0;
+}
+
+static int tdbPCacheCloseImpl(SPCache *pCache) {
+  SPage *pPage;
+
+  for (pPage = pCache->pList; pPage; pPage = pCache->pList) {
+    pCache->pList = pPage->pCacheNext;
+    tdbPCacheDestroyPage(pPage);
+  }
+
+  tdbPCacheDestroyLock(pCache);
+}
diff --git a/source/libs/tdb/src/inc/tdbPCache.h b/source/libs/tdb/src/inc/tdbPCache.h
index f71d34ab53..eab5d7582a 100644
--- a/source/libs/tdb/src/inc/tdbPCache.h
+++ b/source/libs/tdb/src/inc/tdbPCache.h
@@ -25,6 +25,7 @@ extern "C" {
   u8      isLocalPage;  \
   u8      isDirty;      \
   i32     nRef;         \
+  SPage  *pCacheNext;   \
   SPage  *pFreeNext;    \
   SPage  *pHashNext;    \
   SPage  *pLruNext;     \
diff --git a/source/libs/tdb/src/inc/tdbTxn.h b/source/libs/tdb/src/inc/tdbTxn.h
index 0be2dad3c2..d979249f03 100644
--- a/source/libs/tdb/src/inc/tdbTxn.h
+++ b/source/libs/tdb/src/inc/tdbTxn.h
@@ -25,6 +25,7 @@ typedef struct STxn TXN;
 struct STxn {
   u64 txnId;
   void *(*xMalloc)(void *, int);
+  void (*xFree)(void *, void *);
   void *xArg;
 };
 

From 9dea86249cb1fce10aa80ec1cc9fa44707d4b205 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Sat, 2 Apr 2022 05:16:32 +0000
Subject: [PATCH 02/16] refact

---
 source/libs/tdb/src/db/tdbPCache.c  | 23 ++++++++---------------
 source/libs/tdb/src/db/tdbPager.c   |  4 ++--
 source/libs/tdb/src/inc/tdbPCache.h |  2 +-
 3 files changed, 11 insertions(+), 18 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c
index 024143c8b2..5d16aeaa29 100644
--- a/source/libs/tdb/src/db/tdbPCache.c
+++ b/source/libs/tdb/src/db/tdbPCache.c
@@ -36,7 +36,7 @@ struct SPCache {
 #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL)
 
 static int    tdbPCacheOpenImpl(SPCache *pCache);
-static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage);
+static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid);
 static void   tdbPCachePinPage(SPCache *pCache, SPage *pPage);
 static void   tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage);
 static void   tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage);
@@ -78,12 +78,12 @@ int tdbPCacheClose(SPCache *pCache) {
   return 0;
 }
 
-SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) {
+SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid) {
   SPage *pPage;
 
   tdbPCacheLock(pCache);
 
-  pPage = tdbPCacheFetchImpl(pCache, pPgid, alcNewPage);
+  pPage = tdbPCacheFetchImpl(pCache, pPgid);
   if (pPage) {
     TDB_REF_PAGE(pPage);
   }
@@ -106,7 +106,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage) {
 
 int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; }
 
-static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNewPage) {
+static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid) {
   SPage *pPage;
 
   // 1. Search the hash table
@@ -116,12 +116,10 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, bool alcNe
     pPage = pPage->pHashNext;
   }
 
-  if (pPage || !alcNewPage) {
-    if (pPage) {
-      tdbPCachePinPage(pCache, pPage);
-    }
-    return pPage;
+  if (pPage) {
+    tdbPCachePinPage(pCache, pPage);
   }
+  return pPage;
 
   // 2. Try to allocate a new page from the free list
   if (pCache->pFree) {
@@ -261,17 +259,12 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
   return 0;
 }
 
-static int tdbPCacheDestroyPage(SPage *pPage) {
-  // TODO
-  return 0;
-}
-
 static int tdbPCacheCloseImpl(SPCache *pCache) {
   SPage *pPage;
 
   for (pPage = pCache->pList; pPage; pPage = pCache->pList) {
     pCache->pList = pPage->pCacheNext;
-    tdbPCacheDestroyPage(pPage);
+    tdbPageDestroy(pPage, NULL, NULL);
   }
 
   tdbPCacheDestroyLock(pCache);
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 2bc40a6aad..1953419bf4 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -227,7 +227,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage
   // Fetch a page container from the page cache
   memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
   pgid.pgno = pgno;
-  pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1);
+  pPage = tdbPCacheFetch(pPager->pCache, &pgid);
   if (pPage == NULL) {
     return -1;
   }
@@ -263,7 +263,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
   // Fetch a page container from the page cache
   memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
   pgid.pgno = *ppgno;
-  pPage = tdbPCacheFetch(pPager->pCache, &pgid, 1);
+  pPage = tdbPCacheFetch(pPager->pCache, &pgid);
   if (pPage == NULL) {
     return -1;
   }
diff --git a/source/libs/tdb/src/inc/tdbPCache.h b/source/libs/tdb/src/inc/tdbPCache.h
index eab5d7582a..c5eba88545 100644
--- a/source/libs/tdb/src/inc/tdbPCache.h
+++ b/source/libs/tdb/src/inc/tdbPCache.h
@@ -48,7 +48,7 @@ extern "C" {
 
 int    tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
 int    tdbPCacheClose(SPCache *pCache);
-SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, bool alcNewPage);
+SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid);
 void   tdbPCacheRelease(SPCache *pCache, SPage *pPage);
 int    tdbPCacheGetPageSize(SPCache *pCache);
 

From f9e699bea5a3fdf42d57a000e9d0a0afea845496 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Sat, 2 Apr 2022 06:23:04 +0000
Subject: [PATCH 03/16] more TDB

---
 source/libs/tdb/src/db/tdbEnv.c     | 10 +++---
 source/libs/tdb/src/db/tdbPCache.c  | 53 +++++++++++++++++++++--------
 source/libs/tdb/src/db/tdbPager.c   |  8 ++---
 source/libs/tdb/src/inc/tdbEnv.h    |  6 ++--
 source/libs/tdb/src/inc/tdbInt.h    |  7 ++++
 source/libs/tdb/src/inc/tdbPCache.h |  4 +--
 source/libs/tdb/src/inc/tdbPager.h  |  4 +--
 source/libs/tdb/src/inc/tdbTxn.h    |  9 -----
 source/libs/tdb/test/tdbTest.cpp    |  6 ++--
 9 files changed, 63 insertions(+), 44 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c
index 06d37df653..779435861e 100644
--- a/source/libs/tdb/src/db/tdbEnv.c
+++ b/source/libs/tdb/src/db/tdbEnv.c
@@ -73,12 +73,12 @@ int tdbEnvClose(TENV *pEnv) {
   return 0;
 }
 
-int tdbBegin(TENV *pEnv) {
+int tdbBegin(TENV *pEnv, TXN *pTxn) {
   SPager *pPager;
   int     ret;
 
   for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
-    ret = tdbPagerBegin(pPager);
+    ret = tdbPagerBegin(pPager, pTxn);
     if (ret < 0) {
       ASSERT(0);
       return -1;
@@ -88,12 +88,12 @@ int tdbBegin(TENV *pEnv) {
   return 0;
 }
 
-int tdbCommit(TENV *pEnv) {
+int tdbCommit(TENV *pEnv, TXN *pTxn) {
   SPager *pPager;
   int     ret;
 
   for (pPager = pEnv->pgrList; pPager; pPager = pPager->pNext) {
-    ret = tdbPagerCommit(pPager);
+    ret = tdbPagerCommit(pPager, pTxn);
     if (ret < 0) {
       ASSERT(0);
       return -1;
@@ -103,7 +103,7 @@ int tdbCommit(TENV *pEnv) {
   return 0;
 }
 
-int tdbRollback(TENV *pEnv) {
+int tdbRollback(TENV *pEnv, TXN *pTxn) {
   ASSERT(0);
   return 0;
 }
diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c
index 5d16aeaa29..b2d370a668 100644
--- a/source/libs/tdb/src/db/tdbPCache.c
+++ b/source/libs/tdb/src/db/tdbPCache.c
@@ -36,7 +36,7 @@ struct SPCache {
 #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL)
 
 static int    tdbPCacheOpenImpl(SPCache *pCache);
-static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid);
+static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn);
 static void   tdbPCachePinPage(SPCache *pCache, SPage *pPage);
 static void   tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage);
 static void   tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage);
@@ -78,12 +78,12 @@ int tdbPCacheClose(SPCache *pCache) {
   return 0;
 }
 
-SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid) {
+SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
   SPage *pPage;
 
   tdbPCacheLock(pCache);
 
-  pPage = tdbPCacheFetchImpl(pCache, pPgid);
+  pPage = tdbPCacheFetchImpl(pCache, pPgid, pTxn);
   if (pPage) {
     TDB_REF_PAGE(pPage);
   }
@@ -106,7 +106,8 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage) {
 
 int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->pageSize; }
 
-static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid) {
+static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
+  int    ret;
   SPage *pPage;
 
   // 1. Search the hash table
@@ -117,9 +118,11 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid) {
   }
 
   if (pPage) {
+    // TODO: the page need to be copied and
+    // replaced the page in hash table
     tdbPCachePinPage(pCache, pPage);
+    return pPage;
   }
-  return pPage;
 
   // 2. Try to allocate a new page from the free list
   if (pCache->pFree) {
@@ -136,7 +139,20 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid) {
     tdbPCachePinPage(pCache, pPage);
   }
 
-  // 4. Try a stress allocation (TODO)
+  // 4. Try a create new page
+  if (pTxn && pTxn->xMalloc) {
+    ret = tdbPageCreate(pCache->pageSize, &pPage, pTxn->xMalloc, pTxn->xArg);
+    if (ret < 0) {
+      // TODO
+      ASSERT(0);
+      return NULL;
+    }
+
+    // init the page fields
+    pPage->isAnchor = 0;
+    pPage->isLocal = 0;
+    TDB_INIT_PAGE_REF(pPage);
+  }
 
   // 5. Page here are just created from a free list
   // or by recycling or allocated streesly,
@@ -145,6 +161,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid) {
     memcpy(&(pPage->pgid), pPgid, sizeof(*pPgid));
     pPage->pLruNext = NULL;
     pPage->pPager = NULL;
+
+    // TODO: allocated page may not add to hash
     tdbPCacheAddPageToHash(pCache, pPage);
   }
 
@@ -171,17 +189,21 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
   nRef = TDB_GET_PAGE_REF(pPage);
   ASSERT(nRef >= 0);
   if (nRef == 0) {
-    // Add the page to LRU list
-    ASSERT(pPage->pLruNext == NULL);
+    if (1) {
+      // Add the page to LRU list
+      ASSERT(pPage->pLruNext == NULL);
 
-    pPage->pLruPrev = &(pCache->lru);
-    pPage->pLruNext = pCache->lru.pLruNext;
-    pCache->lru.pLruNext->pLruPrev = pPage;
-    pCache->lru.pLruNext = pPage;
+      pPage->pLruPrev = &(pCache->lru);
+      pPage->pLruNext = pCache->lru.pLruNext;
+      pCache->lru.pLruNext->pLruPrev = pPage;
+      pCache->lru.pLruNext = pPage;
+
+      pCache->nRecyclable++;
+    } else {
+      // TODO: may need to free the page
+    }
   }
 
-  pCache->nRecyclable++;
-
   tdbPCacheUnlock(pCache);
 }
 
@@ -229,13 +251,14 @@ static int tdbPCacheOpenImpl(SPCache *pCache) {
 
     // pPage->pgid = 0;
     pPage->isAnchor = 0;
-    pPage->isLocalPage = 1;
+    pPage->isLocal = 1;
     TDB_INIT_PAGE_REF(pPage);
     pPage->pHashNext = NULL;
     pPage->pLruNext = NULL;
     pPage->pLruPrev = NULL;
     pPage->pDirtyNext = NULL;
 
+    // add page to free list
     pPage->pFreeNext = pCache->pFree;
     pCache->pFree = pPage;
     pCache->nFree++;
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 1953419bf4..13db278c6e 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -157,7 +157,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
   return 0;
 }
 
-int tdbPagerBegin(SPager *pPager) {
+int tdbPagerBegin(SPager *pPager, TXN *pTxn) {
   if (pPager->inTran) {
     return 0;
   }
@@ -175,7 +175,7 @@ int tdbPagerBegin(SPager *pPager) {
   return 0;
 }
 
-int tdbPagerCommit(SPager *pPager) {
+int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
   SPage *pPage;
   int    ret;
 
@@ -227,7 +227,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage
   // Fetch a page container from the page cache
   memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
   pgid.pgno = pgno;
-  pPage = tdbPCacheFetch(pPager->pCache, &pgid);
+  pPage = tdbPCacheFetch(pPager->pCache, &pgid, NULL);
   if (pPage == NULL) {
     return -1;
   }
@@ -263,7 +263,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
   // Fetch a page container from the page cache
   memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
   pgid.pgno = *ppgno;
-  pPage = tdbPCacheFetch(pPager->pCache, &pgid);
+  pPage = tdbPCacheFetch(pPager->pCache, &pgid, NULL);
   if (pPage == NULL) {
     return -1;
   }
diff --git a/source/libs/tdb/src/inc/tdbEnv.h b/source/libs/tdb/src/inc/tdbEnv.h
index e10c5d54e0..54f5818be1 100644
--- a/source/libs/tdb/src/inc/tdbEnv.h
+++ b/source/libs/tdb/src/inc/tdbEnv.h
@@ -33,9 +33,9 @@ typedef struct STEnv {
 
 int tdbEnvOpen(const char *rootDir, int pageSize, int cacheSize, TENV **ppEnv);
 int tdbEnvClose(TENV *pEnv);
-int tdbBegin(TENV *pEnv);
-int tdbCommit(TENV *pEnv);
-int tdbRollback(TENV *pEnv);
+int tdbBegin(TENV *pEnv, TXN *pTxn);
+int tdbCommit(TENV *pEnv, TXN *pTxn);
+int tdbRollback(TENV *pEnv, TXN *pTxn);
 
 void    tdbEnvAddPager(TENV *pEnv, SPager *pPager);
 void    tdbEnvRemovePager(TENV *pEnv, SPager *pPager);
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index 57e01f904c..2a196b2cc2 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -111,6 +111,13 @@ typedef struct SPager  SPager;
 typedef struct SPCache SPCache;
 typedef struct SPage   SPage;
 
+typedef struct STxn {
+  u64 txnId;
+  void *(*xMalloc)(void *, size_t);
+  void (*xFree)(void *, void *);
+  void *xArg;
+} TXN;
+
 #include "tdbOs.h"
 
 #include "tdbUtil.h"
diff --git a/source/libs/tdb/src/inc/tdbPCache.h b/source/libs/tdb/src/inc/tdbPCache.h
index c5eba88545..5b24a53591 100644
--- a/source/libs/tdb/src/inc/tdbPCache.h
+++ b/source/libs/tdb/src/inc/tdbPCache.h
@@ -22,7 +22,7 @@ extern "C" {
 
 #define TDB_PCACHE_PAGE \
   u8      isAnchor;     \
-  u8      isLocalPage;  \
+  u8      isLocal;      \
   u8      isDirty;      \
   i32     nRef;         \
   SPage  *pCacheNext;   \
@@ -48,7 +48,7 @@ extern "C" {
 
 int    tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
 int    tdbPCacheClose(SPCache *pCache);
-SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid);
+SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn);
 void   tdbPCacheRelease(SPCache *pCache, SPage *pPage);
 int    tdbPCacheGetPageSize(SPCache *pCache);
 
diff --git a/source/libs/tdb/src/inc/tdbPager.h b/source/libs/tdb/src/inc/tdbPager.h
index 81b6074431..cc8860ae7a 100644
--- a/source/libs/tdb/src/inc/tdbPager.h
+++ b/source/libs/tdb/src/inc/tdbPager.h
@@ -40,8 +40,8 @@ int  tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager);
 int  tdbPagerClose(SPager *pPager);
 int  tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate);
 int  tdbPagerWrite(SPager *pPager, SPage *pPage);
-int  tdbPagerBegin(SPager *pPager);
-int  tdbPagerCommit(SPager *pPager);
+int  tdbPagerBegin(SPager *pPager, TXN *pTxn);
+int  tdbPagerCommit(SPager *pPager, TXN *pTxn);
 int  tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
 int  tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
 void tdbPagerReturnPage(SPager *pPager, SPage *pPage);
diff --git a/source/libs/tdb/src/inc/tdbTxn.h b/source/libs/tdb/src/inc/tdbTxn.h
index d979249f03..13303bffb6 100644
--- a/source/libs/tdb/src/inc/tdbTxn.h
+++ b/source/libs/tdb/src/inc/tdbTxn.h
@@ -20,15 +20,6 @@
 extern "C" {
 #endif
 
-typedef struct STxn TXN;
-
-struct STxn {
-  u64 txnId;
-  void *(*xMalloc)(void *, int);
-  void (*xFree)(void *, void *);
-  void *xArg;
-};
-
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp
index 9e1277a53d..7e5cfeff42 100644
--- a/source/libs/tdb/test/tdbTest.cpp
+++ b/source/libs/tdb/test/tdbTest.cpp
@@ -135,7 +135,7 @@ TEST(tdb_test, simple_test) {
 
     {  // Insert some data
       for (int i = 1; i <= nData;) {
-        tdbBegin(pEnv);
+        tdbBegin(pEnv, NULL);
 
         for (int k = 0; k < 2000; k++) {
           sprintf(key, "key%d", i);
@@ -145,12 +145,10 @@ TEST(tdb_test, simple_test) {
           i++;
         }
 
-        tdbCommit(pEnv);
+        tdbCommit(pEnv, NULL);
       }
     }
 
-    tdbCommit(pEnv);
-
     {  // Query the data
       void *pVal = NULL;
       int   vLen;

From 6b47fd8984f22266b852156d5c4c6c7ea96e707d Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Sat, 2 Apr 2022 11:20:56 +0000
Subject: [PATCH 04/16] TDB with txn

---
 source/libs/tdb/src/db/tdbBtree.c   | 45 ++++++++++++-----------
 source/libs/tdb/src/db/tdbDb.c      |  6 +--
 source/libs/tdb/src/db/tdbPCache.c  | 51 +++++++++++++++-----------
 source/libs/tdb/src/db/tdbPager.c   | 20 ++++++----
 source/libs/tdb/src/db/tdbTxn.c     | 36 ++++++------------
 source/libs/tdb/src/inc/tdbBtree.h  |  5 ++-
 source/libs/tdb/src/inc/tdbDb.h     |  2 +-
 source/libs/tdb/src/inc/tdbInt.h    | 10 ++++-
 source/libs/tdb/src/inc/tdbPCache.h |  2 +-
 source/libs/tdb/src/inc/tdbPager.h  |  9 +++--
 source/libs/tdb/src/inc/tdbTxn.h    |  4 ++
 source/libs/tdb/test/tdbTest.cpp    | 57 ++++++++++++++++++++---------
 12 files changed, 142 insertions(+), 105 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index f4e1621742..285434b402 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -127,7 +127,7 @@ int tdbBtreeClose(SBTree *pBt) {
   return 0;
 }
 
-int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen) {
+int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) {
   SBTC   btc;
   SCell *pCell;
   void  *pBuf;
@@ -137,7 +137,7 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in
   int    idx;
   int    c;
 
-  tdbBtcOpen(&btc, pBt);
+  tdbBtcOpen(&btc, pBt, pTxn);
 
   // move to the position to insert
   ret = tdbBtcMoveTo(&btc, pKey, kLen, &c);
@@ -225,7 +225,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
   void        *pTVal = NULL;
   SCellDecoder cd;
 
-  tdbBtcOpen(&btc, pBt);
+  tdbBtcOpen(&btc, pBt, NULL);
 
   ret = tdbBtcMoveTo(&btc, pKey, kLen, &cret);
   if (ret < 0) {
@@ -307,13 +307,13 @@ static int tdbBtreeOpenImpl(SBTree *pBt) {
 
   // Try to create a new database
   SBtreeInitPageArg zArg = {.flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF, .pBt = pBt};
-  ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg);
+  ret = tdbPagerNewPage(pBt->pPager, &pgno, &pPage, tdbBtreeZeroPage, &zArg, NULL);
   if (ret < 0) {
     return -1;
   }
 
   // TODO: here still has problem
-  tdbPagerReturnPage(pBt->pPager, pPage);
+  tdbPagerReturnPage(pBt->pPager, pPage, NULL);
 
   ASSERT(pgno != 0);
   pBt->root = pgno;
@@ -385,7 +385,7 @@ static int tdbBtreeZeroPage(SPage *pPage, void *arg) {
 }
 
 // TDB_BTREE_BALANCE =====================
-static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) {
+static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN *pTxn) {
   SPager           *pPager;
   SPage            *pChild;
   SPgno             pgnoChild;
@@ -402,7 +402,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) {
   // Allocate a new child page
   zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT);
   zArg.pBt = pBt;
-  ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg);
+  ret = tdbPagerNewPage(pPager, &pgnoChild, &pChild, tdbBtreeZeroPage, &zArg, pTxn);
   if (ret < 0) {
     return -1;
   }
@@ -436,7 +436,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) {
   return 0;
 }
 
-static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
+static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTxn) {
   int ret;
 
   int    nOlds;
@@ -477,7 +477,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
         pgno = *(SPgno *)pCell;
       }
 
-      ret = tdbPagerFetchPage(pBt->pPager, pgno, pOlds + i, tdbBtreeInitPage, pBt);
+      ret = tdbPagerFetchPage(pBt->pPager, pgno, pOlds + i, tdbBtreeInitPage, pBt, pTxn);
       if (ret < 0) {
         ASSERT(0);
         return -1;
@@ -640,7 +640,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
       } else {
         iarg.pBt = pBt;
         iarg.flags = flags;
-        ret = tdbPagerNewPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeZeroPage, &iarg);
+        ret = tdbPagerNewPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeZeroPage, &iarg, pTxn);
         if (ret < 0) {
           ASSERT(0);
         }
@@ -767,9 +767,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) {
   // TODO: here is not corrent for drop case
   for (int i = 0; i < nNews; i++) {
     if (i < nOlds) {
-      tdbPagerReturnPage(pBt->pPager, pOlds[i]);
+      tdbPagerReturnPage(pBt->pPager, pOlds[i], pTxn);
     } else {
-      tdbPagerReturnPage(pBt->pPager, pNews[i]);
+      tdbPagerReturnPage(pBt->pPager, pNews[i], pTxn);
     }
   }
 
@@ -805,7 +805,7 @@ static int tdbBtreeBalance(SBTC *pBtc) {
       // ignore the case of empty
       if (pPage->nOverflow == 0) break;
 
-      ret = tdbBtreeBalanceDeeper(pBtc->pBt, pPage, &(pBtc->pgStack[1]));
+      ret = tdbBtreeBalanceDeeper(pBtc->pBt, pPage, &(pBtc->pgStack[1]), pBtc->pTxn);
       if (ret < 0) {
         return -1;
       }
@@ -819,12 +819,12 @@ static int tdbBtreeBalance(SBTC *pBtc) {
       // Generalized balance step
       pParent = pBtc->pgStack[iPage - 1];
 
-      ret = tdbBtreeBalanceNonRoot(pBtc->pBt, pParent, pBtc->idxStack[pBtc->iPage - 1]);
+      ret = tdbBtreeBalanceNonRoot(pBtc->pBt, pParent, pBtc->idxStack[pBtc->iPage - 1], pBtc->pTxn);
       if (ret < 0) {
         return -1;
       }
 
-      tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage);
+      tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage, pBtc->pTxn);
 
       pBtc->iPage--;
       pBtc->pPage = pBtc->pgStack[pBtc->iPage];
@@ -1024,11 +1024,12 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) {
 // TDB_BTREE_CELL
 
 // TDB_BTREE_CURSOR =====================
-int tdbBtcOpen(SBTC *pBtc, SBTree *pBt) {
+int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn) {
   pBtc->pBt = pBt;
   pBtc->iPage = -1;
   pBtc->pPage = NULL;
   pBtc->idx = -1;
+  pBtc->pTxn = pTxn;
 
   return 0;
 }
@@ -1045,7 +1046,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) {
 
   if (pBtc->iPage < 0) {
     // move a clean cursor
-    ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt);
+    ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn);
     if (ret < 0) {
       ASSERT(0);
       return -1;
@@ -1110,7 +1111,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) {
 
   if (pBtc->iPage < 0) {
     // move a clean cursor
-    ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt);
+    ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn);
     if (ret < 0) {
       ASSERT(0);
       return -1;
@@ -1284,7 +1285,7 @@ static int tdbBtcMoveDownward(SBTC *pBtc) {
   pBtc->pPage = NULL;
   pBtc->idx = -1;
 
-  ret = tdbPagerFetchPage(pBtc->pBt->pPager, pgno, &pBtc->pPage, tdbBtreeInitPage, pBtc->pBt);
+  ret = tdbPagerFetchPage(pBtc->pBt->pPager, pgno, &pBtc->pPage, tdbBtreeInitPage, pBtc->pBt, pBtc->pTxn);
   if (ret < 0) {
     ASSERT(0);
     return -1;
@@ -1296,7 +1297,7 @@ static int tdbBtcMoveDownward(SBTC *pBtc) {
 static int tdbBtcMoveUpward(SBTC *pBtc) {
   if (pBtc->iPage == 0) return -1;
 
-  tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage);
+  tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage, pBtc->pTxn);
 
   pBtc->iPage--;
   pBtc->pPage = pBtc->pgStack[pBtc->iPage];
@@ -1319,7 +1320,7 @@ static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) {
 
   if (pBtc->iPage < 0) {
     // move from a clear cursor
-    ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt);
+    ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt, pBtc->pTxn);
     if (ret < 0) {
       // TODO
       ASSERT(0);
@@ -1456,7 +1457,7 @@ int tdbBtcClose(SBTC *pBtc) {
   for (;;) {
     ASSERT(pBtc->pPage);
 
-    tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage);
+    tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage, pBtc->pTxn);
 
     pBtc->iPage--;
     if (pBtc->iPage < 0) break;
diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c
index fe7b8c6d48..f4468e7e82 100644
--- a/source/libs/tdb/src/db/tdbDb.c
+++ b/source/libs/tdb/src/db/tdbDb.c
@@ -75,8 +75,8 @@ int tdbDbDrop(TDB *pDb) {
   return 0;
 }
 
-int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen) {
-  return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen);
+int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn) {
+  return tdbBtreeInsert(pDb->pBt, pKey, keyLen, pVal, valLen, pTxn);
 }
 
 int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) {
@@ -97,7 +97,7 @@ int tdbDbcOpen(TDB *pDb, TDBC **ppDbc) {
     return -1;
   }
 
-  tdbBtcOpen(&pDbc->btc, pDb->pBt);
+  tdbBtcOpen(&pDbc->btc, pDb->pBt, NULL);
 
   // TODO: move to first now, we can move to any key-value
   // and in any direction, design new APIs.
diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c
index b2d370a668..b4e291f5e4 100644
--- a/source/libs/tdb/src/db/tdbPCache.c
+++ b/source/libs/tdb/src/db/tdbPCache.c
@@ -93,14 +93,33 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
   return pPage;
 }
 
-void tdbPCacheRelease(SPCache *pCache, SPage *pPage) {
+void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
   i32 nRef;
 
   nRef = TDB_UNREF_PAGE(pPage);
   ASSERT(nRef >= 0);
 
   if (nRef == 0) {
-    tdbPCacheUnpinPage(pCache, pPage);
+    tdbPCacheLock(pCache);
+
+    // test the nRef again to make sure
+    // it is safe th handle the page
+    nRef = TDB_GET_PAGE_REF(pPage);
+    if (nRef == 0) {
+      if (pPage->isLocal) {
+        tdbPCacheUnpinPage(pCache, pPage);
+      } else {
+        // remove from hash
+        tdbPCacheRemovePageFromHash(pCache, pPage);
+
+        // free the page
+        if (pTxn && pTxn->xFree) {
+          tdbPageDestroy(pPage, pTxn->xFree, pTxn->xArg);
+        }
+      }
+    }
+
+    tdbPCacheUnlock(pCache);
   }
 }
 
@@ -140,7 +159,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn)
   }
 
   // 4. Try a create new page
-  if (pTxn && pTxn->xMalloc) {
+  if (!pPage && pTxn && pTxn->xMalloc) {
     ret = tdbPageCreate(pCache->pageSize, &pPage, pTxn->xMalloc, pTxn->xArg);
     if (ret < 0) {
       // TODO
@@ -182,29 +201,17 @@ static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) {
 static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) {
   i32 nRef;
 
-  tdbPCacheLock(pCache);
-
   ASSERT(!pPage->isDirty);
+  ASSERT(TDB_GET_PAGE_REF(pPage) == 0);
 
-  nRef = TDB_GET_PAGE_REF(pPage);
-  ASSERT(nRef >= 0);
-  if (nRef == 0) {
-    if (1) {
-      // Add the page to LRU list
-      ASSERT(pPage->pLruNext == NULL);
+  ASSERT(pPage->pLruNext == NULL);
 
-      pPage->pLruPrev = &(pCache->lru);
-      pPage->pLruNext = pCache->lru.pLruNext;
-      pCache->lru.pLruNext->pLruPrev = pPage;
-      pCache->lru.pLruNext = pPage;
+  pPage->pLruPrev = &(pCache->lru);
+  pPage->pLruNext = pCache->lru.pLruNext;
+  pCache->lru.pLruNext->pLruPrev = pPage;
+  pCache->lru.pLruNext = pPage;
 
-      pCache->nRecyclable++;
-    } else {
-      // TODO: may need to free the page
-    }
-  }
-
-  tdbPCacheUnlock(pCache);
+  pCache->nRecyclable++;
 }
 
 static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) {
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index 13db278c6e..f78568b51c 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -27,7 +27,6 @@ TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct")
 
 #define TDB_PAGE_INITIALIZED(pPage) ((pPage)->pPager != NULL)
 
-static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
 static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *), void *arg, u8 loadPage);
 static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage);
 static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage);
@@ -204,7 +203,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
 
     pPage->isDirty = 0;
 
-    tdbPCacheRelease(pPager->pCache, pPage);
+    tdbPCacheRelease(pPager->pCache, pPage, pTxn);
   }
 
   // sync the db file
@@ -219,7 +218,8 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
   return 0;
 }
 
-int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) {
+int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg,
+                      TXN *pTxn) {
   SPage *pPage;
   SPgid  pgid;
   int    ret;
@@ -227,7 +227,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage
   // Fetch a page container from the page cache
   memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
   pgid.pgno = pgno;
-  pPage = tdbPCacheFetch(pPager->pCache, &pgid, NULL);
+  pPage = tdbPCacheFetch(pPager->pCache, &pgid, pTxn);
   if (pPage == NULL) {
     return -1;
   }
@@ -247,7 +247,8 @@ int tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage
   return 0;
 }
 
-int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg) {
+int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg,
+                    TXN *pTxn) {
   int    ret;
   SPage *pPage;
   SPgid  pgid;
@@ -255,6 +256,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
   // Allocate a page number
   ret = tdbPagerAllocPage(pPager, ppgno);
   if (ret < 0) {
+    ASSERT(0);
     return -1;
   }
 
@@ -263,8 +265,9 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
   // Fetch a page container from the page cache
   memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN);
   pgid.pgno = *ppgno;
-  pPage = tdbPCacheFetch(pPager->pCache, &pgid, NULL);
+  pPage = tdbPCacheFetch(pPager->pCache, &pgid, pTxn);
   if (pPage == NULL) {
+    ASSERT(0);
     return -1;
   }
 
@@ -273,6 +276,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
   // Initialize the page if need
   ret = tdbPagerInitPage(pPager, pPage, initPage, arg, 0);
   if (ret < 0) {
+    ASSERT(0);
     return -1;
   }
 
@@ -283,7 +287,7 @@ int tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage
   return 0;
 }
 
-void tdbPagerReturnPage(SPager *pPager, SPage *pPage) { tdbPCacheRelease(pPager->pCache, pPage); }
+void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn) { tdbPCacheRelease(pPager->pCache, pPage, pTxn); }
 
 static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno) {
   // TODO: Allocate a page from the free list
@@ -295,7 +299,7 @@ static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) {
   return 0;
 }
 
-static int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) {
+int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) {
   int ret;
 
   *ppgno = 0;
diff --git a/source/libs/tdb/src/db/tdbTxn.c b/source/libs/tdb/src/db/tdbTxn.c
index 03bcbb44a7..b06fe05acd 100644
--- a/source/libs/tdb/src/db/tdbTxn.c
+++ b/source/libs/tdb/src/db/tdbTxn.c
@@ -15,29 +15,17 @@
 
 #include "tdbInt.h"
 
-// int tdbTxnBegin(TENV *pEnv) {
-//   // TODO
-//   return 0;
-// }
+int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg,
+               int flags) {
+  // not support read-committed version at the moment
+  ASSERT(flags == 0 || flags == TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
 
-// int tdbTxnCommit(TENV *pEnv) {
-//   SPager *pPager = NULL;
-//   int     ret;
+  pTxn->flags = flags;
+  pTxn->txnId = txnid;
+  pTxn->xMalloc = xMalloc;
+  pTxn->xFree = xFree;
+  pTxn->xArg = xArg;
+  return 0;
+}
 
-//   for (;;) {
-//     break;
-//     ret = tdbPagerCommit(pPager);
-//     if (ret < 0) {
-//       ASSERT(0);
-//       return -1;
-//     }
-//   }
-
-//   // TODO
-//   return 0;
-// }
-
-// int tdbTxnRollback(TENV *pEnv) {
-//   // TODO
-//   return 0;
-// }
\ No newline at end of file
+int tdbTxnClose(TXN *pTxn) { return 0; }
\ No newline at end of file
diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h
index 2eba5f4f1a..3cdd30c7b5 100644
--- a/source/libs/tdb/src/inc/tdbBtree.h
+++ b/source/libs/tdb/src/inc/tdbBtree.h
@@ -35,17 +35,18 @@ struct SBTC {
   int     idx;
   int     idxStack[BTREE_MAX_DEPTH + 1];
   SPage  *pgStack[BTREE_MAX_DEPTH + 1];
+  TXN    *pTxn;
 };
 
 // SBTree
 int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SBTree **ppBt);
 int tdbBtreeClose(SBTree *pBt);
-int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen);
+int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
 int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen);
 int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
 
 // SBTC
-int tdbBtcOpen(SBTC *pCur, SBTree *pBt);
+int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn);
 int tdbBtcMoveToFirst(SBTC *pBtc);
 int tdbBtcMoveToLast(SBTC *pBtc);
 int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen);
diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h
index e60371c734..f9ff378633 100644
--- a/source/libs/tdb/src/inc/tdbDb.h
+++ b/source/libs/tdb/src/inc/tdbDb.h
@@ -27,7 +27,7 @@ typedef struct STDBC TDBC;
 int tdbDbOpen(const char *fname, int keyLen, int valLen, FKeyComparator keyCmprFn, TENV *pEnv, TDB **ppDb);
 int tdbDbClose(TDB *pDb);
 int tdbDbDrop(TDB *pDb);
-int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen);
+int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen, TXN *pTxn);
 int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen);
 int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
 
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index 2a196b2cc2..b85b7174bc 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -111,13 +111,21 @@ typedef struct SPager  SPager;
 typedef struct SPCache SPCache;
 typedef struct SPage   SPage;
 
+// transaction
+#define TDB_TXN_WRITE            0x1
+#define TDB_TXN_READ_UNCOMMITTED 0x2
 typedef struct STxn {
-  u64 txnId;
+  int flags;
+  i64 txnId;
   void *(*xMalloc)(void *, size_t);
   void (*xFree)(void *, void *);
   void *xArg;
 } TXN;
 
+#define TDB_TXN_IS_WRITE(PTXN)            ((PTXN)->flags & TDB_TXN_WRITE)
+#define TDB_TXN_IS_READ(PTXN)             (!TDB_TXN_IS_WRITE(PTXN))
+#define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED)
+
 #include "tdbOs.h"
 
 #include "tdbUtil.h"
diff --git a/source/libs/tdb/src/inc/tdbPCache.h b/source/libs/tdb/src/inc/tdbPCache.h
index 5b24a53591..a77cf6248c 100644
--- a/source/libs/tdb/src/inc/tdbPCache.h
+++ b/source/libs/tdb/src/inc/tdbPCache.h
@@ -49,7 +49,7 @@ extern "C" {
 int    tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
 int    tdbPCacheClose(SPCache *pCache);
 SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn);
-void   tdbPCacheRelease(SPCache *pCache, SPage *pPage);
+void   tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn);
 int    tdbPCacheGetPageSize(SPCache *pCache);
 
 #ifdef __cplusplus
diff --git a/source/libs/tdb/src/inc/tdbPager.h b/source/libs/tdb/src/inc/tdbPager.h
index cc8860ae7a..ca196785d0 100644
--- a/source/libs/tdb/src/inc/tdbPager.h
+++ b/source/libs/tdb/src/inc/tdbPager.h
@@ -42,9 +42,12 @@ int  tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate);
 int  tdbPagerWrite(SPager *pPager, SPage *pPage);
 int  tdbPagerBegin(SPager *pPager, TXN *pTxn);
 int  tdbPagerCommit(SPager *pPager, TXN *pTxn);
-int  tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
-int  tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg);
-void tdbPagerReturnPage(SPager *pPager, SPage *pPage);
+int  tdbPagerFetchPage(SPager *pPager, SPgno pgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg,
+                       TXN *pTxn);
+int  tdbPagerNewPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *), void *arg,
+                     TXN *pTxn);
+void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
+int  tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/tdb/src/inc/tdbTxn.h b/source/libs/tdb/src/inc/tdbTxn.h
index 13303bffb6..ed9784614c 100644
--- a/source/libs/tdb/src/inc/tdbTxn.h
+++ b/source/libs/tdb/src/inc/tdbTxn.h
@@ -20,6 +20,10 @@
 extern "C" {
 #endif
 
+int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg,
+               int flags);
+int tdbTxnClose(TXN *pTxn);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp
index 7e5cfeff42..904ad64fe7 100644
--- a/source/libs/tdb/test/tdbTest.cpp
+++ b/source/libs/tdb/test/tdbTest.cpp
@@ -19,7 +19,7 @@ static SPoolMem *openPool() {
   return pPool;
 }
 
-static void closePool(SPoolMem *pPool) {
+static void clearPool(SPoolMem *pPool) {
   SPoolMem *pMem;
 
   do {
@@ -35,13 +35,14 @@ static void closePool(SPoolMem *pPool) {
   } while (1);
 
   assert(pPool->size == 0);
+}
 
+static void closePool(SPoolMem *pPool) {
+  clearPool(pPool);
   tdbOsFree(pPool);
 }
 
-#define clearPool closePool
-
-static void *poolMalloc(void *arg, int size) {
+static void *poolMalloc(void *arg, size_t size) {
   void     *ptr = NULL;
   SPoolMem *pPool = (SPoolMem *)arg;
   SPoolMem *pMem;
@@ -118,7 +119,8 @@ TEST(tdb_test, simple_test) {
   TENV          *pEnv;
   TDB           *pDb;
   FKeyComparator compFunc;
-  int            nData = 50000000;
+  int            nData = 10000000;
+  TXN            txn;
 
   // Open Env
   ret = tdbEnvOpen("tdb", 4096, 64, &pEnv);
@@ -130,25 +132,44 @@ TEST(tdb_test, simple_test) {
   GTEST_ASSERT_EQ(ret, 0);
 
   {
-    char key[64];
-    char val[64];
+    char      key[64];
+    char      val[64];
+    int64_t   poolLimit = 4096;  // 1M pool limit
+    int64_t   txnid = 0;
+    SPoolMem *pPool;
 
-    {  // Insert some data
-      for (int i = 1; i <= nData;) {
-        tdbBegin(pEnv, NULL);
+    // open the pool
+    pPool = openPool();
 
-        for (int k = 0; k < 2000; k++) {
-          sprintf(key, "key%d", i);
-          sprintf(val, "value%d", i);
-          ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val));
-          GTEST_ASSERT_EQ(ret, 0);
-          i++;
-        }
+    // start a transaction
+    txnid++;
+    tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+    tdbBegin(pEnv, &txn);
 
-        tdbCommit(pEnv, NULL);
+    for (int iData = 1; iData <= nData; iData++) {
+      sprintf(key, "key%d", iData);
+      sprintf(val, "value%d", iData);
+      ret = tdbDbInsert(pDb, key, strlen(key), val, strlen(val), &txn);
+      GTEST_ASSERT_EQ(ret, 0);
+
+      // if pool is full, commit the transaction and start a new one
+      if (pPool->size >= poolLimit) {
+        // commit current transaction
+        tdbCommit(pEnv, &txn);
+        tdbTxnClose(&txn);
+
+        // start a new transaction
+        clearPool(pPool);
+        txnid++;
+        tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+        tdbBegin(pEnv, &txn);
       }
     }
 
+    // commit the transaction
+    tdbCommit(pEnv, &txn);
+    tdbTxnClose(&txn);
+
     {  // Query the data
       void *pVal = NULL;
       int   vLen;

From 4540f53b32cedfc3b15646913cdfa6bcd0ea2d44 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 01:45:22 +0000
Subject: [PATCH 05/16] make feature/meta compile

---
 source/dnode/vnode/src/meta/metaTDBImpl.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c
index 8e82cf1abc..a004a96ad0 100644
--- a/source/dnode/vnode/src/meta/metaTDBImpl.c
+++ b/source/dnode/vnode/src/meta/metaTDBImpl.c
@@ -212,7 +212,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
   pVal = pBuf = buf;
   metaEncodeTbInfo(&pBuf, pTbCfg);
   vLen = POINTER_DISTANCE(pBuf, buf);
-  ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen);
+  ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, NULL);
   if (ret < 0) {
     return -1;
   }
@@ -234,7 +234,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     pVal = pBuf = buf;
     metaEncodeSchemaEx(&pBuf, &schemaWrapper);
     vLen = POINTER_DISTANCE(pBuf, buf);
-    ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen);
+    ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, NULL);
     if (ret < 0) {
       return -1;
     }
@@ -248,7 +248,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
   kLen = nameLen + 1 + sizeof(uid);
   pVal = NULL;
   vLen = 0;
-  ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen);
+  ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, NULL);
   if (ret < 0) {
     return -1;
   }
@@ -259,7 +259,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     kLen = sizeof(uid);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen);
+    ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, NULL);
     if (ret < 0) {
       return -1;
     }
@@ -270,7 +270,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     kLen = sizeof(ctbIdxKey);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen);
+    ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, NULL);
     if (ret < 0) {
       return -1;
     }
@@ -279,7 +279,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     kLen = sizeof(uid);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen);
+    ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, NULL);
     if (ret < 0) {
       return -1;
     }

From 906b615c9a138501f0f8e683db86f840fb6b022e Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 01:55:37 +0000
Subject: [PATCH 06/16] make TDB compile

---
 source/dnode/vnode/CMakeLists.txt         | 2 +-
 source/dnode/vnode/src/meta/metaTDBImpl.c | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt
index 56a8698d77..6b21f935c4 100644
--- a/source/dnode/vnode/CMakeLists.txt
+++ b/source/dnode/vnode/CMakeLists.txt
@@ -1,5 +1,5 @@
 set(META_DB_IMPL_LIST "BDB" "TDB")
-set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation")
+set(META_DB_IMPL "TDB" CACHE STRING "Use BDB as the default META implementation")
 set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST})
 
 if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST)
diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c
index a004a96ad0..cfbd83df5c 100644
--- a/source/dnode/vnode/src/meta/metaTDBImpl.c
+++ b/source/dnode/vnode/src/meta/metaTDBImpl.c
@@ -226,10 +226,10 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
 
     if (pTbCfg->type == META_SUPER_TABLE) {
       schemaWrapper.nCols = pTbCfg->stbCfg.nCols;
-      schemaWrapper.pSchema = pTbCfg->stbCfg.pSchema;
+      schemaWrapper.pSchemaEx = pTbCfg->stbCfg.pSchema;
     } else {
       schemaWrapper.nCols = pTbCfg->ntbCfg.nCols;
-      schemaWrapper.pSchema = pTbCfg->ntbCfg.pSchema;
+      schemaWrapper.pSchemaEx = pTbCfg->ntbCfg.pSchema;
     }
     pVal = pBuf = buf;
     metaEncodeSchemaEx(&pBuf, &schemaWrapper);
@@ -349,7 +349,7 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) {
 }
 
 SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) {
-  return *metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false);
+  return metaGetTableSchemaImpl(pMeta, uid, sver, isinline, false);
 }
 
 static SSchemaWrapper *metaGetTableSchemaImpl(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline, bool isGetEx) {

From 317d4ff2eb8dd83c34d6d9fdb6ea6c52df181b5c Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 02:06:59 +0000
Subject: [PATCH 07/16] make tdb work with meta

---
 source/dnode/vnode/src/meta/metaCommit.c  | 21 ---------------------
 source/dnode/vnode/src/meta/metaTDBImpl.c |  7 +++++++
 2 files changed, 7 insertions(+), 21 deletions(-)
 delete mode 100644 source/dnode/vnode/src/meta/metaCommit.c

diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c
deleted file mode 100644
index 805e4f3e3d..0000000000
--- a/source/dnode/vnode/src/meta/metaCommit.c
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "meta.h"
-
-int metaCommit(SMeta *pMeta) {
-  // TODO
-  return 0;
-}
\ No newline at end of file
diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c
index cfbd83df5c..e934caa7e3 100644
--- a/source/dnode/vnode/src/meta/metaTDBImpl.c
+++ b/source/dnode/vnode/src/meta/metaTDBImpl.c
@@ -167,6 +167,8 @@ int metaOpenDB(SMeta *pMeta) {
     return -1;
   }
 
+  tdbBegin(pMetaDb->pEnv, NULL);
+
   pMeta->pDB = pMetaDb;
   return 0;
 }
@@ -701,3 +703,8 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) {
   }
   return buf;
 }
+
+int metaCommit(SMeta *pMeta) {
+  tdbCommit(pMeta->pDB->pEnv, NULL);
+  return 0;
+}
\ No newline at end of file

From 69ff3cc67b4980bb96aa97852b03f3d1eeb3ca09 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 06:37:22 +0000
Subject: [PATCH 08/16] fix an assert bug

---
 source/libs/tdb/src/db/tdbPage.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c
index cc4ce20802..ea75f7fbeb 100644
--- a/source/libs/tdb/src/db/tdbPage.c
+++ b/source/libs/tdb/src/db/tdbPage.c
@@ -278,7 +278,7 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) {
 
   // 2. Try to allocate from the page free list
   cellFree = TDB_PAGE_FCELL(pPage);
-  ASSERT(cellFree == 0 || cellFree > pPage->pFreeEnd - pPage->pData);
+  ASSERT(cellFree == 0 || cellFree >= pPage->pFreeEnd - pPage->pData);
   if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) {
     SCell *pPrevFreeCell = NULL;
     int    szPrevFreeCell;

From 61d0180b852f93333d086cba388e33c0918675f9 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 06:37:37 +0000
Subject: [PATCH 09/16] more TDB integration

---
 source/dnode/vnode/src/meta/metaTDBImpl.c | 125 +++++++++++++++++++---
 source/dnode/vnode/src/meta/metaTbUid.c   |   2 +-
 source/dnode/vnode/src/vnd/vnodeCommit.c  |   2 +-
 3 files changed, 112 insertions(+), 17 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c
index e934caa7e3..282e0a51f6 100644
--- a/source/dnode/vnode/src/meta/metaTDBImpl.c
+++ b/source/dnode/vnode/src/meta/metaTDBImpl.c
@@ -16,15 +16,28 @@
 #include "metaDef.h"
 
 #include "tdbInt.h"
+typedef struct SPoolMem {
+  int64_t          size;
+  struct SPoolMem *prev;
+  struct SPoolMem *next;
+} SPoolMem;
+
+static SPoolMem *openPool();
+static void      clearPool(SPoolMem *pPool);
+static void      closePool(SPoolMem *pPool);
+static void     *poolMalloc(void *arg, size_t size);
+static void      poolFree(void *arg, void *ptr);
 
 struct SMetaDB {
-  TENV *pEnv;
-  TDB  *pTbDB;
-  TDB  *pSchemaDB;
-  TDB  *pNameIdx;
-  TDB  *pStbIdx;
-  TDB  *pNtbIdx;
-  TDB  *pCtbIdx;
+  TXN       txn;
+  TENV     *pEnv;
+  TDB      *pTbDB;
+  TDB      *pSchemaDB;
+  TDB      *pNameIdx;
+  TDB      *pStbIdx;
+  TDB      *pNtbIdx;
+  TDB      *pCtbIdx;
+  SPoolMem *pPool;
 };
 
 typedef struct __attribute__((__packed__)) {
@@ -167,6 +180,8 @@ int metaOpenDB(SMeta *pMeta) {
     return -1;
   }
 
+  pMetaDb->pPool = openPool();
+  tdbTxnOpen(&pMetaDb->txn, 0, poolMalloc, poolFree, pMetaDb->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
   tdbBegin(pMetaDb->pEnv, NULL);
 
   pMeta->pDB = pMetaDb;
@@ -214,7 +229,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
   pVal = pBuf = buf;
   metaEncodeTbInfo(&pBuf, pTbCfg);
   vLen = POINTER_DISTANCE(pBuf, buf);
-  ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, NULL);
+  ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen, &pMetaDb->txn);
   if (ret < 0) {
     return -1;
   }
@@ -236,7 +251,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     pVal = pBuf = buf;
     metaEncodeSchemaEx(&pBuf, &schemaWrapper);
     vLen = POINTER_DISTANCE(pBuf, buf);
-    ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, NULL);
+    ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen, &pMeta->pDB->txn);
     if (ret < 0) {
       return -1;
     }
@@ -250,7 +265,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
   kLen = nameLen + 1 + sizeof(uid);
   pVal = NULL;
   vLen = 0;
-  ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, NULL);
+  ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
   if (ret < 0) {
     return -1;
   }
@@ -261,7 +276,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     kLen = sizeof(uid);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, NULL);
+    ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
     if (ret < 0) {
       return -1;
     }
@@ -272,7 +287,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     kLen = sizeof(ctbIdxKey);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, NULL);
+    ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
     if (ret < 0) {
       return -1;
     }
@@ -281,12 +296,16 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     kLen = sizeof(uid);
     pVal = NULL;
     vLen = 0;
-    ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, NULL);
+    ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen, &pMetaDb->txn);
     if (ret < 0) {
       return -1;
     }
   }
 
+  if (pMeta->pDB->pPool->size > 0) {
+    metaCommit(pMeta);
+  }
+
   return 0;
 }
 
@@ -705,6 +724,82 @@ static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) {
 }
 
 int metaCommit(SMeta *pMeta) {
-  tdbCommit(pMeta->pDB->pEnv, NULL);
+  TXN *pTxn = &pMeta->pDB->txn;
+
+  // Commit current txn
+  tdbCommit(pMeta->pDB->pEnv, pTxn);
+  tdbTxnClose(pTxn);
+  clearPool(pMeta->pDB->pPool);
+
+  // start a new txn
+  tdbTxnOpen(&pMeta->pDB->txn, 0, poolMalloc, poolFree, pMeta->pDB->pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED);
+  tdbBegin(pMeta->pDB->pEnv, pTxn);
   return 0;
-}
\ No newline at end of file
+}
+
+static SPoolMem *openPool() {
+  SPoolMem *pPool = (SPoolMem *)tdbOsMalloc(sizeof(*pPool));
+
+  pPool->prev = pPool->next = pPool;
+  pPool->size = 0;
+
+  return pPool;
+}
+
+static void clearPool(SPoolMem *pPool) {
+  SPoolMem *pMem;
+
+  do {
+    pMem = pPool->next;
+
+    if (pMem == pPool) break;
+
+    pMem->next->prev = pMem->prev;
+    pMem->prev->next = pMem->next;
+    pPool->size -= pMem->size;
+
+    tdbOsFree(pMem);
+  } while (1);
+
+  assert(pPool->size == 0);
+}
+
+static void closePool(SPoolMem *pPool) {
+  clearPool(pPool);
+  tdbOsFree(pPool);
+}
+
+static void *poolMalloc(void *arg, size_t size) {
+  void     *ptr = NULL;
+  SPoolMem *pPool = (SPoolMem *)arg;
+  SPoolMem *pMem;
+
+  pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size);
+  if (pMem == NULL) {
+    assert(0);
+  }
+
+  pMem->size = sizeof(*pMem) + size;
+  pMem->next = pPool->next;
+  pMem->prev = pPool;
+
+  pPool->next->prev = pMem;
+  pPool->next = pMem;
+  pPool->size += pMem->size;
+
+  ptr = (void *)(&pMem[1]);
+  return ptr;
+}
+
+static void poolFree(void *arg, void *ptr) {
+  SPoolMem *pPool = (SPoolMem *)arg;
+  SPoolMem *pMem;
+
+  pMem = &(((SPoolMem *)ptr)[-1]);
+
+  pMem->next->prev = pMem->prev;
+  pMem->prev->next = pMem->next;
+  pPool->size -= pMem->size;
+
+  tdbOsFree(pMem);
+}
diff --git a/source/dnode/vnode/src/meta/metaTbUid.c b/source/dnode/vnode/src/meta/metaTbUid.c
index 1f57d1396a..b488630024 100644
--- a/source/dnode/vnode/src/meta/metaTbUid.c
+++ b/source/dnode/vnode/src/meta/metaTbUid.c
@@ -27,5 +27,5 @@ void metaCloseUidGnrt(SMeta *pMeta) { /* TODO */
 
 tb_uid_t metaGenerateUid(SMeta *pMeta) {
   // Generate a new table UID
-  return tGenIdPI32();
+  return tGenIdPI64();
 }
diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c
index 696c5f39f6..1717721bb2 100644
--- a/source/dnode/vnode/src/vnd/vnodeCommit.c
+++ b/source/dnode/vnode/src/vnd/vnodeCommit.c
@@ -47,7 +47,7 @@ int vnodeSyncCommit(SVnode *pVnode) {
 static int vnodeCommit(void *arg) {
   SVnode *pVnode = (SVnode *)arg;
 
-  metaCommit(pVnode->pMeta);
+  // metaCommit(pVnode->pMeta);
   tqCommit(pVnode->pTq);
   tsdbCommit(pVnode->pTsdb);
 

From cf727c9db30104bfc86ffcc8fab4488d25708336 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 07:54:52 +0000
Subject: [PATCH 10/16] fix show tables problem

---
 source/libs/executor/src/executorimpl.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 2ca06cb15f..148d16aaf8 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -310,6 +310,10 @@ SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) {
     idata.info.slotId = pDescNode->slotId;
     idata.info.precision = pDescNode->dataType.precision;
 
+    if (IS_VAR_DATA_TYPE(idata.info.type)) {
+      pBlock->info.hasVarCol = true;
+    }
+
     taosArrayPush(pBlock->pDataBlock, &idata);
   }
 

From e40dc9b9291b193886ec2757b0c8a2380ba9c022 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 08:06:28 +0000
Subject: [PATCH 11/16] commit meta when close vnode

---
 source/dnode/vnode/src/meta/metaTDBImpl.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c
index 282e0a51f6..c273f584ff 100644
--- a/source/dnode/vnode/src/meta/metaTDBImpl.c
+++ b/source/dnode/vnode/src/meta/metaTDBImpl.c
@@ -190,6 +190,9 @@ int metaOpenDB(SMeta *pMeta) {
 
 void metaCloseDB(SMeta *pMeta) {
   if (pMeta->pDB) {
+    tdbCommit(pMeta->pDB->pEnv, &pMeta->pDB->txn);
+    tdbTxnClose(&pMeta->pDB->txn);
+    clearPool(pMeta->pDB->pPool);
     tdbDbClose(pMeta->pDB->pCtbIdx);
     tdbDbClose(pMeta->pDB->pNtbIdx);
     tdbDbClose(pMeta->pDB->pStbIdx);

From 610832f0b47d2d535bfaf12c60368708d7b74098 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 08:41:53 +0000
Subject: [PATCH 12/16] keep committed data in TDB

---
 source/libs/tdb/src/db/tdbBtree.c |  3 ++-
 source/libs/tdb/src/db/tdbPager.c | 36 +++++++++++++++++++------------
 source/libs/tdb/src/db/tdbUtil.c  | 13 +++++++++++
 source/libs/tdb/src/inc/tdbOs.h   | 20 +++++++++--------
 source/libs/tdb/src/inc/tdbUtil.h |  1 +
 5 files changed, 49 insertions(+), 24 deletions(-)

diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 285434b402..f760598714 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -297,7 +297,8 @@ static int tdbBtreeOpenImpl(SBTree *pBt) {
 
   {
     // 1. TODO: Search the main DB to check if the DB exists
-    pgno = 0;
+    ret = tdbPagerOpenDB(pBt->pPager, &pgno, true);
+    ASSERT(ret == 0);
   }
 
   if (pgno != 0) {
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index f78568b51c..7672ccf34e 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -76,6 +76,8 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
 
   // pPager->jfd = -1;
   pPager->pageSize = tdbPCacheGetPageSize(pCache);
+  // pPager->dbOrigSize
+  ret = tdbGetFileSize(pPager->fd, pPager->pageSize, &(pPager->dbOrigSize));
 
   *ppPager = pPager;
   return 0;
@@ -91,26 +93,32 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate) {
   SPage *pPage;
   int    ret;
 
-  {
-    // TODO: try to search the main DB to get the page number
+  if (pPager->dbOrigSize > 0) {
+    pgno = 1;
+  } else {
     pgno = 0;
   }
 
-  // if (pgno == 0 && toCreate) {
-  //   ret = tdbPagerAllocPage(pPager, &pPage, &pgno);
-  //   if (ret < 0) {
-  //     return -1;
-  //   }
+  {
+      // TODO: try to search the main DB to get the page number
+      // pgno = 0;
+  }
 
-  //   // TODO: Need to zero the page
+      // if (pgno == 0 && toCreate) {
+      //   ret = tdbPagerAllocPage(pPager, &pPage, &pgno);
+      //   if (ret < 0) {
+      //     return -1;
+      //   }
 
-  //   ret = tdbPagerWrite(pPager, pPage);
-  //   if (ret < 0) {
-  //     return -1;
-  //   }
-  // }
+      //   // TODO: Need to zero the page
 
-  *ppgno = pgno;
+      //   ret = tdbPagerWrite(pPager, pPage);
+      //   if (ret < 0) {
+      //     return -1;
+      //   }
+      // }
+
+      *ppgno = pgno;
   return 0;
 }
 
diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c
index fc299b3fc1..09e126772b 100644
--- a/source/libs/tdb/src/db/tdbUtil.c
+++ b/source/libs/tdb/src/db/tdbUtil.c
@@ -30,5 +30,18 @@ int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique) {
     ((uint64_t *)fileid)[2] = taosRand();
   }
 
+  return 0;
+}
+
+int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size) {
+  int     ret;
+  int64_t szBytes;
+
+  ret = tdbOsFileSize(fd, &szBytes);
+  if (ret < 0) {
+    return -1;
+  }
+
+  *size = szBytes / szPage;
   return 0;
 }
\ No newline at end of file
diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h
index ae389708f4..1d87285091 100644
--- a/source/libs/tdb/src/inc/tdbOs.h
+++ b/source/libs/tdb/src/inc/tdbOs.h
@@ -46,13 +46,14 @@ typedef TdFilePtr tdb_fd_t;
 
 #define tdbOsOpen(PATH, OPTION, MODE) taosOpenFile((PATH), (OPTION))
 
-#define tdbOsClose(FD) taosCloseFile(&(FD))
-#define tdbOsRead      taosReadFile
-#define tdbOsPRead     taosPReadFile
-#define tdbOsWrite     taosWriteFile
-#define tdbOsFSync     taosFsyncFile
-#define tdbOsLSeek     taosLSeekFile
-#define tdbOsRemove    remove
+#define tdbOsClose(FD)           taosCloseFile(&(FD))
+#define tdbOsRead                taosReadFile
+#define tdbOsPRead               taosPReadFile
+#define tdbOsWrite               taosWriteFile
+#define tdbOsFSync               taosFsyncFile
+#define tdbOsLSeek               taosLSeekFile
+#define tdbOsRemove              remove
+#define tdbOsFileSize(FD, PSIZE) taosFStatFile(FD, PSIZE, NULL)
 
 /* directory */
 #define tdbOsMkdir taosMkDir
@@ -110,10 +111,11 @@ i64 tdbOsWrite(tdb_fd_t fd, const void *pData, i64 nBytes);
 #define tdbOsFSync  fsync
 #define tdbOsLSeek  lseek
 #define tdbOsRemove remove
+#define tdbOsFileSize(FD, PSIZE)
 
 /* directory */
-#define tdbOsMkdir  mkdir
-#define tdbOsRmdir  rmdir
+#define tdbOsMkdir mkdir
+#define tdbOsRmdir rmdir
 
 // For threads and lock -----------------
 /* spin lock */
diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h
index 6abddb5b22..042026d0a4 100644
--- a/source/libs/tdb/src/inc/tdbUtil.h
+++ b/source/libs/tdb/src/inc/tdbUtil.h
@@ -29,6 +29,7 @@ extern "C" {
 #define TDB_ROUND8(x) (((x) + 7) & ~7)
 
 int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique);
+int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size);
 
 #define TDB_REALLOC(PTR, SIZE)                                                               \
   ({                                                                                         \

From e707047d6c9654ba68ebea2a3460b83d06413d28 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 09:39:05 +0000
Subject: [PATCH 13/16] fix a small bug

---
 source/dnode/vnode/src/meta/metaTDBImpl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c
index c273f584ff..3ab26c3d4d 100644
--- a/source/dnode/vnode/src/meta/metaTDBImpl.c
+++ b/source/dnode/vnode/src/meta/metaTDBImpl.c
@@ -547,7 +547,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) {
     return 0;
   }
 
-  pCtbIdxKey = pCtbCur->pVal;
+  pCtbIdxKey = pCtbCur->pKey;
 
   return pCtbIdxKey->uid;
 }

From f136211bad2d0b78e31ce032282c04c1249e10f3 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 10:21:36 +0000
Subject: [PATCH 14/16] fix TDB encode and decode bug

---
 source/libs/tdb/src/inc/tdbUtil.h    |  7 +++++--
 source/libs/tdb/test/CMakeLists.txt  |  6 +++++-
 source/libs/tdb/test/tdbUtilTest.cpp | 20 ++++++++++++++++++++
 3 files changed, 30 insertions(+), 3 deletions(-)
 create mode 100644 source/libs/tdb/test/tdbUtilTest.cpp

diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h
index 042026d0a4..7085e71567 100644
--- a/source/libs/tdb/src/inc/tdbUtil.h
+++ b/source/libs/tdb/src/inc/tdbUtil.h
@@ -84,15 +84,18 @@ static inline int tdbPutVarInt(u8 *p, int v) {
 static inline int tdbGetVarInt(const u8 *p, int *v) {
   int n = 0;
   int tv = 0;
+  int t;
 
   for (;;) {
     if (p[n] <= 0x7f) {
-      tv = (tv << 7) | p[n];
+      t = p[n];
+      tv |= (t << (7 * n));
       n++;
       break;
     }
 
-    tv = (tv << 7) | (p[n] & 0x7f);
+    t = p[n] & 0x7f;
+    tv |= (t << (7 * n));
     n++;
   }
 
diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt
index 2d77c1f4e9..b2c8aaf9bc 100644
--- a/source/libs/tdb/test/CMakeLists.txt
+++ b/source/libs/tdb/test/CMakeLists.txt
@@ -1,3 +1,7 @@
 # tdbTest
 add_executable(tdbTest "tdbTest.cpp")
-target_link_libraries(tdbTest tdb gtest gtest_main)
\ No newline at end of file
+target_link_libraries(tdbTest tdb gtest gtest_main)
+
+# tdbUtilTest
+add_executable(tdbUtilTest "tdbUtilTest.cpp")
+target_link_libraries(tdbUtilTest tdb gtest gtest_main)
\ No newline at end of file
diff --git a/source/libs/tdb/test/tdbUtilTest.cpp b/source/libs/tdb/test/tdbUtilTest.cpp
new file mode 100644
index 0000000000..5fd45daa3e
--- /dev/null
+++ b/source/libs/tdb/test/tdbUtilTest.cpp
@@ -0,0 +1,20 @@
+#include <gtest/gtest.h>
+
+#include "tdbInt.h"
+
+#include <string>
+
+TEST(tdb_util_test, simple_test) {
+  int vEncode = 5000;
+  int vDecode;
+  int nEncode;
+  int nDecode;
+  u8  buffer[128];
+
+  nEncode = tdbPutVarInt(buffer, vEncode);
+
+  nDecode = tdbGetVarInt(buffer, &vDecode);
+
+  GTEST_ASSERT_EQ(nEncode, nDecode);
+  GTEST_ASSERT_EQ(vEncode, vDecode);
+}
\ No newline at end of file

From eafc0f88bc5f6655a3906ede85ffcc9f15d43f29 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 11:00:52 +0000
Subject: [PATCH 15/16] remove tid from tsdb

---
 source/dnode/vnode/src/inc/tsdbReadImpl.h  | 32 +++++++++++-----------
 source/dnode/vnode/src/tsdb/tsdbCommit.c   |  1 -
 source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 31 +++++++++++----------
 3 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h
index 90a877bb83..682306221d 100644
--- a/source/dnode/vnode/src/inc/tsdbReadImpl.h
+++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h
@@ -17,12 +17,12 @@
 #define _TD_TSDB_READ_IMPL_H_
 
 #include "os.h"
+#include "tcommon.h"
 #include "tfs.h"
 #include "tsdb.h"
 #include "tsdbFile.h"
-#include "tskiplist.h"
 #include "tsdbMemory.h"
-#include "tcommon.h"
+#include "tskiplist.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -31,7 +31,6 @@ extern "C" {
 typedef struct SReadH SReadH;
 
 typedef struct {
-  int32_t  tid;
   uint32_t len;
   uint32_t offset;
   uint32_t hasLast : 2;
@@ -81,7 +80,7 @@ typedef struct {
   TSKEY    keyLast;
 } SBlockV0;
 
-#define SBlock          SBlockV0  // latest SBlock definition
+#define SBlock SBlockV0  // latest SBlock definition
 
 #endif
 
@@ -165,19 +164,19 @@ typedef struct {
 typedef void SAggrBlkData;  // SBlockCol cols[];
 
 struct SReadH {
-  STsdb *       pRepo;
+  STsdb        *pRepo;
   SDFileSet     rSet;     // FSET to read
-  SArray *      aBlkIdx;  // SBlockIdx array
-  STable *      pTable;   // table to read
-  SBlockIdx *   pBlkIdx;  // current reading table SBlockIdx
+  SArray       *aBlkIdx;  // SBlockIdx array
+  STable       *pTable;   // table to read
+  SBlockIdx    *pBlkIdx;  // current reading table SBlockIdx
   int           cidx;
-  SBlockInfo *  pBlkInfo;
-  SBlockData *  pBlkData;      // Block info
+  SBlockInfo   *pBlkInfo;
+  SBlockData   *pBlkData;      // Block info
   SAggrBlkData *pAggrBlkData;  // Aggregate Block info
-  SDataCols *   pDCols[2];
-  void *        pBuf;    // buffer
-  void *        pCBuf;   // compression buffer
-  void *        pExBuf;  // extra buffer
+  SDataCols    *pDCols[2];
+  void         *pBuf;    // buffer
+  void         *pCBuf;   // compression buffer
+  void         *pExBuf;  // extra buffer
 };
 
 #define TSDB_READ_REPO(rh)      ((rh)->pRepo)
@@ -222,14 +221,15 @@ int   tsdbLoadBlockIdx(SReadH *pReadh);
 int   tsdbSetReadTable(SReadH *pReadh, STable *pTable);
 int   tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget);
 int   tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
-int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds);
+int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds,
+                            int numOfColsIds);
 int   tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
 int   tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
 void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
 void  tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock);
 
 static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
-  void * pBuf = *ppBuf;
+  void  *pBuf = *ppBuf;
   size_t tsize = taosTSizeof(pBuf);
 
   if (tsize < size) {
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index 2a8875fefe..18d3de94cc 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -701,7 +701,6 @@ int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray
   // Set pIdx
   pBlock = taosArrayGetLast(pSupA);
 
-  pIdx->tid = TABLE_TID(pTable);
   pIdx->uid = TABLE_UID(pTable);
   pIdx->hasLast = pBlock->last ? 1 : 0;
   pIdx->maxKey = pBlock->keyLast;
diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c
index 8cb90d76a6..687e2ca776 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c
@@ -98,7 +98,7 @@ int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) {
 void tsdbCloseAndUnsetFSet(SReadH *pReadh) { tsdbResetReadFile(pReadh); }
 
 int tsdbLoadBlockIdx(SReadH *pReadh) {
-  SDFile *  pHeadf = TSDB_READ_HEAD_FILE(pReadh);
+  SDFile   *pHeadf = TSDB_READ_HEAD_FILE(pReadh);
   SBlockIdx blkIdx;
 
   ASSERT(taosArrayGetSize(pReadh->aBlkIdx) == 0);
@@ -149,8 +149,8 @@ int tsdbLoadBlockIdx(SReadH *pReadh) {
     }
 
     tsize++;
-    ASSERT(tsize == 1 || ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 2))->tid <
-                             ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 1))->tid);
+    // ASSERT(tsize == 1 || ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 2))->tid <
+    //                          ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 1))->tid);
   }
 
   return 0;
@@ -180,7 +180,7 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
       }
 
       SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx);
-      if (pBlkIdx->tid == TABLE_TID(pTable)) {
+      if (pBlkIdx->uid == TABLE_TID(pTable)) {
         if (pBlkIdx->uid == TABLE_UID(pTable)) {
           pReadh->pBlkIdx = pBlkIdx;
         } else {
@@ -188,7 +188,7 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
         }
         pReadh->cidx++;
         break;
-      } else if (pBlkIdx->tid > TABLE_TID(pTable)) {
+      } else if (pBlkIdx->uid > TABLE_TID(pTable)) {
         pReadh->pBlkIdx = NULL;
         break;
       } else {
@@ -205,7 +205,7 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
 int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) {
   ASSERT(pReadh->pBlkIdx != NULL);
 
-  SDFile *   pHeadf = TSDB_READ_HEAD_FILE(pReadh);
+  SDFile    *pHeadf = TSDB_READ_HEAD_FILE(pReadh);
   SBlockIdx *pBlkIdx = pReadh->pBlkIdx;
 
   if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) {
@@ -237,7 +237,7 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) {
     return -1;
   }
 
-  ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid);
+  // ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid);
 
   if (pTarget) {
     memcpy(pTarget, (void *)(pReadh->pBlkInfo), pBlkIdx->len);
@@ -275,7 +275,8 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
   return 0;
 }
 
-int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds) {
+int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds,
+                          int numOfColsIds) {
   ASSERT(pBlock->numOfSubBlocks > 0);
   int8_t update = pReadh->pRepo->config.update;
 
@@ -388,7 +389,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) {
 int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) {
   int tlen = 0;
 
-  tlen += taosEncodeVariantI32(buf, pIdx->tid);
+  // tlen += taosEncodeVariantI32(buf, pIdx->tid);
   tlen += taosEncodeVariantU32(buf, pIdx->len);
   tlen += taosEncodeVariantU32(buf, pIdx->offset);
   tlen += taosEncodeFixedU8(buf, pIdx->hasLast);
@@ -404,7 +405,7 @@ void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) {
   uint32_t numOfBlocks = 0;
   uint64_t value = 0;
 
-  if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL;
+  // if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL;
   if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL;
   if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL;
   if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL;
@@ -538,9 +539,9 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
   pDataCols->numOfRows = pBlock->numOfRows;
 
   // Recover the data
-  int ccol = 0;  // loop iter for SBlockCol object
-  int dcol = 0;  // loop iter for SDataCols object
-  int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows);
+  int        ccol = 0;  // loop iter for SBlockCol object
+  int        dcol = 0;  // loop iter for SDataCols object
+  int        nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows);
   SBlockCol *pBlockCol = NULL;
   while (dcol < pDataCols->numOfCols) {
     SDataCol *pDataCol = &(pDataCols->cols[dcol]);
@@ -686,7 +687,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
   ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
   ASSERT(colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID);
 
-  SDFile *  pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
+  SDFile   *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
   SBlockCol blockCol = {0};
 
   tdResetDataCols(pDataCols);
@@ -700,7 +701,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
   int ccol = 0;
   for (int i = 0; i < numOfColIds; i++) {
     int16_t    colId = colIds[i];
-    SDataCol * pDataCol = NULL;
+    SDataCol  *pDataCol = NULL;
     SBlockCol *pBlockCol = NULL;
 
     while (true) {

From 70810ddb39382910b950390897d582cc30eecd90 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Wed, 6 Apr 2022 11:37:34 +0000
Subject: [PATCH 16/16] fix a small bug

---
 source/dnode/vnode/src/meta/metaTDBImpl.c |  8 ++++++++
 source/libs/tdb/src/db/tdbBtree.c         | 20 +++++++++++---------
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c
index 3ab26c3d4d..812ec84a9a 100644
--- a/source/dnode/vnode/src/meta/metaTDBImpl.c
+++ b/source/dnode/vnode/src/meta/metaTDBImpl.c
@@ -226,6 +226,14 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) {
     uid = metaGenerateUid(pMeta);
   }
 
+  // check name and uid unique
+  if (tdbDbGet(pMetaDb->pTbDB, &uid, sizeof(uid), NULL, NULL) == 0) {
+    return -1;
+  }
+  if (tdbDbGet(pMetaDb->pNameIdx, pTbCfg->name, strlen(pTbCfg->name) + 1, NULL, NULL) == 0) {
+    return -1;
+  }
+
   // save to table.db
   pKey = &uid;
   kLen = sizeof(uid);
diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index f760598714..9df749bec7 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -233,7 +233,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
     ASSERT(0);
   }
 
-  if (cret) {
+  if (btc.idx < 0 || cret) {
     tdbBtcClose(&btc);
     return -1;
   }
@@ -253,15 +253,17 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL
     memcpy(*ppKey, cd.pKey, cd.kLen);
   }
 
-  pTVal = TDB_REALLOC(*ppVal, cd.vLen);
-  if (pTVal == NULL) {
-    tdbBtcClose(&btc);
-    ASSERT(0);
-    return -1;
+  if (ppVal) {
+    pTVal = TDB_REALLOC(*ppVal, cd.vLen);
+    if (pTVal == NULL) {
+      tdbBtcClose(&btc);
+      ASSERT(0);
+      return -1;
+    }
+    *ppVal = pTVal;
+    *vLen = cd.vLen;
+    memcpy(*ppVal, cd.pVal, cd.vLen);
   }
-  *ppVal = pTVal;
-  *vLen = cd.vLen;
-  memcpy(*ppVal, cd.pVal, cd.vLen);
 
   tdbBtcClose(&btc);