From a94d6aef031405bb3b870acd5298a2043ddbe270 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 7 Mar 2025 14:29:29 +0800 Subject: [PATCH 1/2] fix(tdb): restrict tdb allocation from buffer pool --- source/dnode/vnode/src/meta/metaCommit.c | 9 + source/libs/tdb/src/db/tdbPCache.c | 5 +- source/libs/tdb/src/db/tdbTable.c | 9 +- source/libs/tdb/test/CMakeLists.txt | 3 + source/libs/tdb/test/tdbPageFlushTest.cpp | 1115 +++++++++++++++++++++ 5 files changed, 1138 insertions(+), 3 deletions(-) create mode 100644 source/libs/tdb/test/tdbPageFlushTest.cpp diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index 6ed4991679..3b9857a6a2 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -14,10 +14,19 @@ */ #include "meta.h" +#include "vnd.h" static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { + SVBufPool *pool = (SVBufPool *)pPool; + SVnode *pVnode = pool->pVnode; + + if (pVnode->inUse && pVnode->inUse->size > pVnode->inUse->node.size) { + return NULL; + } + return vnodeBufPoolMallocAligned((SVBufPool *)pPool, size); } + static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } // begin a meta txn diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index b0bcbd1a4c..23e6e5414e 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -345,7 +345,9 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) if (!pPage && pTxn->xMalloc != NULL) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); if (ret < 0 || pPage == NULL) { - tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); + // when allocating from bufpool failed, it's time to flush cache. + // tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); + terrno = ret; return NULL; } @@ -551,5 +553,4 @@ static void tdbPCacheCloseImpl(SPCache *pCache) { tdbOsFree(pCache->pgHash); tdbPCacheDestroyLock(pCache); - return ; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 6dc6aa0940..4c73d6470e 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -104,7 +104,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF } #endif - + /* if (rollback) { ret = tdbPagerRestoreJournals(pPager); if (ret < 0) { @@ -118,6 +118,13 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF return ret; } } + */ + // Always restore journal files with page flushing + ret = tdbPagerRestoreJournals(pPager); + if (ret < 0) { + tdbOsFree(pTb); + return ret; + } // pTb->pBt ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt)); diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index 4715ccbd41..6319fce86b 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -18,3 +18,6 @@ target_link_libraries(tdbPageDefragmentTest tdb gtest gtest_main) add_executable(tdbPageRecycleTest "tdbPageRecycleTest.cpp") target_link_libraries(tdbPageRecycleTest tdb gtest gtest_main) +# page flush testing +add_executable(tdbPageFlushTest "tdbPageFlushTest.cpp") +target_link_libraries(tdbPageFlushTest tdb gtest gtest_main) diff --git a/source/libs/tdb/test/tdbPageFlushTest.cpp b/source/libs/tdb/test/tdbPageFlushTest.cpp new file mode 100644 index 0000000000..6aa69b4b1d --- /dev/null +++ b/source/libs/tdb/test/tdbPageFlushTest.cpp @@ -0,0 +1,1115 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#define ALLOW_FORBID_FUNC +#include "os.h" +#include "tdb.h" + +#include +#include +#include +#include +#include "tlog.h" + +typedef struct SPoolMem { + int64_t size; + struct SPoolMem *prev; + struct SPoolMem *next; +} SPoolMem; + +static SPoolMem *openPool() { + SPoolMem *pPool = (SPoolMem *)taosMemoryMalloc(sizeof(*pPool)); + + pPool->prev = pPool->next = pPool; + pPool->size = 0; + + return pPool; +} + +static void clearPool(SPoolMem *pPool) { + SPoolMem *pMem; + + do { + pMem = pPool->next; + + if (pMem == pPool) break; + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); + } while (1); + + assert(pPool->size == 0); +} + +static void closePool(SPoolMem *pPool) { + clearPool(pPool); + taosMemoryFree(pPool); +} + +static void *poolMalloc(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void *poolMallocRestricted(void *arg, size_t size) { + void *ptr = NULL; + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + if (pPool->size > 1024 * 1024 * 10) { + return NULL; + } + + pMem = (SPoolMem *)taosMemoryMalloc(sizeof(*pMem) + size); + if (pMem == NULL) { + assert(0); + } + + pMem->size = sizeof(*pMem) + size; + pMem->next = pPool->next; + pMem->prev = pPool; + + pPool->next->prev = pMem; + pPool->next = pMem; + pPool->size += pMem->size; + + ptr = (void *)(&pMem[1]); + return ptr; +} + +static void poolFree(void *arg, void *ptr) { + SPoolMem *pPool = (SPoolMem *)arg; + SPoolMem *pMem; + + pMem = &(((SPoolMem *)ptr)[-1]); + + pMem->next->prev = pMem->prev; + pMem->prev->next = pMem->next; + pPool->size -= pMem->size; + + taosMemoryFree(pMem); +} + +static int tDefaultKeyCmpr(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { + int mlen; + int cret; + + ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + + mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; + cret = memcmp(pKey1, pKey2, mlen); + if (cret == 0) { + if (keyLen1 < keyLen2) { + cret = -1; + } else if (keyLen1 > keyLen2) { + cret = 1; + } else { + cret = 0; + } + } + return cret; +} + +static int tKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) { + int k1, k2; + + std::string s1((char *)pKey1 + 3, kLen1 - 3); + std::string s2((char *)pKey2 + 3, kLen2 - 3); + k1 = stoi(s1); + k2 = stoi(s2); + + if (k1 < k2) { + return -1; + } else if (k1 > k2) { + return 1; + } else { + return 0; + } +} + +static TDB *openEnv(char const *envName, int const pageSize, int const pageNum) { + TDB *pEnv = NULL; + + int ret = tdbOpen(envName, pageSize, pageNum, &pEnv, 0, 0, NULL); + if (ret) { + pEnv = NULL; + } + + return pEnv; +} + +static void clearDb(char const *db) { taosRemoveDir(db); } + +static void generateBigVal(char *val, int valLen) { + for (int i = 0; i < valLen; ++i) { + char c = char(i & 0xff); + if (c == 0) { + c = 1; + } + val[i] = c; + } +} + +static void insertOfp(void) { + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + char const *key = "key123456789"; + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +static void insertMultipleOfp(void) { + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest) { +TEST(TdbPageFlushTest, TbRestoreTest) { + clearDb("tdb"); + + insertMultipleOfp(); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest2) { +TEST(TdbPageFlushTest, TbRestoreTest2) { + clearDb("tdb"); + + int ret = 0; + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); + + // exit in the middle of page flushing +} + +TEST(TdbPageFlushTest, DISABLED_TbRestoreTest3) { + // TEST(TdbPageFlushTest, TbRestoreTest3) { + // open db + + // complete the inserting after journal restoring + + // tranversing the completed db + + // close db + + // reopen for tranvering again + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // int ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + int ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + { // Query the data + void *pVal = NULL; + int vLen; + + // char const *key = "key1"; + char const *key = "key123456789"; + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, valLen); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + + tdbFree(pVal); + } + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest3) { +TEST(TdbPageFlushTest, TbRestoreTest3) { + // delete db with page flushing + + // exit in the middle of page flushing +} + +TEST(TdbPageFlushTest, DISABLED_TbRestoreTest4) { + // TEST(TdbPageFlushTest, TbRestoreTest4) { + // open db + + // complete the deleting after journal restoring + + // tranversing the deleted db + + // close db + + // reopen for tranvering again + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // int ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + int ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + { // Query the data + void *pVal = NULL; + int vLen; + + // char const *key = "key1"; + char const *key = "key123456789"; + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, valLen); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + + tdbFree(pVal); + } + + // commit current transaction + // tdbCommit(pEnv, txn); + // tdbPostCommit(pEnv, txn); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); + + clearDb("tdb"); +} + +TEST(TdbPageFlushTest, DISABLED_TbInsertTest) { + // TEST(TdbPageFlushTest, TbInsertTest) { + clearDb("tdb"); + insertOfp(); +} + +TEST(TdbPageFlushTest, DISABLED_TbGetTest) { + // TEST(TdbPageFlushTest, TbGetTest) { + clearDb("tdb"); + insertOfp(); + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + // int ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + int ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + { // Query the data + void *pVal = NULL; + int vLen; + + // char const *key = "key1"; + char const *key = "key123456789"; + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, valLen); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + + tdbFree(pVal); + } +} + +TEST(TdbPageFlushTest, DISABLED_TbDeleteTest) { + // TEST(TdbPageFlushTest, TbDeleteTest) { + int ret = 0; + + taosRemoveDir("tdb"); + + // open Env + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[((4083 - 4 - 3 - 2) + 1) * 2]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + { // insert the generated big data + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the data + void *pVal = NULL; + int vLen; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, valLen); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + + tdbFree(pVal); + } + /* open to debug committed file +tdbCommit(pEnv, &txn); +tdbTxnClose(&txn); + +++txnid; +tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); +tdbBegin(pEnv, &txn); + */ + { // upsert the data + ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the upserted data + void *pVal = NULL; + int vLen; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, strlen("value1")); + GTEST_ASSERT_EQ(memcmp("value1", pVal, vLen), 0); + + tdbFree(pVal); + } + + { // delete the data + ret = tdbTbDelete(pDb, "key1", strlen("key1"), txn); + GTEST_ASSERT_EQ(ret, 0); + } + + { // query the deleted data + void *pVal = NULL; + int vLen = -1; + + ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); + ASSERT(ret == -1); + GTEST_ASSERT_EQ(ret, -1); + + GTEST_ASSERT_EQ(vLen, -1); + GTEST_ASSERT_EQ(pVal, nullptr); + + tdbFree(pVal); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); +} + +TEST(TdbPageFlushTest, DISABLED_simple_insert1) { + // TEST(TdbPageFlushTest, simple_insert1) { + int ret; + TDB *pEnv; + TTB *pDb; + tdb_cmpr_fn_t compFunc; + int nData = 1; + TXN *txn; + int const pageSize = 4096; + + taosRemoveDir("tdb"); + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + { + char key[64]; + // char val[(4083 - 4 - 3 - 2)]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 1; iData <= nData; iData++) { + sprintf(key, "key0"); + sprintf(val, "value%d", iData); + + // ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + // GTEST_ASSERT_EQ(ret, 0); + + // generate value payload + int valLen = sizeof(val) / sizeof(val[0]); + for (int i = 6; i < valLen; ++i) { + char c = char(i & 0xff); + if (c == 0) { + c = 1; + } + val[i] = c; + } + + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + { // Query the data + void *pVal = NULL; + int vLen; + + for (int i = 1; i <= nData; i++) { + sprintf(key, "key%d", i); + // sprintf(val, "value%d", i); + + ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); + ASSERT(ret == 0); + GTEST_ASSERT_EQ(ret, 0); + + GTEST_ASSERT_EQ(vLen, sizeof(val) / sizeof(val[0])); + GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); + } + + tdbFree(pVal); + } + + { // Iterate to query the DB data + TBC *pDBC; + void *pKey = NULL; + void *pVal = NULL; + int vLen, kLen; + int count = 0; + + ret = tdbTbcOpen(pDb, &pDBC, NULL); + GTEST_ASSERT_EQ(ret, 0); + + tdbTbcMoveToFirst(pDBC); + + for (;;) { + ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + + // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; + // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; + // std::cout << std::endl; + + count++; + } + + GTEST_ASSERT_EQ(count, nData); + + tdbTbcClose(pDBC); + + tdbFree(pKey); + tdbFree(pVal); + } + } + + ret = tdbTbDrop(pDb); + GTEST_ASSERT_EQ(ret, 0); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +static void insertDb(int nData) { + int ret = 0; + TDB *pEnv = NULL; + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc; + TXN *txn = NULL; + int const pageSize = 4 * 1024; + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // 1, insert nData kv + { + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 0; iData < nData; ++iData) { + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); + GTEST_ASSERT_EQ(ret, 0); + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // 2, delete nData/2 records + + closePool(pPool); + } + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); + + system("ls -l ./tdb"); +} + +static void deleteDb(int nData) { + int ret = 0; + TDB *pEnv = NULL; + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc; + TXN *txn = NULL; + int const pageSize = 4 * 1024; + + // Open Env + ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); + GTEST_ASSERT_EQ(ret, 0); + + // Create a database + compFunc = tKeyCmpr; + ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // 2, delete nData/2 records + { + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + SPoolMem *pPool; + + // open the pool + pPool = openPool(); + + // start a transaction + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int iData = 0; iData < nData; iData++) { + // if (iData % 2 == 0) continue; + + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + { // delete the data + ret = tdbTbDelete(pDb, key, strlen(key), txn); + GTEST_ASSERT_EQ(ret, 0); + } + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + + // commit the transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + } + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); + + system("ls -l ./tdb"); +} + +static const int nDataConst = 256 * 19; + +// TEST(TdbPageFlushTest, DISABLED_seq_insert) { +TEST(TdbPageFlushTest, seq_insert) { + clearDb("tdb"); + insertDb(nDataConst); +} + +// TEST(TdbPageFlushTest, DISABLED_seq_delete) { +TEST(TdbPageFlushTest, seq_delete) { deleteDb(nDataConst); } + +// TEST(TdbPageFlushTest, DISABLED_recycly_insert) { +TEST(TdbPageFlushTest, recycly_insert) { insertDb(nDataConst); } + +// TEST(TdbPageFlushTest, DISABLED_recycly_seq_insert_ofp) { +TEST(TdbPageFlushTest, recycly_seq_insert_ofp) { + clearDb("tdb"); + insertOfp(); + system("ls -l ./tdb"); +} + +static void deleteOfp(void) { + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + { // delete the data + char const *key = "key123456789"; + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + ret = tdbTbDrop(pDb); + GTEST_ASSERT_EQ(ret, 0); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); +} + +// TEST(TdbPageFlushTest, DISABLED_seq_delete_ofp) { +TEST(TdbPageFlushTest, seq_delete_ofp) { + deleteOfp(); + system("ls -l ./tdb"); +} + +// TEST(TdbPageFlushTest, DISABLED_recycly_seq_insert_ofp_again) { +TEST(TdbPageFlushTest, recycly_seq_insert_ofp_again) { + insertOfp(); + system("ls -l ./tdb"); +} + +// TEST(TdbPageFlushTest, DISABLED_recycly_seq_insert_ofp_nocommit) { +TEST(TdbPageFlushTest, recycly_seq_insert_ofp_nocommit) { + clearDb("tdb"); + insertOfp(); + system("ls -l ./tdb"); + + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + { // delete the data + char const *key = "key123456789"; + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + + // 1, insert nData kv + { + int nData = nDataConst; + char key[64]; + char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + int64_t poolLimit = 4096; // 1M pool limit + + for (int iData = 0; iData < nData; ++iData) { + sprintf(key, "key%03d", iData); + sprintf(val, "value%03d", iData); + + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); + GTEST_ASSERT_EQ(ret, 0); + // if pool is full, commit the transaction and start a new one + if (pPool->size >= poolLimit) { + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + // start a new transaction + clearPool(pPool); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + } + } + } + + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); + + system("ls -l ./tdb"); +} + +// TEST(TdbPageFlushTest, DISABLED_recycly_delete_interior_ofp_nocommit) { +TEST(TdbPageFlushTest, recycly_delete_interior_ofp_nocommit) { + clearDb("tdb"); + + // open Env + int ret = 0; + int const pageSize = 4096; + int const pageNum = 64; + TDB *pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + // open db + TTB *pDb = NULL; + tdb_cmpr_fn_t compFunc = NULL; // tKeyCmpr; + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + // open the pool + SPoolMem *pPool = openPool(); + + // start a transaction + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + char key[1024] = {0}; + int count = sizeof(key) / sizeof(key[0]); + for (int i = 0; i < count - 1; ++i) { + key[i] = 'a'; + } + + // insert n ofp keys to form 2-layer btree + { + for (int i = 0; i < 7; ++i) { + // sprintf(&key[count - 2], "%c", i); + key[count - 2] = '0' + i; + + ret = tdbTbInsert(pDb, key, count, NULL, 0, txn); + GTEST_ASSERT_EQ(ret, 0); + } + } + /* + // delete one interior key + { + sprintf(&key[count - 2], "%c", 2); + key[count - 2] = '0' + 2; + + ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + GTEST_ASSERT_EQ(ret, 0); + } + */ + // commit current transaction + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); + + closePool(pPool); + + // Close a database + tdbTbClose(pDb); + + // Close Env + tdbClose(pEnv); + + system("ls -l ./tdb"); +} From 3574fdb4160f4d8df1264586cacdda1b4d89c6e8 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 7 Mar 2025 14:44:44 +0800 Subject: [PATCH 2/2] refactor: new restore after abort testing case --- source/libs/tdb/test/tdbPageFlushTest.cpp | 715 +++------------------- 1 file changed, 91 insertions(+), 624 deletions(-) diff --git a/source/libs/tdb/test/tdbPageFlushTest.cpp b/source/libs/tdb/test/tdbPageFlushTest.cpp index 6aa69b4b1d..170a28af76 100644 --- a/source/libs/tdb/test/tdbPageFlushTest.cpp +++ b/source/libs/tdb/test/tdbPageFlushTest.cpp @@ -357,182 +357,14 @@ TEST(TdbPageFlushTest, TbRestoreTest2) { // Close Env tdbClose(pEnv); - - // exit in the middle of page flushing -} - -TEST(TdbPageFlushTest, DISABLED_TbRestoreTest3) { - // TEST(TdbPageFlushTest, TbRestoreTest3) { - // open db - - // complete the inserting after journal restoring - - // tranversing the completed db - - // close db - - // reopen for tranvering again - // open Env - int const pageSize = 4096; - int const pageNum = 64; - TDB *pEnv = openEnv("tdb", pageSize, pageNum); - GTEST_ASSERT_NE(pEnv, nullptr); - - // open db - TTB *pDb = NULL; - tdb_cmpr_fn_t compFunc = tKeyCmpr; - // int ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); - int ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0); - GTEST_ASSERT_EQ(ret, 0); - - // generate value payload - // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) - char val[32605]; - int valLen = sizeof(val) / sizeof(val[0]); - generateBigVal(val, valLen); - - { // Query the data - void *pVal = NULL; - int vLen; - - // char const *key = "key1"; - char const *key = "key123456789"; - ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); - ASSERT(ret == 0); - GTEST_ASSERT_EQ(ret, 0); - - GTEST_ASSERT_EQ(vLen, valLen); - GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); - - tdbFree(pVal); - } - - // Close a database - tdbTbClose(pDb); - - // Close Env - tdbClose(pEnv); } // TEST(TdbPageFlushTest, DISABLED_TbRestoreTest3) { TEST(TdbPageFlushTest, TbRestoreTest3) { - // delete db with page flushing - - // exit in the middle of page flushing -} - -TEST(TdbPageFlushTest, DISABLED_TbRestoreTest4) { - // TEST(TdbPageFlushTest, TbRestoreTest4) { - // open db - - // complete the deleting after journal restoring - - // tranversing the deleted db - - // close db - - // reopen for tranvering again - // open Env - int const pageSize = 4096; - int const pageNum = 64; - TDB *pEnv = openEnv("tdb", pageSize, pageNum); - GTEST_ASSERT_NE(pEnv, nullptr); - - // open db - TTB *pDb = NULL; - tdb_cmpr_fn_t compFunc = tKeyCmpr; - // int ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); - int ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0); - GTEST_ASSERT_EQ(ret, 0); - - // generate value payload - // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) - char val[32605]; - int valLen = sizeof(val) / sizeof(val[0]); - generateBigVal(val, valLen); - - { // Query the data - void *pVal = NULL; - int vLen; - - // char const *key = "key1"; - char const *key = "key123456789"; - ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); - ASSERT(ret == 0); - GTEST_ASSERT_EQ(ret, 0); - - GTEST_ASSERT_EQ(vLen, valLen); - GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); - - tdbFree(pVal); - } - - // commit current transaction - // tdbCommit(pEnv, txn); - // tdbPostCommit(pEnv, txn); - - // Close a database - tdbTbClose(pDb); - - // Close Env - tdbClose(pEnv); - clearDb("tdb"); -} -TEST(TdbPageFlushTest, DISABLED_TbInsertTest) { - // TEST(TdbPageFlushTest, TbInsertTest) { - clearDb("tdb"); - insertOfp(); -} - -TEST(TdbPageFlushTest, DISABLED_TbGetTest) { - // TEST(TdbPageFlushTest, TbGetTest) { - clearDb("tdb"); - insertOfp(); - - // open Env - int const pageSize = 4096; - int const pageNum = 64; - TDB *pEnv = openEnv("tdb", pageSize, pageNum); - GTEST_ASSERT_NE(pEnv, nullptr); - - // open db - TTB *pDb = NULL; - tdb_cmpr_fn_t compFunc = tKeyCmpr; - // int ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); - int ret = tdbTbOpen("ofp_insert.db", 12, -1, compFunc, pEnv, &pDb, 0); - GTEST_ASSERT_EQ(ret, 0); - - // generate value payload - // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) - char val[32605]; - int valLen = sizeof(val) / sizeof(val[0]); - generateBigVal(val, valLen); - - { // Query the data - void *pVal = NULL; - int vLen; - - // char const *key = "key1"; - char const *key = "key123456789"; - ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); - ASSERT(ret == 0); - GTEST_ASSERT_EQ(ret, 0); - - GTEST_ASSERT_EQ(vLen, valLen); - GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); - - tdbFree(pVal); - } -} - -TEST(TdbPageFlushTest, DISABLED_TbDeleteTest) { - // TEST(TdbPageFlushTest, TbDeleteTest) { int ret = 0; - taosRemoveDir("tdb"); - // open Env int const pageSize = 4096; int const pageNum = 64; @@ -542,6 +374,7 @@ TEST(TdbPageFlushTest, DISABLED_TbDeleteTest) { // open db TTB *pDb = NULL; tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); GTEST_ASSERT_EQ(ret, 0); @@ -549,209 +382,42 @@ TEST(TdbPageFlushTest, DISABLED_TbDeleteTest) { SPoolMem *pPool = openPool(); // start a transaction - TXN *txn; + TXN *txn = NULL; - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); // generate value payload // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) - char val[((4083 - 4 - 3 - 2) + 1) * 2]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; int valLen = sizeof(val) / sizeof(val[0]); generateBigVal(val, valLen); - { // insert the generated big data - ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); GTEST_ASSERT_EQ(ret, 0); } - { // query the data - void *pVal = NULL; - int vLen; - - ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); - ASSERT(ret == 0); - GTEST_ASSERT_EQ(ret, 0); - - GTEST_ASSERT_EQ(vLen, valLen); - GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); - - tdbFree(pVal); - } - /* open to debug committed file -tdbCommit(pEnv, &txn); -tdbTxnClose(&txn); - -++txnid; -tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); -tdbBegin(pEnv, &txn); - */ - { // upsert the data - ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), txn); - GTEST_ASSERT_EQ(ret, 0); - } - - { // query the upserted data - void *pVal = NULL; - int vLen; - - ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); - ASSERT(ret == 0); - GTEST_ASSERT_EQ(ret, 0); - - GTEST_ASSERT_EQ(vLen, strlen("value1")); - GTEST_ASSERT_EQ(memcmp("value1", pVal, vLen), 0); - - tdbFree(pVal); - } - - { // delete the data - ret = tdbTbDelete(pDb, "key1", strlen("key1"), txn); - GTEST_ASSERT_EQ(ret, 0); - } - - { // query the deleted data - void *pVal = NULL; - int vLen = -1; - - ret = tdbTbGet(pDb, "key1", strlen("key1"), &pVal, &vLen); - ASSERT(ret == -1); - GTEST_ASSERT_EQ(ret, -1); - - GTEST_ASSERT_EQ(vLen, -1); - GTEST_ASSERT_EQ(pVal, nullptr); - - tdbFree(pVal); - } - // commit current transaction + tdbAbort(pEnv, txn); + + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); -} -TEST(TdbPageFlushTest, DISABLED_simple_insert1) { - // TEST(TdbPageFlushTest, simple_insert1) { - int ret; - TDB *pEnv; - TTB *pDb; - tdb_cmpr_fn_t compFunc; - int nData = 1; - TXN *txn; - int const pageSize = 4096; - - taosRemoveDir("tdb"); - - // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); - GTEST_ASSERT_EQ(ret, 0); - - // Create a database - compFunc = tKeyCmpr; - ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); - GTEST_ASSERT_EQ(ret, 0); - - { - char key[64]; - // char val[(4083 - 4 - 3 - 2)]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) - char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) - int64_t poolLimit = 4096; // 1M pool limit - SPoolMem *pPool; - - // open the pool - pPool = openPool(); - - // start a transaction - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - - for (int iData = 1; iData <= nData; iData++) { - sprintf(key, "key0"); - sprintf(val, "value%d", iData); - - // ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); - // GTEST_ASSERT_EQ(ret, 0); - - // generate value payload - int valLen = sizeof(val) / sizeof(val[0]); - for (int i = 6; i < valLen; ++i) { - char c = char(i & 0xff); - if (c == 0) { - c = 1; - } - val[i] = c; - } - - ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); - GTEST_ASSERT_EQ(ret, 0); - - // if pool is full, commit the transaction and start a new one - if (pPool->size >= poolLimit) { - // commit current transaction - tdbCommit(pEnv, txn); - tdbPostCommit(pEnv, txn); - - // start a new transaction - clearPool(pPool); - - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - } - } - - // commit the transaction - tdbCommit(pEnv, txn); - tdbPostCommit(pEnv, txn); - - { // Query the data - void *pVal = NULL; - int vLen; - - for (int i = 1; i <= nData; i++) { - sprintf(key, "key%d", i); - // sprintf(val, "value%d", i); - - ret = tdbTbGet(pDb, key, strlen(key), &pVal, &vLen); - ASSERT(ret == 0); - GTEST_ASSERT_EQ(ret, 0); - - GTEST_ASSERT_EQ(vLen, sizeof(val) / sizeof(val[0])); - GTEST_ASSERT_EQ(memcmp(val, pVal, vLen), 0); - } - - tdbFree(pVal); - } - - { // Iterate to query the DB data - TBC *pDBC; - void *pKey = NULL; - void *pVal = NULL; - int vLen, kLen; - int count = 0; - - ret = tdbTbcOpen(pDb, &pDBC, NULL); - GTEST_ASSERT_EQ(ret, 0); - - tdbTbcMoveToFirst(pDBC); - - for (;;) { - ret = tdbTbcNext(pDBC, &pKey, &kLen, &pVal, &vLen); - if (ret < 0) break; - - // std::cout.write((char *)pKey, kLen) /* << " " << kLen */ << " "; - // std::cout.write((char *)pVal, vLen) /* << " " << vLen */; - // std::cout << std::endl; - - count++; - } - - GTEST_ASSERT_EQ(count, nData); - - tdbTbcClose(pDBC); - - tdbFree(pKey); - tdbFree(pVal); - } - } - - ret = tdbTbDrop(pDb); - GTEST_ASSERT_EQ(ret, 0); + closePool(pPool); // Close a database tdbTbClose(pDb); @@ -760,166 +426,13 @@ TEST(TdbPageFlushTest, DISABLED_simple_insert1) { tdbClose(pEnv); } -static void insertDb(int nData) { - int ret = 0; - TDB *pEnv = NULL; - TTB *pDb = NULL; - tdb_cmpr_fn_t compFunc; - TXN *txn = NULL; - int const pageSize = 4 * 1024; - - // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); - GTEST_ASSERT_EQ(ret, 0); - - // Create a database - compFunc = tKeyCmpr; - ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); - GTEST_ASSERT_EQ(ret, 0); - - // 1, insert nData kv - { - char key[64]; - char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) - int64_t poolLimit = 4096; // 1M pool limit - SPoolMem *pPool; - - // open the pool - pPool = openPool(); - - // start a transaction - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - - for (int iData = 0; iData < nData; ++iData) { - sprintf(key, "key%03d", iData); - sprintf(val, "value%03d", iData); - - ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); - GTEST_ASSERT_EQ(ret, 0); - // if pool is full, commit the transaction and start a new one - if (pPool->size >= poolLimit) { - // commit current transaction - tdbCommit(pEnv, txn); - tdbPostCommit(pEnv, txn); - - // start a new transaction - clearPool(pPool); - - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - } - } - - // commit the transaction - tdbCommit(pEnv, txn); - tdbPostCommit(pEnv, txn); - - // 2, delete nData/2 records - - closePool(pPool); - } - - // Close a database - tdbTbClose(pDb); - - // Close Env - tdbClose(pEnv); - - system("ls -l ./tdb"); -} - -static void deleteDb(int nData) { - int ret = 0; - TDB *pEnv = NULL; - TTB *pDb = NULL; - tdb_cmpr_fn_t compFunc; - TXN *txn = NULL; - int const pageSize = 4 * 1024; - - // Open Env - ret = tdbOpen("tdb", pageSize, 64, &pEnv, 0, 0, NULL); - GTEST_ASSERT_EQ(ret, 0); - - // Create a database - compFunc = tKeyCmpr; - ret = tdbTbOpen("db.db", -1, -1, compFunc, pEnv, &pDb, 0); - GTEST_ASSERT_EQ(ret, 0); - - // 2, delete nData/2 records - { - char key[64]; - char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) - int64_t poolLimit = 4096; // 1M pool limit - SPoolMem *pPool; - - // open the pool - pPool = openPool(); - - // start a transaction - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - - for (int iData = 0; iData < nData; iData++) { - // if (iData % 2 == 0) continue; - - sprintf(key, "key%03d", iData); - sprintf(val, "value%03d", iData); - - { // delete the data - ret = tdbTbDelete(pDb, key, strlen(key), txn); - GTEST_ASSERT_EQ(ret, 0); - } - // if pool is full, commit the transaction and start a new one - if (pPool->size >= poolLimit) { - // commit current transaction - tdbCommit(pEnv, txn); - tdbPostCommit(pEnv, txn); - - // start a new transaction - clearPool(pPool); - - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - } - } - - // commit the transaction - tdbCommit(pEnv, txn); - tdbPostCommit(pEnv, txn); - - closePool(pPool); - } - - // Close a database - tdbTbClose(pDb); - - // Close Env - tdbClose(pEnv); - - system("ls -l ./tdb"); -} - -static const int nDataConst = 256 * 19; - -// TEST(TdbPageFlushTest, DISABLED_seq_insert) { -TEST(TdbPageFlushTest, seq_insert) { +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest4) { +TEST(TdbPageFlushTest, TbRestoreTest4) { clearDb("tdb"); - insertDb(nDataConst); -} -// TEST(TdbPageFlushTest, DISABLED_seq_delete) { -TEST(TdbPageFlushTest, seq_delete) { deleteDb(nDataConst); } + int ret = 0; -// TEST(TdbPageFlushTest, DISABLED_recycly_insert) { -TEST(TdbPageFlushTest, recycly_insert) { insertDb(nDataConst); } - -// TEST(TdbPageFlushTest, DISABLED_recycly_seq_insert_ofp) { -TEST(TdbPageFlushTest, recycly_seq_insert_ofp) { - clearDb("tdb"); - insertOfp(); - system("ls -l ./tdb"); -} - -static void deleteOfp(void) { // open Env - int ret = 0; int const pageSize = 4096; int const pageNum = 64; TDB *pEnv = openEnv("tdb", pageSize, pageNum); @@ -928,6 +441,7 @@ static void deleteOfp(void) { // open db TTB *pDb = NULL; tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); GTEST_ASSERT_EQ(ret, 0); @@ -935,25 +449,53 @@ static void deleteOfp(void) { SPoolMem *pPool = openPool(); // start a transaction - TXN *txn; + TXN *txn = NULL; - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - { // delete the data - char const *key = "key123456789"; - ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); GTEST_ASSERT_EQ(ret, 0); } // commit current transaction + tdbAbort(pEnv, txn); + closePool(pPool); + tdbTbClose(pDb); + tdbClose(pEnv); + + pEnv = openEnv("tdb", pageSize, pageNum); + GTEST_ASSERT_NE(pEnv, nullptr); + + ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); + GTEST_ASSERT_EQ(ret, 0); + + pPool = openPool(); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); + GTEST_ASSERT_EQ(ret, 0); + } + tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); closePool(pPool); - ret = tdbTbDrop(pDb); - GTEST_ASSERT_EQ(ret, 0); - // Close a database tdbTbClose(pDb); @@ -961,26 +503,13 @@ static void deleteOfp(void) { tdbClose(pEnv); } -// TEST(TdbPageFlushTest, DISABLED_seq_delete_ofp) { -TEST(TdbPageFlushTest, seq_delete_ofp) { - deleteOfp(); - system("ls -l ./tdb"); -} - -// TEST(TdbPageFlushTest, DISABLED_recycly_seq_insert_ofp_again) { -TEST(TdbPageFlushTest, recycly_seq_insert_ofp_again) { - insertOfp(); - system("ls -l ./tdb"); -} - -// TEST(TdbPageFlushTest, DISABLED_recycly_seq_insert_ofp_nocommit) { -TEST(TdbPageFlushTest, recycly_seq_insert_ofp_nocommit) { +// TEST(TdbPageFlushTest, DISABLED_TbRestoreTest5) { +TEST(TdbPageFlushTest, TbRestoreTest5) { clearDb("tdb"); - insertOfp(); - system("ls -l ./tdb"); + + int ret = 0; // open Env - int ret = 0; int const pageSize = 4096; int const pageNum = 64; TDB *pEnv = openEnv("tdb", pageSize, pageNum); @@ -989,6 +518,7 @@ TEST(TdbPageFlushTest, recycly_seq_insert_ofp_nocommit) { // open db TTB *pDb = NULL; tdb_cmpr_fn_t compFunc = tKeyCmpr; + // ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); GTEST_ASSERT_EQ(ret, 0); @@ -996,110 +526,49 @@ TEST(TdbPageFlushTest, recycly_seq_insert_ofp_nocommit) { SPoolMem *pPool = openPool(); // start a transaction - TXN *txn; + TXN *txn = NULL; - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - { // delete the data - char const *key = "key123456789"; - ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + // generate value payload + // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) + char val[32605]; + int valLen = sizeof(val) / sizeof(val[0]); + generateBigVal(val, valLen); + + // insert the generated big data + // char const *key = "key1"; + for (int i = 0; i < 1024 * 4; ++i) { + // char const *key = "key123456789"; + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); GTEST_ASSERT_EQ(ret, 0); } - // 1, insert nData kv - { - int nData = nDataConst; - char key[64]; - char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) - int64_t poolLimit = 4096; // 1M pool limit - - for (int iData = 0; iData < nData; ++iData) { - sprintf(key, "key%03d", iData); - sprintf(val, "value%03d", iData); - - ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); - GTEST_ASSERT_EQ(ret, 0); - // if pool is full, commit the transaction and start a new one - if (pPool->size >= poolLimit) { - // commit current transaction - tdbCommit(pEnv, txn); - tdbPostCommit(pEnv, txn); - - // start a new transaction - clearPool(pPool); - - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - } - } - } - // commit current transaction tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); - closePool(pPool); - - // Close a database tdbTbClose(pDb); - - // Close Env tdbClose(pEnv); - system("ls -l ./tdb"); -} - -// TEST(TdbPageFlushTest, DISABLED_recycly_delete_interior_ofp_nocommit) { -TEST(TdbPageFlushTest, recycly_delete_interior_ofp_nocommit) { - clearDb("tdb"); - - // open Env - int ret = 0; - int const pageSize = 4096; - int const pageNum = 64; - TDB *pEnv = openEnv("tdb", pageSize, pageNum); + pEnv = openEnv("tdb", pageSize, pageNum); GTEST_ASSERT_NE(pEnv, nullptr); - // open db - TTB *pDb = NULL; - tdb_cmpr_fn_t compFunc = NULL; // tKeyCmpr; ret = tdbTbOpen("ofp_insert.db", -1, -1, compFunc, pEnv, &pDb, 0); GTEST_ASSERT_EQ(ret, 0); - // open the pool - SPoolMem *pPool = openPool(); + pPool = openPool(); + tdbBegin(pEnv, &txn, poolMallocRestricted, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - // start a transaction - TXN *txn; - - tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - - char key[1024] = {0}; - int count = sizeof(key) / sizeof(key[0]); - for (int i = 0; i < count - 1; ++i) { - key[i] = 'a'; - } - - // insert n ofp keys to form 2-layer btree - { - for (int i = 0; i < 7; ++i) { - // sprintf(&key[count - 2], "%c", i); - key[count - 2] = '0' + i; - - ret = tdbTbInsert(pDb, key, count, NULL, 0, txn); - GTEST_ASSERT_EQ(ret, 0); - } - } - /* - // delete one interior key - { - sprintf(&key[count - 2], "%c", 2); - key[count - 2] = '0' + 2; - - ret = tdbTbDelete(pDb, key, strlen(key) + 1, txn); + for (int i = 1024 * 4; i < 1024 * 8; ++i) { + char key[32] = {0}; + sprintf(key, "key-%d", i); + ret = tdbTbInsert(pDb, key, strlen(key) + 1, val, valLen, txn); GTEST_ASSERT_EQ(ret, 0); } - */ - // commit current transaction + tdbCommit(pEnv, txn); tdbPostCommit(pEnv, txn); @@ -1110,6 +579,4 @@ TEST(TdbPageFlushTest, recycly_delete_interior_ofp_nocommit) { // Close Env tdbClose(pEnv); - - system("ls -l ./tdb"); }