From 007552afb2ab7199407d51ca829c993d4c4db896 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 18 Jan 2022 03:23:31 +0000 Subject: [PATCH 01/32] more --- source/libs/tdb/inc/tdb.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index eee8f8ed33..02b4422882 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -26,26 +26,23 @@ extern "C" { #define TDB_PUBLIC #define TDB_STATIC static -typedef enum { - TDB_BTREE_T = 0, - TDB_HASH_T, - TDB_HEAP_T, -} tdb_db_t; +typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; -// Forward declaration +// Forward declarations typedef struct TDB TDB; typedef struct TDB_CURSOR TDB_CURSOR; -// SKey typedef struct { void* bdata; uint32_t size; } TDB_KEY, TDB_VALUE; // TDB Operations -TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type); -TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); -TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags); +int tdbCreateDB(TDB** dbpp, tdb_db_t type); +int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); +int tdbCloseDB(TDB* dbp, uint32_t flags); +int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); +int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); #ifdef __cplusplus } From 1e84bce89fac773651535066a54239a56572e4ab Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 18 Jan 2022 07:53:02 +0000 Subject: [PATCH 02/32] more tkv --- source/libs/tdb/inc/tdb.h | 14 +++++ source/libs/tdb/src/inc/tdbDB.h | 4 -- source/libs/tdb/src/inc/tdbDef.h | 1 + .../src/inc/{tdbBufPool.h => tdb_mpfile.h} | 22 ++++---- source/libs/tdb/src/inc/tdb_mpool.h | 35 ++++++++++++ source/libs/tdb/src/mpool/tdbBufPool.c | 54 ------------------- source/libs/tdb/src/mpool/tdb_mpool.c | 14 +++++ 7 files changed, 74 insertions(+), 70 deletions(-) rename source/libs/tdb/src/inc/{tdbBufPool.h => tdb_mpfile.h} (57%) create mode 100644 source/libs/tdb/src/inc/tdb_mpool.h delete mode 100644 source/libs/tdb/src/mpool/tdbBufPool.c create mode 100644 source/libs/tdb/src/mpool/tdb_mpool.c diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 02b4422882..3a1523e6c6 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -30,6 +30,8 @@ typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; // Forward declarations typedef struct TDB TDB; +typedef struct TDB_MPOOL TDB_MPOOL; +typedef struct TDB_MPFILE TDB_MPFILE; typedef struct TDB_CURSOR TDB_CURSOR; typedef struct { @@ -44,6 +46,18 @@ int tdbCloseDB(TDB* dbp, uint32_t flags); int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); +// TDB_MPOOL +int tdbOpenMPool(TDB_MPOOL** mp); +int tdbCloseMPool(TDB_MPOOL* mp); + +// TDB_MPFILE +int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp); +int tdbCloseMPFile(TDB_MPFILE** mpf); + +// TDB_CURSOR +int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp); +int tdbCloseCurosr(TDB_CURSOR* tdbcp); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index d0ef9e22d0..261fb587b4 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -25,10 +25,6 @@ extern "C" { #endif -typedef struct { - // TODO -} TDB_MPOOL; - typedef struct { int fd; } TDB_FH; diff --git a/source/libs/tdb/src/inc/tdbDef.h b/source/libs/tdb/src/inc/tdbDef.h index 4b5e54368b..48a6831f86 100644 --- a/source/libs/tdb/src/inc/tdbDef.h +++ b/source/libs/tdb/src/inc/tdbDef.h @@ -17,6 +17,7 @@ #define _TD_TDB_DEF_H_ #include "os.h" +#include "tlist.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/tdb/src/inc/tdbBufPool.h b/source/libs/tdb/src/inc/tdb_mpfile.h similarity index 57% rename from source/libs/tdb/src/inc/tdbBufPool.h rename to source/libs/tdb/src/inc/tdb_mpfile.h index 5200d22faa..8eaafafd41 100644 --- a/source/libs/tdb/src/inc/tdbBufPool.h +++ b/source/libs/tdb/src/inc/tdb_mpfile.h @@ -13,27 +13,25 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_BUF_POOL_H_ -#define _TD_TDB_BUF_POOL_H_ +#ifndef _TD_TDB_MPFILE_H_ +#define _TD_TDB_MPFILE_H_ -#include "tdbPage.h" +#include "tdbDef.h" +#include "tdb_mpool.h" #ifdef __cplusplus extern "C" { #endif -typedef struct STdbBufPool STdbBufPool; +struct TDB_MPFILE { + TDB_MPOOL *mp; // memory pool used to get/put pages in this file -int tbpOpen(STdbBufPool **ppTkvBufPool); -int tbpClose(STdbBufPool *pTkvBufPool); -STdbPage *tbpNewPage(STdbBufPool *pTkvBufPool); -int tbpDelPage(STdbBufPool *pTkvBufPool); -STdbPage *tbpFetchPage(STdbBufPool *pTkvBufPool, pgid_t pgid); -int tbpUnpinPage(STdbBufPool *pTkvBufPool, pgid_t pgid); -void tbpFlushPages(STdbBufPool *pTkvBufPool); + char *fname; + int fd; +}; #ifdef __cplusplus } #endif -#endif /*_TD_TDB_BUF_POOL_H_*/ \ No newline at end of file +#endif /*_TD_TDB_MPFILE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h new file mode 100644 index 0000000000..9e34362c40 --- /dev/null +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TDB_MPOOL_H_ +#define _TD_TDB_MPOOL_H_ + +#include "tdbDef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct TDB_MPOOL { + pthread_mutex_t mutex; + int64_t cachesize; + pgsize_t pgsize; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TDB_MPOOL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/mpool/tdbBufPool.c b/source/libs/tdb/src/mpool/tdbBufPool.c deleted file mode 100644 index bc3c386b0f..0000000000 --- a/source/libs/tdb/src/mpool/tdbBufPool.c +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "thash.h" -#include "tlist.h" - -#include "tdbBufPool.h" -#include "tdbDiskMgr.h" -#include "tdbPage.h" - -struct SFrameIdWrapper { - TD_SLIST_NODE(SFrameIdWrapper); - frame_id_t id; -}; - -struct STdbBufPool { - STdbPage* pages; - STkvDiskMgr* pDiskMgr; - SHashObj* pgTb; // page_id_t --> frame_id_t - TD_SLIST(SFrameIdWrapper) freeList; - pthread_mutex_t mutex; -}; - -typedef struct STkvLRUReplacer { -} STkvLRUReplacer; - -typedef struct STkvLFUReplacer { -} STkvLFUReplacer; - -typedef struct STkvCLKReplacer { -} STkvCLKReplacer; - -typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t; - -typedef struct STkvReplacer { - tkv_replacer_t type; - union { - STkvLRUReplacer lruRep; - STkvLFUReplacer lfuRep; - STkvCLKReplacer clkRep; - }; -} STkvReplacer; \ No newline at end of file diff --git a/source/libs/tdb/src/mpool/tdb_mpool.c b/source/libs/tdb/src/mpool/tdb_mpool.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/mpool/tdb_mpool.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file From 73f802c3f5a16217ee9ff51fd586b0b7698da4ad Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 19 Jan 2022 08:17:40 +0000 Subject: [PATCH 03/32] more tkv --- source/libs/tdb/inc/tdb.h | 26 ++++---- .../src/{inc/tdb_mpool.h => devinc/tdb_inc.h} | 14 +--- .../libs/tdb/src/{inc => devinc}/tdb_mpfile.h | 18 +++-- source/libs/tdb/src/devinc/tdb_mpool.h | 65 +++++++++++++++++++ source/libs/tdb/src/inc/tdbDB.h | 4 +- 5 files changed, 97 insertions(+), 30 deletions(-) rename source/libs/tdb/src/{inc/tdb_mpool.h => devinc/tdb_inc.h} (77%) rename source/libs/tdb/src/{inc => devinc}/tdb_mpfile.h (64%) create mode 100644 source/libs/tdb/src/devinc/tdb_mpool.h diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 3a1523e6c6..5264707eb8 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -29,10 +29,10 @@ extern "C" { typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; // Forward declarations -typedef struct TDB TDB; -typedef struct TDB_MPOOL TDB_MPOOL; -typedef struct TDB_MPFILE TDB_MPFILE; -typedef struct TDB_CURSOR TDB_CURSOR; +typedef struct TDB TDB; +// typedef struct TDB_MPOOL TDB_MPOOL; +// typedef struct TDB_MPFILE TDB_MPFILE; +// typedef struct TDB_CURSOR TDB_CURSOR; typedef struct { void* bdata; @@ -46,17 +46,17 @@ int tdbCloseDB(TDB* dbp, uint32_t flags); int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); -// TDB_MPOOL -int tdbOpenMPool(TDB_MPOOL** mp); -int tdbCloseMPool(TDB_MPOOL* mp); +// // TDB_MPOOL +// int tdbOpenMPool(TDB_MPOOL** mp); +// int tdbCloseMPool(TDB_MPOOL* mp); -// TDB_MPFILE -int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp); -int tdbCloseMPFile(TDB_MPFILE** mpf); +// // TDB_MPFILE +// int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp); +// int tdbCloseMPFile(TDB_MPFILE** mpf); -// TDB_CURSOR -int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp); -int tdbCloseCurosr(TDB_CURSOR* tdbcp); +// // TDB_CURSOR +// int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp); +// int tdbCloseCurosr(TDB_CURSOR* tdbcp); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/devinc/tdb_inc.h similarity index 77% rename from source/libs/tdb/src/inc/tdb_mpool.h rename to source/libs/tdb/src/devinc/tdb_inc.h index 9e34362c40..c05247c44a 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/devinc/tdb_inc.h @@ -13,23 +13,15 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_MPOOL_H_ -#define _TD_TDB_MPOOL_H_ - -#include "tdbDef.h" +#ifndef _TD_TDB_INC_H_ +#define _TD_TDB_INC_H_ #ifdef __cplusplus extern "C" { #endif -struct TDB_MPOOL { - pthread_mutex_t mutex; - int64_t cachesize; - pgsize_t pgsize; -}; - #ifdef __cplusplus } #endif -#endif /*_TD_TDB_MPOOL_H_*/ \ No newline at end of file +#endif /*_TD_TDB_INC_H_*/ diff --git a/source/libs/tdb/src/inc/tdb_mpfile.h b/source/libs/tdb/src/devinc/tdb_mpfile.h similarity index 64% rename from source/libs/tdb/src/inc/tdb_mpfile.h rename to source/libs/tdb/src/devinc/tdb_mpfile.h index 8eaafafd41..74593af2f6 100644 --- a/source/libs/tdb/src/inc/tdb_mpfile.h +++ b/source/libs/tdb/src/devinc/tdb_mpfile.h @@ -23,11 +23,21 @@ extern "C" { #endif -struct TDB_MPFILE { - TDB_MPOOL *mp; // memory pool used to get/put pages in this file +// Exposed handle +typedef struct TDB_MPFILE TDB_MPFILE; - char *fname; - int fd; +// Exposed apis +int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); +int tdbMPFClose(TDB_MPFILE *mpf); +int tdbMPFGet(TDB_MPFILE *mpf, pgid_t pgid, void *addr); +int tdbMPFPut(TDB_MPOOL *mpf, pgid_t pgid, void *addr); + +// Hidden structures +struct TDB_MPFILE { + uint8_t fuid[20]; // file unique ID + TDB_MPOOL *mp; // underlying memory pool + char * fname; // file name + int fd; // fd }; #ifdef __cplusplus diff --git a/source/libs/tdb/src/devinc/tdb_mpool.h b/source/libs/tdb/src/devinc/tdb_mpool.h new file mode 100644 index 0000000000..4acf8bef8c --- /dev/null +++ b/source/libs/tdb/src/devinc/tdb_mpool.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TDB_MPOOL_H_ +#define _TD_TDB_MPOOL_H_ + +#include "tdbDef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Exposed handle +typedef struct TDB_MPOOL TDB_MPOOL; + +// Exposed apis +int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); +int tdbCloseMP(TDB_MPOOL *mp); +int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); +int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); + +// Hidden impls +#define TDB_FILE_UID_LEN 20 + +typedef struct { + uint8_t fuid[TDB_FILE_UID_LEN]; + pgid_t pgid; +} mp_pgid_t; + +typedef struct MP_PAGE { + SRWLatch rwLatch; + mp_pgid_t mpgid; + uint8_t dirty; + int32_t pinRef; + // TD_DLIST_NODE(MP_PAGE); // The free list handle + char *page[]; +} MP_PAGE; + +struct TDB_MPOOL { + pthread_mutex_t mutex; + int64_t cachesize; + pgsize_t pgsize; + MP_PAGE * pages; + // TD_DBLIST(MP_PAGE) freeList; + // TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool + // Hash hash; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TDB_MPOOL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index 261fb587b4..c246c18775 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -40,8 +40,8 @@ struct TDB { TDB_HEAP * heap; } dbam; // db access method - TDB_FH * fhp; // The backup file handle - TDB_MPOOL *mph; // The memory pool handle + // TDB_FH * fhp; // The backup file handle + // TDB_MPOOL *mph; // The memory pool handle }; #ifdef __cplusplus From 1793f616cafa5b8e28931df2b86626d4f15e7cd7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 19 Jan 2022 08:31:35 +0000 Subject: [PATCH 04/32] refact --- source/libs/tdb/CMakeLists.txt | 4 +- source/libs/tdb/inc/tdb.h | 38 +++++----- source/libs/tdb/src/db/tdbDB.c | 4 +- source/libs/tdb/src/devinc/tdb_inc.h | 27 ------- source/libs/tdb/src/dmgr/tdbDiskMgr.c | 74 ------------------- source/libs/tdb/src/inc/tdbBtree.h | 35 --------- source/libs/tdb/src/inc/tdbDB.h | 12 ++- source/libs/tdb/src/inc/tdbDiskMgr.h | 40 ---------- source/libs/tdb/src/inc/tdbHash.h | 35 --------- source/libs/tdb/src/inc/tdbHeap.h | 35 --------- source/libs/tdb/src/inc/tdbPage.h | 43 ----------- .../libs/tdb/src/inc/{tdbDef.h => tdb_inc.h} | 7 +- .../libs/tdb/src/{devinc => inc}/tdb_mpfile.h | 0 .../libs/tdb/src/{devinc => inc}/tdb_mpool.h | 2 +- source/libs/tdb/src/mpool/tdb_mpool.c | 14 ---- 15 files changed, 33 insertions(+), 337 deletions(-) delete mode 100644 source/libs/tdb/src/devinc/tdb_inc.h delete mode 100644 source/libs/tdb/src/dmgr/tdbDiskMgr.c delete mode 100644 source/libs/tdb/src/inc/tdbBtree.h delete mode 100644 source/libs/tdb/src/inc/tdbDiskMgr.h delete mode 100644 source/libs/tdb/src/inc/tdbHash.h delete mode 100644 source/libs/tdb/src/inc/tdbHeap.h delete mode 100644 source/libs/tdb/src/inc/tdbPage.h rename source/libs/tdb/src/inc/{tdbDef.h => tdb_inc.h} (92%) rename source/libs/tdb/src/{devinc => inc}/tdb_mpfile.h (100%) rename source/libs/tdb/src/{devinc => inc}/tdb_mpool.h (98%) delete mode 100644 source/libs/tdb/src/mpool/tdb_mpool.c diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index eb63f2b144..bb8904117c 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,5 +1,5 @@ -set(TDB_SUBDIRS "btree" "db" "hash" "mpool" "dmgr") +set(TDB_SUBDIRS "db") foreach(TDB_SUBDIR ${TDB_SUBDIRS}) aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC) endforeach() @@ -18,5 +18,5 @@ target_link_libraries( ) if(${BUILD_TEST}) - add_subdirectory(test) + # add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 5264707eb8..faf9208f8f 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,29 +22,29 @@ extern "C" { #endif -#define TDB_EXTERN -#define TDB_PUBLIC -#define TDB_STATIC static +// #define TDB_EXTERN +// #define TDB_PUBLIC +// #define TDB_STATIC static -typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; +// typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; -// Forward declarations -typedef struct TDB TDB; -// typedef struct TDB_MPOOL TDB_MPOOL; -// typedef struct TDB_MPFILE TDB_MPFILE; -// typedef struct TDB_CURSOR TDB_CURSOR; +// // Forward declarations +// typedef struct TDB TDB; +// // typedef struct TDB_MPOOL TDB_MPOOL; +// // typedef struct TDB_MPFILE TDB_MPFILE; +// // typedef struct TDB_CURSOR TDB_CURSOR; -typedef struct { - void* bdata; - uint32_t size; -} TDB_KEY, TDB_VALUE; +// typedef struct { +// void* bdata; +// uint32_t size; +// } TDB_KEY, TDB_VALUE; -// TDB Operations -int tdbCreateDB(TDB** dbpp, tdb_db_t type); -int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); -int tdbCloseDB(TDB* dbp, uint32_t flags); -int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); -int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); +// // TDB Operations +// int tdbCreateDB(TDB** dbpp, tdb_db_t type); +// int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); +// int tdbCloseDB(TDB* dbp, uint32_t flags); +// int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); +// int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); // // TDB_MPOOL // int tdbOpenMPool(TDB_MPOOL** mp); diff --git a/source/libs/tdb/src/db/tdbDB.c b/source/libs/tdb/src/db/tdbDB.c index eaf85ea4a1..9fac9887d0 100644 --- a/source/libs/tdb/src/db/tdbDB.c +++ b/source/libs/tdb/src/db/tdbDB.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#if 0 #include "tdbDB.h" #include "tdb.h" @@ -83,4 +84,5 @@ TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32 TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) { // TODO return 0; -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/libs/tdb/src/devinc/tdb_inc.h b/source/libs/tdb/src/devinc/tdb_inc.h deleted file mode 100644 index c05247c44a..0000000000 --- a/source/libs/tdb/src/devinc/tdb_inc.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_INC_H_ -#define _TD_TDB_INC_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_INC_H_*/ diff --git a/source/libs/tdb/src/dmgr/tdbDiskMgr.c b/source/libs/tdb/src/dmgr/tdbDiskMgr.c deleted file mode 100644 index 71ab5f2589..0000000000 --- a/source/libs/tdb/src/dmgr/tdbDiskMgr.c +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbDiskMgr.h" - -struct STkvDiskMgr { - char * fname; - uint16_t pgsize; - FileFd fd; - pgid_t npgid; -}; - -#define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) - -int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { - STkvDiskMgr *pDiskMgr; - - pDiskMgr = malloc(sizeof(*pDiskMgr)); - if (pDiskMgr == NULL) { - return -1; - } - - pDiskMgr->fname = strdup(fname); - if (pDiskMgr->fname == NULL) { - free(pDiskMgr); - return -1; - } - pDiskMgr->pgsize = pgsize; - pDiskMgr->fd = open(fname, O_CREAT | O_RDWR, 0755); - if (pDiskMgr->fd < 0) { - free(pDiskMgr->fname); - free(pDiskMgr); - return -1; - } - - *ppDiskMgr = pDiskMgr; - - return 0; -} - -int tdmClose(STkvDiskMgr *pDiskMgr) { - close(pDiskMgr->fd); - free(pDiskMgr->fname); - free(pDiskMgr); - return 0; -} - -int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { - taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); - taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); - return 0; -} - -int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { - taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); - taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); - return 0; -} - -int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } - -int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h deleted file mode 100644 index 28258b8e60..0000000000 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_BTREE_H_ -#define _TD_TDB_BTREE_H_ - -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - pgid_t root; // root page number -} TDB_BTREE; - -TDB_PUBLIC int tdbInitBtreeDB(TDB *dbp); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index c246c18775..7607580d3e 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -18,8 +18,6 @@ #include "tdb.h" #include "tdbBtree.h" -#include "tdbHash.h" -#include "tdbHeap.h" #ifdef __cplusplus extern "C" { @@ -34,11 +32,11 @@ struct TDB { tdb_db_t type; char * fname; char * dbname; - union { - TDB_BTREE *btree; - TDB_HASH * hash; - TDB_HEAP * heap; - } dbam; // db access method + // union { + // TDB_BTREE *btree; + // TDB_HASH * hash; + // TDB_HEAP * heap; + // } dbam; // db access method // TDB_FH * fhp; // The backup file handle // TDB_MPOOL *mph; // The memory pool handle diff --git a/source/libs/tdb/src/inc/tdbDiskMgr.h b/source/libs/tdb/src/inc/tdbDiskMgr.h deleted file mode 100644 index b83a147437..0000000000 --- a/source/libs/tdb/src/inc/tdbDiskMgr.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDISK_MGR_H_ -#define _TD_TDISK_MGR_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" - -#include "tdbDef.h" - -typedef struct STkvDiskMgr STkvDiskMgr; - -int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); -int tdmClose(STkvDiskMgr *pDiskMgr); -int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData); -int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); -int tdmFlush(STkvDiskMgr *pDiskMgr); -pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDISK_MGR_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbHash.h b/source/libs/tdb/src/inc/tdbHash.h deleted file mode 100644 index 8219bda2f8..0000000000 --- a/source/libs/tdb/src/inc/tdbHash.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_HASH_H_ -#define _TD_TDB_HASH_H_ - -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - // TODO -} TDB_HASH; - -TDB_PUBLIC int tdbInitHashDB(TDB *dbp); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_HASH_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbHeap.h b/source/libs/tdb/src/inc/tdbHeap.h deleted file mode 100644 index 25a812fa5f..0000000000 --- a/source/libs/tdb/src/inc/tdbHeap.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_HEAP_H_ -#define _TD_TDB_HEAP_H_ - -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - // TODO -} TDB_HEAP; - -TDB_PUBLIC int tdbInitHeapDB(TDB *dbp); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_HEAP_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h deleted file mode 100644 index e7245b6c39..0000000000 --- a/source/libs/tdb/src/inc/tdbPage.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TKV_PAGE_H_ -#define _TD_TKV_PAGE_H_ - -#include "os.h" -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - pgid_t pgid; - int32_t pinCount; - bool idDirty; - char* pData; -} STdbPage; - -typedef struct { - uint16_t dbver; - uint16_t pgsize; - uint32_t cksm; -} STdbPgHdr; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TKV_PAGE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDef.h b/source/libs/tdb/src/inc/tdb_inc.h similarity index 92% rename from source/libs/tdb/src/inc/tdbDef.h rename to source/libs/tdb/src/inc/tdb_inc.h index 48a6831f86..1e9c3dd8cc 100644 --- a/source/libs/tdb/src/inc/tdbDef.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -13,11 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_DEF_H_ -#define _TD_TDB_DEF_H_ +#ifndef _TD_TDB_INC_H_ +#define _TD_TDB_INC_H_ #include "os.h" -#include "tlist.h" #ifdef __cplusplus extern "C" { @@ -41,4 +40,4 @@ typedef int32_t pgsize_t; } #endif -#endif /*_TD_TDB_DEF_H_*/ \ No newline at end of file +#endif /*_TD_TDB_INC_H_*/ diff --git a/source/libs/tdb/src/devinc/tdb_mpfile.h b/source/libs/tdb/src/inc/tdb_mpfile.h similarity index 100% rename from source/libs/tdb/src/devinc/tdb_mpfile.h rename to source/libs/tdb/src/inc/tdb_mpfile.h diff --git a/source/libs/tdb/src/devinc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h similarity index 98% rename from source/libs/tdb/src/devinc/tdb_mpool.h rename to source/libs/tdb/src/inc/tdb_mpool.h index 4acf8bef8c..1023b14a50 100644 --- a/source/libs/tdb/src/devinc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -16,7 +16,7 @@ #ifndef _TD_TDB_MPOOL_H_ #define _TD_TDB_MPOOL_H_ -#include "tdbDef.h" +#include "tdb_inc.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/tdb/src/mpool/tdb_mpool.c b/source/libs/tdb/src/mpool/tdb_mpool.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/libs/tdb/src/mpool/tdb_mpool.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file From 131b3f16e8b7ac5e6c6cad606518890cc709f0a3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 19 Jan 2022 08:38:03 +0000 Subject: [PATCH 05/32] refact --- source/libs/tdb/src/db/{tdbDB.c => tdb_db.c} | 0 source/libs/tdb/src/inc/{tdbDB.h => tdb_db.h} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename source/libs/tdb/src/db/{tdbDB.c => tdb_db.c} (100%) rename source/libs/tdb/src/inc/{tdbDB.h => tdb_db.h} (100%) diff --git a/source/libs/tdb/src/db/tdbDB.c b/source/libs/tdb/src/db/tdb_db.c similarity index 100% rename from source/libs/tdb/src/db/tdbDB.c rename to source/libs/tdb/src/db/tdb_db.c diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdb_db.h similarity index 100% rename from source/libs/tdb/src/inc/tdbDB.h rename to source/libs/tdb/src/inc/tdb_db.h From a243752b420119bf5a7cf2542915ff94b1dc9b32 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 19 Jan 2022 08:44:38 +0000 Subject: [PATCH 06/32] refact --- source/libs/tdb/src/db/tdb_mpfile.c | 14 ++++++++++++++ source/libs/tdb/src/db/tdb_mpool.c | 16 ++++++++++++++++ source/libs/tdb/src/inc/tdb_inc.h | 1 + source/libs/tdb/src/inc/tdb_mpfile.h | 1 - source/libs/tdb/src/inc/tdb_mpool.h | 14 +++++++------- 5 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 source/libs/tdb/src/db/tdb_mpfile.c create mode 100644 source/libs/tdb/src/db/tdb_mpool.c diff --git a/source/libs/tdb/src/db/tdb_mpfile.c b/source/libs/tdb/src/db/tdb_mpfile.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/libs/tdb/src/db/tdb_mpfile.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c new file mode 100644 index 0000000000..ac4018d348 --- /dev/null +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdb_mpool.h" \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index 1e9c3dd8cc..cce6bc3bda 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -17,6 +17,7 @@ #define _TD_TDB_INC_H_ #include "os.h" +#include "tlist.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/tdb/src/inc/tdb_mpfile.h b/source/libs/tdb/src/inc/tdb_mpfile.h index 74593af2f6..cd4823081a 100644 --- a/source/libs/tdb/src/inc/tdb_mpfile.h +++ b/source/libs/tdb/src/inc/tdb_mpfile.h @@ -16,7 +16,6 @@ #ifndef _TD_TDB_MPFILE_H_ #define _TD_TDB_MPFILE_H_ -#include "tdbDef.h" #include "tdb_mpool.h" #ifdef __cplusplus diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 1023b14a50..8d113e5edc 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -25,6 +25,12 @@ extern "C" { // Exposed handle typedef struct TDB_MPOOL TDB_MPOOL; +#define TDB_FILE_UID_LEN 20 +typedef struct { + uint8_t fuid[TDB_FILE_UID_LEN]; + pgid_t pgid; +} mp_pgid_t; + // Exposed apis int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); int tdbCloseMP(TDB_MPOOL *mp); @@ -32,15 +38,9 @@ int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); // Hidden impls -#define TDB_FILE_UID_LEN 20 - -typedef struct { - uint8_t fuid[TDB_FILE_UID_LEN]; - pgid_t pgid; -} mp_pgid_t; typedef struct MP_PAGE { - SRWLatch rwLatch; + // SRWLatch rwLatch; mp_pgid_t mpgid; uint8_t dirty; int32_t pinRef; From a751c97f2061484876f73cc913405a60675a904b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 19 Jan 2022 09:50:34 +0000 Subject: [PATCH 07/32] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 57 ++++++++++++++++++- source/libs/tdb/src/inc/tdb_inc.h | 8 ++- .../tdb/src/inc/{tdb_mpool.h => tdb_mp.h} | 39 +++++++------ 3 files changed, 85 insertions(+), 19 deletions(-) rename source/libs/tdb/src/inc/{tdb_mpool.h => tdb_mp.h} (68%) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index ac4018d348..6d4f7c4bdc 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -13,4 +13,59 @@ * along with this program. If not, see . */ -#include "tdb_mpool.h" \ No newline at end of file +#include "tdb_mp.h" + +int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { + TDB_MPOOL *mp; + size_t tsize; + MP_PAGE * pagep; + + // check parameters + if (!TDB_IS_PGSIZE_VLD(pgsize)) { + tdbError("invalid page size"); + return -1; + } + + // allocate handle + mp = (TDB_MPOOL *)calloc(1, sizeof(*mp)); + if (mp == NULL) { + tdbError("failed to malloc memory pool handle"); + return -1; + } + + // initialize the handle + mp->cachesize = cachesize; + mp->pgsize = pgsize; + mp->npages = cachesize / pgsize; + mp->pages = (MP_PAGE *)calloc(mp->npages, MP_PAGE_SIZE(pgsize)); + if (mp->pages == NULL) { + tdbError("failed to malloc memory pool pages"); + free(mp); + return -1; + } + + TD_DLIST_INIT(&(mp->freeList)); + + mp->nbucket = mp->npages; + mp->hashtab = (MP_PAGE_LIST *)calloc(mp->nbucket, sizeof(MP_PAGE_LIST)); + if (mp->hashtab == NULL) { + tdbError("failed to malloc memory pool hash table"); + free(mp->pages); + free(mp); + return -1; + } + + for (int i = 0; i < mp->npages; i++) { + pagep = (MP_PAGE *)MP_PAGE_AT(mp, i); + TD_DLIST_APPEND(&mp->freeList, pagep); + } + + // return + *mpp = mp; + return 0; +} + +int tdbCloseMP(TDB_MPOOL *mp) { + // TODO + return 0; +} diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index cce6bc3bda..2eed9a8dc7 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -35,7 +35,13 @@ typedef int32_t pgsize_t; #define TDB_MIN_PGSIZE 512 #define TDB_MAX_PGSIZE 16384 #define TDB_DEFAULT_PGSIZE 4096 -#define TDB_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE)) +#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) + +// fileid +#define TDB_FILE_UID_LEN 20 + +// tdb_log +#define tdbError(var) #ifdef __cplusplus } diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mp.h similarity index 68% rename from source/libs/tdb/src/inc/tdb_mpool.h rename to source/libs/tdb/src/inc/tdb_mp.h index 8d113e5edc..d4e7457b05 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mp.h @@ -25,39 +25,44 @@ extern "C" { // Exposed handle typedef struct TDB_MPOOL TDB_MPOOL; -#define TDB_FILE_UID_LEN 20 typedef struct { uint8_t fuid[TDB_FILE_UID_LEN]; pgid_t pgid; } mp_pgid_t; -// Exposed apis -int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); -int tdbCloseMP(TDB_MPOOL *mp); -int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); -int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); - -// Hidden impls - typedef struct MP_PAGE { // SRWLatch rwLatch; mp_pgid_t mpgid; uint8_t dirty; int32_t pinRef; - // TD_DLIST_NODE(MP_PAGE); // The free list handle + TD_DLIST_NODE(MP_PAGE); char *page[]; } MP_PAGE; +#define MP_PAGE_SIZE(pgsize) (sizeof(MP_PAGE) + (pgsize)) + +typedef TD_DLIST(MP_PAGE) MP_PAGE_LIST; struct TDB_MPOOL { - pthread_mutex_t mutex; - int64_t cachesize; - pgsize_t pgsize; - MP_PAGE * pages; - // TD_DBLIST(MP_PAGE) freeList; - // TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool - // Hash hash; + int64_t cachesize; + pgsize_t pgsize; + int32_t npages; + MP_PAGE * pages; + MP_PAGE_LIST freeList; + // Hash + int32_t nbucket; + MP_PAGE_LIST *hashtab; + // TODO: TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool }; +#define MP_PAGE_AT(mp, idx) ((char *)((mp)->pages) + MP_PAGE_SIZE((mp)->pgsize) * (idx)) + +// Exposed apis ===================================================================================================== + +int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); +int tdbCloseMP(TDB_MPOOL *mp); +int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); +int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); + #ifdef __cplusplus } #endif From f207fbc0afed1e934c8620c0d7659f4a44f6f262 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 01:58:19 +0000 Subject: [PATCH 08/32] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 11 +++++++++++ source/libs/tdb/src/inc/tdb_mp.h | 1 + 2 files changed, 12 insertions(+) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 6d4f7c4bdc..f9a939b5e9 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -69,3 +69,14 @@ int tdbCloseMP(TDB_MPOOL *mp) { // TODO return 0; } + +int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p) { + // Search the hash + // TODO + return 0; +} + +int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mp.h b/source/libs/tdb/src/inc/tdb_mp.h index d4e7457b05..4a53c7b8d7 100644 --- a/source/libs/tdb/src/inc/tdb_mp.h +++ b/source/libs/tdb/src/inc/tdb_mp.h @@ -34,6 +34,7 @@ typedef struct MP_PAGE { // SRWLatch rwLatch; mp_pgid_t mpgid; uint8_t dirty; + uint8_t fileid[TDB_FILE_UID_LEN]; int32_t pinRef; TD_DLIST_NODE(MP_PAGE); char *page[]; From 0230a529f74d9dfefe12de2229fcb6f00922e523 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 05:12:50 +0000 Subject: [PATCH 09/32] refact tdb --- source/libs/tdb/src/db/tdb_mpfile.c | 14 ------ source/libs/tdb/src/db/tdb_mpool.c | 2 +- source/libs/tdb/src/inc/tdb_inc.h | 6 +-- source/libs/tdb/src/inc/tdb_mpfile.h | 46 ------------------- .../tdb/src/inc/{tdb_mp.h => tdb_mpool.h} | 22 +++++++-- 5 files changed, 22 insertions(+), 68 deletions(-) delete mode 100644 source/libs/tdb/src/db/tdb_mpfile.c delete mode 100644 source/libs/tdb/src/inc/tdb_mpfile.h rename source/libs/tdb/src/inc/{tdb_mp.h => tdb_mpool.h} (73%) diff --git a/source/libs/tdb/src/db/tdb_mpfile.c b/source/libs/tdb/src/db/tdb_mpfile.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/libs/tdb/src/db/tdb_mpfile.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index f9a939b5e9..d61c0de2c3 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tdb_mp.h" +#include "tdb_mpool.h" int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp; diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index 2eed9a8dc7..3e16036722 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -23,9 +23,9 @@ extern "C" { #endif -// pgid_t -typedef int32_t pgid_t; -#define TDB_IVLD_PGID ((pgid_t)-1) +// pgno_t +typedef int32_t pgno_t; +#define TDB_IVLD_PGID ((pgno_t)-1) // framd_id_t typedef int32_t frame_id_t; diff --git a/source/libs/tdb/src/inc/tdb_mpfile.h b/source/libs/tdb/src/inc/tdb_mpfile.h deleted file mode 100644 index cd4823081a..0000000000 --- a/source/libs/tdb/src/inc/tdb_mpfile.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_MPFILE_H_ -#define _TD_TDB_MPFILE_H_ - -#include "tdb_mpool.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Exposed handle -typedef struct TDB_MPFILE TDB_MPFILE; - -// Exposed apis -int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); -int tdbMPFClose(TDB_MPFILE *mpf); -int tdbMPFGet(TDB_MPFILE *mpf, pgid_t pgid, void *addr); -int tdbMPFPut(TDB_MPOOL *mpf, pgid_t pgid, void *addr); - -// Hidden structures -struct TDB_MPFILE { - uint8_t fuid[20]; // file unique ID - TDB_MPOOL *mp; // underlying memory pool - char * fname; // file name - int fd; // fd -}; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_MPFILE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mp.h b/source/libs/tdb/src/inc/tdb_mpool.h similarity index 73% rename from source/libs/tdb/src/inc/tdb_mp.h rename to source/libs/tdb/src/inc/tdb_mpool.h index 4a53c7b8d7..d84c2ffaeb 100644 --- a/source/libs/tdb/src/inc/tdb_mp.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -23,11 +23,12 @@ extern "C" { #endif // Exposed handle -typedef struct TDB_MPOOL TDB_MPOOL; +typedef struct TDB_MPOOL TDB_MPOOL; +typedef struct TDB_MPFILE TDB_MPFILE; typedef struct { uint8_t fuid[TDB_FILE_UID_LEN]; - pgid_t pgid; + pgno_t pgid; } mp_pgid_t; typedef struct MP_PAGE { @@ -55,15 +56,28 @@ struct TDB_MPOOL { // TODO: TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool }; +struct TDB_MPFILE { + uint8_t fuid[20]; // file unique ID + TDB_MPOOL *mp; // underlying memory pool + char * fname; // file name + int fd; // fd +}; + #define MP_PAGE_AT(mp, idx) ((char *)((mp)->pages) + MP_PAGE_SIZE((mp)->pgsize) * (idx)) -// Exposed apis ===================================================================================================== - +/*=================================================== Exposed apis ==================================================*/ +// TDB_MPOOL int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); int tdbCloseMP(TDB_MPOOL *mp); int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); +// TDB_MPFILE +int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); +int tdbMPFClose(TDB_MPFILE *mpf); +int tdbMPFGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr); +int tdbMPFPut(TDB_MPOOL *mpf, pgno_t pgid, void *addr); + #ifdef __cplusplus } #endif From 632bb1b4d2c63e24b6fdf9f61ebb03a05102aa54 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 05:13:32 +0000 Subject: [PATCH 10/32] refact --- source/libs/tdb/src/db/tdb_mpool.c | 4 ++-- source/libs/tdb/src/inc/tdb_mpool.h | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index d61c0de2c3..c387b6adcc 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -70,13 +70,13 @@ int tdbCloseMP(TDB_MPOOL *mp) { return 0; } -int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p) { +int tdbMPFetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p) { // Search the hash // TODO return 0; } -int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p) { +int tdbMpUnfetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p) { // TODO return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index d84c2ffaeb..2e9e9be73e 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -29,11 +29,11 @@ typedef struct TDB_MPFILE TDB_MPFILE; typedef struct { uint8_t fuid[TDB_FILE_UID_LEN]; pgno_t pgid; -} mp_pgid_t; +} pgid_t; typedef struct MP_PAGE { // SRWLatch rwLatch; - mp_pgid_t mpgid; + pgid_t mpgid; uint8_t dirty; uint8_t fileid[TDB_FILE_UID_LEN]; int32_t pinRef; @@ -50,7 +50,7 @@ struct TDB_MPOOL { int32_t npages; MP_PAGE * pages; MP_PAGE_LIST freeList; - // Hash + // Hash int32_t nbucket; MP_PAGE_LIST *hashtab; // TODO: TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool @@ -69,8 +69,8 @@ struct TDB_MPFILE { // TDB_MPOOL int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); int tdbCloseMP(TDB_MPOOL *mp); -int tdbMPFetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); -int tdbMpUnfetchPage(TDB_MPOOL *mp, mp_pgid_t mpgid, void *p); +int tdbMPFetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p); +int tdbMpUnfetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p); // TDB_MPFILE int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); From 8bbc5603216d0f79668bb503b64771820187b1e7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 05:20:06 +0000 Subject: [PATCH 11/32] refact TDB --- source/libs/tdb/src/db/tdb_mpool.c | 15 ++------------- source/libs/tdb/src/inc/tdb_inc.h | 12 +++++++++--- source/libs/tdb/src/inc/tdb_mpool.h | 19 ++++++------------- 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index c387b6adcc..0d9ddd9d46 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -15,7 +15,7 @@ #include "tdb_mpool.h" -int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp; size_t tsize; MP_PAGE * pagep; @@ -65,18 +65,7 @@ int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { return 0; } -int tdbCloseMP(TDB_MPOOL *mp) { - // TODO - return 0; -} - -int tdbMPFetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p) { - // Search the hash - // TODO - return 0; -} - -int tdbMpUnfetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p) { +int tdbMPoolClose(TDB_MPOOL *mp) { // TODO return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index 3e16036722..17dd5254d6 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -27,6 +27,15 @@ extern "C" { typedef int32_t pgno_t; #define TDB_IVLD_PGID ((pgno_t)-1) +// fileid +#define TDB_FILE_UID_LEN 20 + +// pgid_t +typedef struct { + uint8_t fuid[TDB_FILE_UID_LEN]; + pgno_t pgid; +} pgid_t; + // framd_id_t typedef int32_t frame_id_t; @@ -37,9 +46,6 @@ typedef int32_t pgsize_t; #define TDB_DEFAULT_PGSIZE 4096 #define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) -// fileid -#define TDB_FILE_UID_LEN 20 - // tdb_log #define tdbError(var) diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 2e9e9be73e..73596d91dd 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -26,17 +26,12 @@ extern "C" { typedef struct TDB_MPOOL TDB_MPOOL; typedef struct TDB_MPFILE TDB_MPFILE; -typedef struct { - uint8_t fuid[TDB_FILE_UID_LEN]; - pgno_t pgid; -} pgid_t; - typedef struct MP_PAGE { // SRWLatch rwLatch; - pgid_t mpgid; - uint8_t dirty; - uint8_t fileid[TDB_FILE_UID_LEN]; - int32_t pinRef; + pgid_t mpgid; + uint8_t dirty; + uint8_t fileid[TDB_FILE_UID_LEN]; + int32_t pinRef; TD_DLIST_NODE(MP_PAGE); char *page[]; } MP_PAGE; @@ -67,10 +62,8 @@ struct TDB_MPFILE { /*=================================================== Exposed apis ==================================================*/ // TDB_MPOOL -int tdbOpenMP(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); -int tdbCloseMP(TDB_MPOOL *mp); -int tdbMPFetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p); -int tdbMpUnfetchPage(TDB_MPOOL *mp, pgid_t mpgid, void *p); +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); +int tdbMPoolClose(TDB_MPOOL *mp); // TDB_MPFILE int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); From a00243ca404c4c28d406b40070f952dde9ebb04c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 05:25:54 +0000 Subject: [PATCH 12/32] refact --- source/libs/tdb/src/inc/tdb_inc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index 17dd5254d6..82947e2965 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -32,7 +32,7 @@ typedef int32_t pgno_t; // pgid_t typedef struct { - uint8_t fuid[TDB_FILE_UID_LEN]; + uint8_t fileid[TDB_FILE_UID_LEN]; pgno_t pgid; } pgid_t; From ed06d82230e4ffd687b26fb3c2384ff53c954e04 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 05:58:38 +0000 Subject: [PATCH 13/32] refact --- source/libs/tdb/src/db/tdb_mpool.c | 8 +++---- source/libs/tdb/src/inc/tdb_inc.h | 1 + source/libs/tdb/src/inc/tdb_mpool.h | 33 ++++++++++++++--------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 0d9ddd9d46..4e005b9e03 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -18,7 +18,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp; size_t tsize; - MP_PAGE * pagep; + pg_t * pagep; // check parameters if (!TDB_IS_PGSIZE_VLD(pgsize)) { @@ -37,7 +37,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { mp->cachesize = cachesize; mp->pgsize = pgsize; mp->npages = cachesize / pgsize; - mp->pages = (MP_PAGE *)calloc(mp->npages, MP_PAGE_SIZE(pgsize)); + mp->pages = (pg_t *)calloc(mp->npages, MP_PAGE_SIZE(pgsize)); if (mp->pages == NULL) { tdbError("failed to malloc memory pool pages"); free(mp); @@ -47,7 +47,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TD_DLIST_INIT(&(mp->freeList)); mp->nbucket = mp->npages; - mp->hashtab = (MP_PAGE_LIST *)calloc(mp->nbucket, sizeof(MP_PAGE_LIST)); + mp->hashtab = (pg_list_t *)calloc(mp->nbucket, sizeof(pg_list_t)); if (mp->hashtab == NULL) { tdbError("failed to malloc memory pool hash table"); free(mp->pages); @@ -56,7 +56,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { } for (int i = 0; i < mp->npages; i++) { - pagep = (MP_PAGE *)MP_PAGE_AT(mp, i); + pagep = (pg_t *)MP_PAGE_AT(mp, i); TD_DLIST_APPEND(&mp->freeList, pagep); } diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index 82947e2965..ba9d6a560c 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -18,6 +18,7 @@ #include "os.h" #include "tlist.h" +#include "tlockfree.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 73596d91dd..6b8cb8f844 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -26,28 +26,27 @@ extern "C" { typedef struct TDB_MPOOL TDB_MPOOL; typedef struct TDB_MPFILE TDB_MPFILE; -typedef struct MP_PAGE { - // SRWLatch rwLatch; - pgid_t mpgid; - uint8_t dirty; - uint8_t fileid[TDB_FILE_UID_LEN]; - int32_t pinRef; - TD_DLIST_NODE(MP_PAGE); +typedef struct pg_t { + SRWLatch rwLatch; + pgid_t mpgid; + uint8_t dirty; + int32_t pinRef; + TD_DLIST_NODE(pg_t); char *page[]; -} MP_PAGE; +} pg_t; -#define MP_PAGE_SIZE(pgsize) (sizeof(MP_PAGE) + (pgsize)) +#define MP_PAGE_SIZE(pgsize) (sizeof(pg_t) + (pgsize)) -typedef TD_DLIST(MP_PAGE) MP_PAGE_LIST; +typedef TD_DLIST(pg_t) pg_list_t; struct TDB_MPOOL { - int64_t cachesize; - pgsize_t pgsize; - int32_t npages; - MP_PAGE * pages; - MP_PAGE_LIST freeList; + int64_t cachesize; + pgsize_t pgsize; + int32_t npages; + pg_t * pages; + pg_list_t freeList; // Hash - int32_t nbucket; - MP_PAGE_LIST *hashtab; + int32_t nbucket; + pg_list_t *hashtab; // TODO: TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool }; From bfd427ae842ef06af29c12821f4c85bdf37bf427 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 07:03:06 +0000 Subject: [PATCH 14/32] more TDB --- source/libs/tdb/src/db/tdb_mpool.c | 96 ++++++++++++++++++++++++----- source/libs/tdb/src/inc/tdb_inc.h | 7 ++- source/libs/tdb/src/inc/tdb_mpool.h | 45 +++++++------- 3 files changed, 107 insertions(+), 41 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 4e005b9e03..60829ede94 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -15,8 +15,10 @@ #include "tdb_mpool.h" +static int tdbGnrtFileID(const char *fname, uint8_t *fileid); + int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { - TDB_MPOOL *mp; + TDB_MPOOL *mp = NULL; size_t tsize; pg_t * pagep; @@ -30,42 +32,102 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { mp = (TDB_MPOOL *)calloc(1, sizeof(*mp)); if (mp == NULL) { tdbError("failed to malloc memory pool handle"); - return -1; + goto _err; } // initialize the handle mp->cachesize = cachesize; mp->pgsize = pgsize; mp->npages = cachesize / pgsize; - mp->pages = (pg_t *)calloc(mp->npages, MP_PAGE_SIZE(pgsize)); + + TD_DLIST_INIT(&mp->freeList); + + mp->pages = (pg_t **)calloc(mp->npages, sizeof(pg_t *)); if (mp->pages == NULL) { tdbError("failed to malloc memory pool pages"); - free(mp); - return -1; + goto _err; } - TD_DLIST_INIT(&(mp->freeList)); + for (frame_id_t i = 0; i < mp->npages; i++) { + mp->pages[i] = (pg_t *)calloc(1, MP_PAGE_SIZE(pgsize)); + if (mp->pages[i] == NULL) { + goto _err; + } - mp->nbucket = mp->npages; - mp->hashtab = (pg_list_t *)calloc(mp->nbucket, sizeof(pg_list_t)); - if (mp->hashtab == NULL) { + taosInitRWLatch(&mp->pages[i]->rwLatch); + mp->pages[i]->frameid = i; + mp->pages[i]->pgid = TDB_IVLD_PGID; + + // TODO: add the new page to the free list + // TD_DLIST_APPEND(&mp->freeList, mp->pages[i]); + } + +#define PGTAB_FACTOR 1.0 + mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR; + mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t)); + if (mp->pgtab.hashtab == NULL) { tdbError("failed to malloc memory pool hash table"); - free(mp->pages); - free(mp); - return -1; - } - - for (int i = 0; i < mp->npages; i++) { - pagep = (pg_t *)MP_PAGE_AT(mp, i); - TD_DLIST_APPEND(&mp->freeList, pagep); + goto _err; } // return *mpp = mp; return 0; + +_err: + tdbMPoolClose(mp); + *mpp = NULL; + return -1; } int tdbMPoolClose(TDB_MPOOL *mp) { + if (mp) { + tfree(mp->pgtab.hashtab); + if (mp->pages) { + for (int i = 0; i < mp->npages; i++) { + tfree(mp->pages[i]); + } + + free(mp->pages); + } + + free(mp); + } + return 0; +} + +int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { + TDB_MPFILE *mpf; + + if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) { + return -1; + } + + mpf->fd = -1; + + if ((mpf->fname = strdup(fname)) == NULL) { + goto _err; + } + + if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) { + goto _err; + } + + *mpfp = mpf; + return 0; + +_err: + tdbMPoolFileClose(mpf); + *mpfp = NULL; + return -1; +} + +int tdbMPoolFileClose(TDB_MPFILE *mpf) { + // TODO + return 0; +} + +static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { // TODO return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index ba9d6a560c..d4c9796e54 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -26,16 +26,17 @@ extern "C" { // pgno_t typedef int32_t pgno_t; -#define TDB_IVLD_PGID ((pgno_t)-1) +#define TDB_IVLD_PGNO ((pgno_t)-1) // fileid -#define TDB_FILE_UID_LEN 20 +#define TDB_FILE_ID_LEN 24 // pgid_t typedef struct { - uint8_t fileid[TDB_FILE_UID_LEN]; + uint8_t fileid[TDB_FILE_ID_LEN]; pgno_t pgid; } pgid_t; +#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; // framd_id_t typedef int32_t frame_id_t; diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 6b8cb8f844..4bd4d12108 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -26,13 +26,16 @@ extern "C" { typedef struct TDB_MPOOL TDB_MPOOL; typedef struct TDB_MPFILE TDB_MPFILE; +typedef TD_DLIST_NODE(pg_t) pg_free_list_node_t, pg_hash_list_node_t; typedef struct pg_t { - SRWLatch rwLatch; - pgid_t mpgid; - uint8_t dirty; - int32_t pinRef; - TD_DLIST_NODE(pg_t); - char *page[]; + SRWLatch rwLatch; + frame_id_t frameid; + pgid_t pgid; + uint8_t dirty; + int32_t pinRef; + pg_free_list_node_t free; + pg_hash_list_node_t hash; + uint8_t data[]; } pg_t; #define MP_PAGE_SIZE(pgsize) (sizeof(pg_t) + (pgsize)) @@ -42,33 +45,33 @@ struct TDB_MPOOL { int64_t cachesize; pgsize_t pgsize; int32_t npages; - pg_t * pages; + pg_t ** pages; pg_list_t freeList; - // Hash - int32_t nbucket; - pg_list_t *hashtab; - // TODO: TD_DLIST(TD_MPFILE) mpfList; // MPFILE registered on this memory pool + struct { + int32_t nbucket; + pg_list_t *hashtab; + } pgtab; // page table, hash }; +#define MP_PAGE_AT(mp, idx) (mp)->pages[idx] + struct TDB_MPFILE { - uint8_t fuid[20]; // file unique ID - TDB_MPOOL *mp; // underlying memory pool - char * fname; // file name - int fd; // fd + uint8_t fileid[TDB_FILE_ID_LEN]; // file ID + TDB_MPOOL *mp; // underlying memory pool + char * fname; // file name + int fd; // fd }; -#define MP_PAGE_AT(mp, idx) ((char *)((mp)->pages) + MP_PAGE_SIZE((mp)->pgsize) * (idx)) - /*=================================================== Exposed apis ==================================================*/ // TDB_MPOOL int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); int tdbMPoolClose(TDB_MPOOL *mp); // TDB_MPFILE -int tdbMPFOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); -int tdbMPFClose(TDB_MPFILE *mpf); -int tdbMPFGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr); -int tdbMPFPut(TDB_MPOOL *mpf, pgno_t pgid, void *addr); +int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); +int tdbMPoolFileClose(TDB_MPFILE *mpf); +int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr); +int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgid, void *addr); #ifdef __cplusplus } From 2394e0e0ffb168a1c10cc1a0b11eb711fa65a4e9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 07:28:58 +0000 Subject: [PATCH 15/32] more --- source/libs/tdb/src/db/tdb_mpool.c | 26 ++++++++++++++++++++++++-- source/libs/tdb/src/inc/tdb_mpool.h | 4 ++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 60829ede94..6191d8d4d1 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -104,6 +104,7 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { } mpf->fd = -1; + mpf->mp = mp; if ((mpf->fname = strdup(fname)) == NULL) { goto _err; @@ -113,6 +114,10 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { goto _err; } + if (tdbGnrtFileID(fname, mpf->fileid) < 0) { + goto _err; + } + *mpfp = mpf; return 0; @@ -123,11 +128,28 @@ _err: } int tdbMPoolFileClose(TDB_MPFILE *mpf) { - // TODO + if (mpf) { + if (mpf->fd > 0) { + close(mpf->fd); + } + tfree(mpf->fname); + free(mpf); + } return 0; } static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { - // TODO + struct stat statbuf; + + if (stat(fname, &statbuf) < 0) { + return -1; + } + + memset(fileid, 0, TDB_FILE_ID_LEN); + + ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino; + ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev; + ((uint64_t *)fileid)[2] = rand(); + return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 4bd4d12108..dd1d802fa5 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -56,10 +56,10 @@ struct TDB_MPOOL { #define MP_PAGE_AT(mp, idx) (mp)->pages[idx] struct TDB_MPFILE { - uint8_t fileid[TDB_FILE_ID_LEN]; // file ID - TDB_MPOOL *mp; // underlying memory pool char * fname; // file name int fd; // fd + uint8_t fileid[TDB_FILE_ID_LEN]; // file ID + TDB_MPOOL *mp; // underlying memory pool }; /*=================================================== Exposed apis ==================================================*/ From 9f3d9db127895cdecd0abf8001c65676fb94d1a5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 08:03:33 +0000 Subject: [PATCH 16/32] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 45 ++++++++++++++++++++++++++++- source/libs/tdb/src/inc/tdb_mpool.h | 5 ++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 6191d8d4d1..aea6211493 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -15,7 +15,8 @@ #include "tdb_mpool.h" -static int tdbGnrtFileID(const char *fname, uint8_t *fileid); +static int tdbGnrtFileID(const char *fname, uint8_t *fileid); +static void tdbMpoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp = NULL; @@ -118,6 +119,9 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { goto _err; } + // Register current MPF to MP + tdbMpoolRegFile(mp, mpf); + *mpfp = mpf; return 0; @@ -138,6 +142,41 @@ int tdbMPoolFileClose(TDB_MPFILE *mpf) { return 0; } +int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr) { + pg_t * pagep; + TDB_MPOOL *mp; + + mp = mpf->mp; + + // get page in the cache + if (pagep) { + // page is found in the page table + // todo: pin the page and return + *(void **)addr = pagep->data; + return 0; + } else { + // page not found + pagep = TD_DLIST_HEAD(&mp->freeList); + if (pagep) { + // TD_DLIST_POP(&(mp->freeList), pagep); + } else { + // no page found in the freelist, need to evict + // pagep = tdbMpoolEvict(mp); + if (pagep) { + } else { + // TODO: Cannot find a page to evict + } + } + } + + return 0; +} + +int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgid, void *addr) { + // TODO + return 0; +} + static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { struct stat statbuf; @@ -152,4 +191,8 @@ static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { ((uint64_t *)fileid)[2] = rand(); return 0; +} + +static void tdbMpoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + // TODO } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index dd1d802fa5..e300cd2240 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -51,6 +51,10 @@ struct TDB_MPOOL { int32_t nbucket; pg_list_t *hashtab; } pgtab; // page table, hash + struct { + int32_t nbucket; + TD_DLIST(TDB_MPFILE) hashtab[8]; + } mpftab; }; #define MP_PAGE_AT(mp, idx) (mp)->pages[idx] @@ -60,6 +64,7 @@ struct TDB_MPFILE { int fd; // fd uint8_t fileid[TDB_FILE_ID_LEN]; // file ID TDB_MPOOL *mp; // underlying memory pool + TD_DLIST_NODE(TDB_MPFILE); }; /*=================================================== Exposed apis ==================================================*/ From b6ddd323564fbb0827e51116aad29172e75ba523 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 08:23:38 +0000 Subject: [PATCH 17/32] more tdv --- include/util/tlist.h | 64 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/include/util/tlist.h b/include/util/tlist.h index 06d0abc797..f318c3c58f 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -34,6 +34,7 @@ extern "C" { #define TD_SLIST_HEAD(sl) ((sl)->sl_head_) #define TD_SLIST_NELES(sl) ((sl)->sl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) +#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) ((sln)->(feild).sl_next_) #define TD_SLIST_INIT(sl) \ do { \ @@ -48,12 +49,25 @@ extern "C" { TD_SLIST_NELES(sl) += 1; \ } while (0) +#define TD_SLIST_PUSH_WITH_FIELD(sl, sln, feild) \ + do { \ + TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) = TD_SLIST_HEAD(sl); \ + TD_SLIST_HEAD(sl) = (sln); \ + TD_SLIST_NELES(sl) += 1; \ + } while (0) + #define TD_SLIST_POP(sl) \ do { \ TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ TD_SLIST_NELES(sl) -= 1; \ } while (0) +#define TD_SLIST_POP_WITH_FIELD(sl, feild) \ + do { \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT_WITH_FIELD(TD_SLIST_HEAD(sl), feild); \ + TD_SLIST_NELES(sl) -= 1; \ + } while (0) + // Double linked list ================ #define TD_DLIST_NODE(TYPE) \ struct { \ @@ -70,6 +84,8 @@ extern "C" { #define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) #define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) ((dln)->(feild).dl_prev_) +#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) ((dln)->(feild).dl_next_) #define TD_DLIST_HEAD(dl) ((dl)->dl_head_) #define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) #define TD_DLIST_NELES(dl) ((dl)->dl_neles_) @@ -94,6 +110,20 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) +#define TD_DLIST_APPEND_WITH_FEILD(dl, dln, feild) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_TAIL(dl), feild) = (dln); \ + TD_DLIST_TAIL(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + #define TD_DLIST_PREPEND(dl, dln) \ do { \ if (TD_DLIST_HEAD(dl) == NULL) { \ @@ -108,6 +138,20 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) +#define TD_DLIST_PREPEND_WITH_FIELD(dl, dln, feild) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_HEAD(dl), feild) = (dln); \ + TD_DLIST_HEAD(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + #define TD_DLIST_POP(dl, dln) \ do { \ if (TD_DLIST_HEAD(dl) == (dln)) { \ @@ -126,6 +170,26 @@ extern "C" { TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ } while (0) +#define TD_DLIST_POP_WITH_FIELD(dl, dln, field) \ + do { \ + if (TD_DLIST_HEAD(dl) == (dln)) { \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild); \ + } \ + if (TD_DLIST_TAIL(dl) == (dln)) { \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild); \ + } \ + if (TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) != NULL) { \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild), feild) = \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild); \ + } \ + if (TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) != NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild), feild) = \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild); \ + } \ + TD_DLIST_NELES(dl) -= 1; \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + } while (0) + // General double linked list typedef enum { TD_LIST_FORWARD, TD_LIST_BACKWARD } TD_LIST_DIRECTION_T; From c50b787fdaf138dbc93a1c64e2735bcd0c2743c0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 08:37:46 +0000 Subject: [PATCH 18/32] more tdb --- include/util/tlist.h | 6 +++--- source/libs/tdb/src/db/tdb_mpool.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/util/tlist.h b/include/util/tlist.h index f318c3c58f..4311356cd8 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -34,7 +34,7 @@ extern "C" { #define TD_SLIST_HEAD(sl) ((sl)->sl_head_) #define TD_SLIST_NELES(sl) ((sl)->sl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) -#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) ((sln)->(feild).sl_next_) +#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) ((sln)->feild.sl_next_) #define TD_SLIST_INIT(sl) \ do { \ @@ -84,8 +84,8 @@ extern "C" { #define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) #define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) -#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) ((dln)->(feild).dl_prev_) -#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) ((dln)->(feild).dl_next_) +#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) ((dln)->feild.dl_prev_) +#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) ((dln)->feild.dl_next_) #define TD_DLIST_HEAD(dl) ((dl)->dl_head_) #define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) #define TD_DLIST_NELES(dl) ((dl)->dl_neles_) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index aea6211493..9f2b8bb15b 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -59,8 +59,8 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { mp->pages[i]->frameid = i; mp->pages[i]->pgid = TDB_IVLD_PGID; - // TODO: add the new page to the free list - // TD_DLIST_APPEND(&mp->freeList, mp->pages[i]); + // add new page to the free list + TD_DLIST_APPEND_WITH_FEILD(&(mp->freeList), mp->pages[i], free); } #define PGTAB_FACTOR 1.0 From a9eeafe319af9d22d64033305c4c68feb56d99fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 09:41:28 +0000 Subject: [PATCH 19/32] more tdb --- include/util/tlist.h | 52 +++++++++--------- source/libs/tdb/src/db/tdb_mpool.c | 85 ++++++++++++++++++++++++++--- source/libs/tdb/src/inc/tdb_mpool.h | 39 +++++++------ 3 files changed, 126 insertions(+), 50 deletions(-) diff --git a/include/util/tlist.h b/include/util/tlist.h index 4311356cd8..134873a993 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -34,7 +34,7 @@ extern "C" { #define TD_SLIST_HEAD(sl) ((sl)->sl_head_) #define TD_SLIST_NELES(sl) ((sl)->sl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) -#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) ((sln)->feild.sl_next_) +#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) ((sln)->field.sl_next_) #define TD_SLIST_INIT(sl) \ do { \ @@ -49,9 +49,9 @@ extern "C" { TD_SLIST_NELES(sl) += 1; \ } while (0) -#define TD_SLIST_PUSH_WITH_FIELD(sl, sln, feild) \ +#define TD_SLIST_PUSH_WITH_FIELD(sl, sln, field) \ do { \ - TD_SLIST_NODE_NEXT_WITH_FIELD(sln, feild) = TD_SLIST_HEAD(sl); \ + TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) = TD_SLIST_HEAD(sl); \ TD_SLIST_HEAD(sl) = (sln); \ TD_SLIST_NELES(sl) += 1; \ } while (0) @@ -62,9 +62,9 @@ extern "C" { TD_SLIST_NELES(sl) -= 1; \ } while (0) -#define TD_SLIST_POP_WITH_FIELD(sl, feild) \ +#define TD_SLIST_POP_WITH_FIELD(sl, field) \ do { \ - TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT_WITH_FIELD(TD_SLIST_HEAD(sl), feild); \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT_WITH_FIELD(TD_SLIST_HEAD(sl), field); \ TD_SLIST_NELES(sl) -= 1; \ } while (0) @@ -84,8 +84,8 @@ extern "C" { #define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) #define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) -#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) ((dln)->feild.dl_prev_) -#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) ((dln)->feild.dl_next_) +#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) ((dln)->field.dl_prev_) +#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) ((dln)->field.dl_next_) #define TD_DLIST_HEAD(dl) ((dl)->dl_head_) #define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) #define TD_DLIST_NELES(dl) ((dl)->dl_neles_) @@ -110,15 +110,15 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) -#define TD_DLIST_APPEND_WITH_FEILD(dl, dln, feild) \ +#define TD_DLIST_APPEND_WITH_FIELD(dl, dln, field) \ do { \ if (TD_DLIST_HEAD(dl) == NULL) { \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ } else { \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_TAIL(dl); \ - TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ - TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_TAIL(dl), feild) = (dln); \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_TAIL(dl), field) = (dln); \ TD_DLIST_TAIL(dl) = (dln); \ } \ TD_DLIST_NELES(dl) += 1; \ @@ -138,15 +138,15 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) -#define TD_DLIST_PREPEND_WITH_FIELD(dl, dln, feild) \ +#define TD_DLIST_PREPEND_WITH_FIELD(dl, dln, field) \ do { \ if (TD_DLIST_HEAD(dl) == NULL) { \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ } else { \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = NULL; \ - TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = TD_DLIST_HEAD(dl); \ - TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_HEAD(dl), feild) = (dln); \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_HEAD(dl), field) = (dln); \ TD_DLIST_HEAD(dl) = (dln); \ } \ TD_DLIST_NELES(dl) += 1; \ @@ -173,21 +173,21 @@ extern "C" { #define TD_DLIST_POP_WITH_FIELD(dl, dln, field) \ do { \ if (TD_DLIST_HEAD(dl) == (dln)) { \ - TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild); \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field); \ } \ if (TD_DLIST_TAIL(dl) == (dln)) { \ - TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild); \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV_WITH_FIELD(dln, field); \ } \ - if (TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) != NULL) { \ - TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild), feild) = \ - TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild); \ + if (TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) != NULL) { \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_NODE_PREV_WITH_FIELD(dln, field), field) = \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field); \ } \ - if (TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) != NULL) { \ - TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild), feild) = \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild); \ + if (TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) != NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field), field) = \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field); \ } \ TD_DLIST_NELES(dl) -= 1; \ - TD_DLIST_NODE_PREV_WITH_FIELD(dln, feild) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, feild) = NULL; \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ } while (0) // General double linked list diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 9f2b8bb15b..d0092a44c9 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -15,8 +15,10 @@ #include "tdb_mpool.h" -static int tdbGnrtFileID(const char *fname, uint8_t *fileid); -static void tdbMpoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static int tdbGnrtFileID(const char *fname, uint8_t *fileid); +static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp = NULL; @@ -60,7 +62,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { mp->pages[i]->pgid = TDB_IVLD_PGID; // add new page to the free list - TD_DLIST_APPEND_WITH_FEILD(&(mp->freeList), mp->pages[i], free); + TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), mp->pages[i], free); } #define PGTAB_FACTOR 1.0 @@ -105,7 +107,6 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { } mpf->fd = -1; - mpf->mp = mp; if ((mpf->fname = strdup(fname)) == NULL) { goto _err; @@ -120,7 +121,7 @@ int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { } // Register current MPF to MP - tdbMpoolRegFile(mp, mpf); + tdbMPoolRegFile(mp, mpf); *mpfp = mpf; return 0; @@ -193,6 +194,76 @@ static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { return 0; } -static void tdbMpoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { - // TODO +#define MPF_GET_BUCKETID(fileid) \ + ({ \ + uint64_t *tmp = fileid; \ + (tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \ + }) + +static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + int bucketid; + mpf_bucket_t *bktp; + + bucketid = MPF_GET_BUCKETID(mpf->fileid); + bktp = mp->mpfht.buckets + bucketid; + + taosWLockLatch(&(bktp->latch)); + + TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node); + + taosWUnLockLatch(&(bktp->latch)); + + mpf->mp = mp; +} + +static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) { + int bucketid; + TDB_MPFILE * mpf = NULL; + mpf_bucket_t *bktp; + + bucketid = MPF_GET_BUCKETID(fileid); + bktp = mp->mpfht.buckets + bucketid; + + taosRLockLatch(&(bktp->latch)); + + mpf = TD_DLIST_HEAD(bktp); + while (mpf) { + if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) { + break; + } + + mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node); + } + + taosRUnLockLatch(&(bktp->latch)); + + return mpf; +} + +static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + mpf_bucket_t *bktp; + TDB_MPFILE * tmpf; + + if (mpf->mp == NULL) return; + + ASSERT(mpf->mp == mp); + + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); + + taosWLockLatch(&(bktp->latch)); + + tmpf = TD_DLIST_HEAD(bktp); + + while (tmpf) { + if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) { + TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node); + break; + } + + tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node); + } + + taosWUnLockLatch(&(bktp->latch)); + + ASSERT(tmpf == mpf); } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index e300cd2240..d77e14715d 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -26,21 +26,25 @@ extern "C" { typedef struct TDB_MPOOL TDB_MPOOL; typedef struct TDB_MPFILE TDB_MPFILE; -typedef TD_DLIST_NODE(pg_t) pg_free_list_node_t, pg_hash_list_node_t; +typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t; typedef struct pg_t { - SRWLatch rwLatch; - frame_id_t frameid; - pgid_t pgid; - uint8_t dirty; - int32_t pinRef; - pg_free_list_node_t free; - pg_hash_list_node_t hash; - uint8_t data[]; + SRWLatch rwLatch; + frame_id_t frameid; + pgid_t pgid; + uint8_t dirty; + int32_t pinRef; + pg_free_dlist_node_t free; + pg_hash_dlist_node_t hash; + uint8_t data[]; } pg_t; #define MP_PAGE_SIZE(pgsize) (sizeof(pg_t) + (pgsize)) typedef TD_DLIST(pg_t) pg_list_t; +typedef struct { + SRWLatch latch; + TD_DLIST(TDB_MPFILE); +} mpf_bucket_t; struct TDB_MPOOL { int64_t cachesize; pgsize_t pgsize; @@ -52,19 +56,20 @@ struct TDB_MPOOL { pg_list_t *hashtab; } pgtab; // page table, hash struct { - int32_t nbucket; - TD_DLIST(TDB_MPFILE) hashtab[8]; - } mpftab; +#define MPF_HASH_BUCKETS 16 + mpf_bucket_t buckets[MPF_HASH_BUCKETS]; + } mpfht; // MPF hash table. MPFs using this MP will be put in this hash table }; #define MP_PAGE_AT(mp, idx) (mp)->pages[idx] +typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t; struct TDB_MPFILE { - char * fname; // file name - int fd; // fd - uint8_t fileid[TDB_FILE_ID_LEN]; // file ID - TDB_MPOOL *mp; // underlying memory pool - TD_DLIST_NODE(TDB_MPFILE); + char * fname; // file name + int fd; // fd + uint8_t fileid[TDB_FILE_ID_LEN]; // file ID + TDB_MPOOL * mp; // underlying memory pool + td_mpf_dlist_node_t node; }; /*=================================================== Exposed apis ==================================================*/ From ca25ddc730d8bd2f27ef89adf758dbebd0f63c63 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 20 Jan 2022 10:15:07 +0000 Subject: [PATCH 20/32] refact --- source/libs/tdb/src/db/tdb_mpool.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index d0092a44c9..2c536a8e90 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -196,16 +196,14 @@ static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { #define MPF_GET_BUCKETID(fileid) \ ({ \ - uint64_t *tmp = fileid; \ + uint64_t *tmp = (uint64_t *)fileid; \ (tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \ }) static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { - int bucketid; mpf_bucket_t *bktp; - bucketid = MPF_GET_BUCKETID(mpf->fileid); - bktp = mp->mpfht.buckets + bucketid; + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); taosWLockLatch(&(bktp->latch)); @@ -217,12 +215,10 @@ static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { } static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) { - int bucketid; TDB_MPFILE * mpf = NULL; mpf_bucket_t *bktp; - bucketid = MPF_GET_BUCKETID(fileid); - bktp = mp->mpfht.buckets + bucketid; + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid); taosRLockLatch(&(bktp->latch)); From 9c984f9a906fa65767008f7657edb513a5110044 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 02:43:10 +0000 Subject: [PATCH 21/32] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 21 +++++++++++++++++++-- source/libs/tdb/src/inc/tdb_inc.h | 2 +- source/libs/tdb/src/inc/tdb_mpool.h | 4 ++-- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 2c536a8e90..3450f7c582 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -143,13 +143,30 @@ int tdbMPoolFileClose(TDB_MPFILE *mpf) { return 0; } -int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr) { +#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \ + ({ \ + uint64_t *tmp = (uint64_t *)fileid; \ + (tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \ + }) + +int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { pg_t * pagep; TDB_MPOOL *mp; + pg_list_t *pglist; mp = mpf->mp; // get page in the cache + pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket); + pagep = TD_DLIST_HEAD(pglist); + while (pagep) { + if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) { + break; + } + + pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash); + } + if (pagep) { // page is found in the page table // todo: pin the page and return @@ -173,7 +190,7 @@ int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr) { return 0; } -int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgid, void *addr) { +int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgno, void *addr) { // TODO return 0; } diff --git a/source/libs/tdb/src/inc/tdb_inc.h b/source/libs/tdb/src/inc/tdb_inc.h index d4c9796e54..885191477c 100644 --- a/source/libs/tdb/src/inc/tdb_inc.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -34,7 +34,7 @@ typedef int32_t pgno_t; // pgid_t typedef struct { uint8_t fileid[TDB_FILE_ID_LEN]; - pgno_t pgid; + pgno_t pgno; } pgid_t; #define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index d77e14715d..d9ee0bfdcd 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -80,8 +80,8 @@ int tdbMPoolClose(TDB_MPOOL *mp); // TDB_MPFILE int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); int tdbMPoolFileClose(TDB_MPFILE *mpf); -int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgid, void *addr); -int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgid, void *addr); +int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgno, void *addr); #ifdef __cplusplus } From a0d7e2a155afdb286af53b4ee6ea05e6742ef08b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 02:59:49 +0000 Subject: [PATCH 22/32] more TDB --- source/libs/tdb/src/db/tdb_mpool.c | 35 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 3450f7c582..965c3e023b 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -156,7 +156,7 @@ int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { mp = mpf->mp; - // get page in the cache + // check if the page already in pool pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket); pagep = TD_DLIST_HEAD(pglist); while (pagep) { @@ -168,25 +168,38 @@ int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { } if (pagep) { - // page is found in the page table + // page is found // todo: pin the page and return *(void **)addr = pagep->data; return 0; + } + + // page not found + pagep = TD_DLIST_HEAD(&mp->freeList); + if (pagep) { + // has free page + TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); + // todo: load the page from file and pin the page } else { - // page not found - pagep = TD_DLIST_HEAD(&mp->freeList); + // no free page available + // pagep = tdbMpoolEvict(mp); if (pagep) { - // TD_DLIST_POP(&(mp->freeList), pagep); } else { - // no page found in the freelist, need to evict - // pagep = tdbMpoolEvict(mp); - if (pagep) { - } else { - // TODO: Cannot find a page to evict - } + // TODO: Cannot find a page to evict } } + if (pagep == NULL) { + // no available container page + return -1; + } + + // load page from the disk if a container page is available + // TODO: load the page from the disk + + // add current page to page table + TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash); + return 0; } From 7ef98d1af96df22985c67049ccf4629b3bc01aad Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 05:20:18 +0000 Subject: [PATCH 23/32] more TDB --- source/libs/tdb/src/db/tdb_mpool.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 965c3e023b..abcce801bb 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -19,6 +19,7 @@ static int tdbGnrtFileID(const char *fname, uint8_t *fileid); static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); +static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp = NULL; @@ -196,6 +197,14 @@ int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { // load page from the disk if a container page is available // TODO: load the page from the disk + if (tdbMPoolFileReadPage(mpf, pgno, pagep->data) < 0) { + return -1; + } + + memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN); + pagep->pgid.pgno = pgno; + pagep->dirty = 0; + pagep->pinRef = 1; // add current page to page table TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash); @@ -292,4 +301,21 @@ static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { taosWUnLockLatch(&(bktp->latch)); ASSERT(tmpf == mpf); +} + +static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { + pgsize_t pgsize; + TDB_MPOOL *mp; + off_t offset; + size_t rsize; + + mp = mpf->mp; + pgsize = mp->pgsize; + offset = pgno * pgsize; + + // TODO: use loop to read all data + rsize = pread(mpf->fd, p, pgsize, offset); + // TODO: error handle + + return 0; } \ No newline at end of file From 8dfad06979b7b4de024b641548908788c3d20539 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 05:24:25 +0000 Subject: [PATCH 24/32] more --- source/libs/tdb/src/db/tdb_mpool.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index abcce801bb..5f3108a487 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -20,6 +20,7 @@ static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); +static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p); int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp = NULL; @@ -317,5 +318,23 @@ static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { rsize = pread(mpf->fd, p, pgsize, offset); // TODO: error handle + return 0; +} + +static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) { + pgsize_t pgsize; + TDB_MPOOL *mp; + off_t offset; + + mp = mpf->mp; + pgsize = mp->pgsize; + offset = pgno * pgsize; + + lseek(mpf->fd, offset, SEEK_SET); + // TODO: handle error + + write(mpf->fd, p, pgsize); + // TODO: handle error + return 0; } \ No newline at end of file From 4824ec7d7ac802636a8d407ffa6969b9171c389c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 05:29:19 +0000 Subject: [PATCH 25/32] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 14 ++++++++++++-- source/libs/tdb/src/inc/tdb_mpool.h | 6 ++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 5f3108a487..e368801900 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -151,7 +151,17 @@ int tdbMPoolFileClose(TDB_MPFILE *mpf) { (tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \ }) -int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { +int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) { + // TODO + return 0; +} + +int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) { + // TODO + return 0; +} + +int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { pg_t * pagep; TDB_MPOOL *mp; pg_list_t *pglist; @@ -213,7 +223,7 @@ int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { return 0; } -int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgno, void *addr) { +int tdbMPoolFilePutPage(TDB_MPOOL *mpf, pgno_t pgno, void *addr) { // TODO return 0; } diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index d9ee0bfdcd..c92038dfc8 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -80,8 +80,10 @@ int tdbMPoolClose(TDB_MPOOL *mp); // TDB_MPFILE int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); int tdbMPoolFileClose(TDB_MPFILE *mpf); -int tdbMPoolFileGet(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFilePut(TDB_MPOOL *mpf, pgno_t pgno, void *addr); +int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFilePutPage(TDB_MPOOL *mpf, pgno_t pgno, void *addr); #ifdef __cplusplus } From e9984c992d582933ad99f58d5cfc893f6d807588 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 09:13:22 +0000 Subject: [PATCH 26/32] refact --- source/libs/tdb/src/db/tdb_mpool.c | 21 ++++++++++----------- source/libs/tdb/src/inc/tdb_mpool.h | 7 +++---- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index e368801900..69b9a8a12f 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -47,24 +47,24 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TD_DLIST_INIT(&mp->freeList); - mp->pages = (pg_t **)calloc(mp->npages, sizeof(pg_t *)); + mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t)); if (mp->pages == NULL) { tdbError("failed to malloc memory pool pages"); goto _err; } for (frame_id_t i = 0; i < mp->npages; i++) { - mp->pages[i] = (pg_t *)calloc(1, MP_PAGE_SIZE(pgsize)); - if (mp->pages[i] == NULL) { + mp->pages[i].p = malloc(pgsize); + if (mp->pages[i].p == NULL) { goto _err; } - taosInitRWLatch(&mp->pages[i]->rwLatch); - mp->pages[i]->frameid = i; - mp->pages[i]->pgid = TDB_IVLD_PGID; + taosInitRWLatch(&mp->pages[i].rwLatch); + mp->pages[i].frameid = i; + mp->pages[i].pgid = TDB_IVLD_PGID; // add new page to the free list - TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), mp->pages[i], free); + TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free); } #define PGTAB_FACTOR 1.0 @@ -90,7 +90,7 @@ int tdbMPoolClose(TDB_MPOOL *mp) { tfree(mp->pgtab.hashtab); if (mp->pages) { for (int i = 0; i < mp->npages; i++) { - tfree(mp->pages[i]); + tfree(mp->pages[i].p); } free(mp->pages); @@ -182,7 +182,7 @@ int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { if (pagep) { // page is found // todo: pin the page and return - *(void **)addr = pagep->data; + *(void **)addr = pagep->p; return 0; } @@ -191,7 +191,6 @@ int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { if (pagep) { // has free page TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); - // todo: load the page from file and pin the page } else { // no free page available // pagep = tdbMpoolEvict(mp); @@ -208,7 +207,7 @@ int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { // load page from the disk if a container page is available // TODO: load the page from the disk - if (tdbMPoolFileReadPage(mpf, pgno, pagep->data) < 0) { + if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) { return -1; } diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index c92038dfc8..0c7613da63 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -32,14 +32,13 @@ typedef struct pg_t { frame_id_t frameid; pgid_t pgid; uint8_t dirty; + uint8_t rbit; int32_t pinRef; pg_free_dlist_node_t free; pg_hash_dlist_node_t hash; - uint8_t data[]; + void * p; } pg_t; -#define MP_PAGE_SIZE(pgsize) (sizeof(pg_t) + (pgsize)) - typedef TD_DLIST(pg_t) pg_list_t; typedef struct { SRWLatch latch; @@ -49,7 +48,7 @@ struct TDB_MPOOL { int64_t cachesize; pgsize_t pgsize; int32_t npages; - pg_t ** pages; + pg_t * pages; pg_list_t freeList; struct { int32_t nbucket; From daa29bca42e3d91787d1d45db563a1fa8cdae60d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 09:35:33 +0000 Subject: [PATCH 27/32] more tdb --- source/libs/tdb/src/db/tdb_mpool.c | 36 ++++++++++++++++++++++++++--- source/libs/tdb/src/inc/tdb_mpool.h | 11 +++++---- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index 69b9a8a12f..a6df79c9b3 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -21,6 +21,7 @@ static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p); +static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp); int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { TDB_MPOOL *mp = NULL; @@ -44,6 +45,7 @@ int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { mp->cachesize = cachesize; mp->pgsize = pgsize; mp->npages = cachesize / pgsize; + mp->clockHand = 0; TD_DLIST_INIT(&mp->freeList); @@ -193,10 +195,11 @@ int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); } else { // no free page available - // pagep = tdbMpoolEvict(mp); + tdbMPoolClockEvictPage(mp, &pagep); if (pagep) { - } else { - // TODO: Cannot find a page to evict + if (pagep->dirty) { + // TODO: Handle dirty page eviction + } } } @@ -346,4 +349,31 @@ static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) { // TODO: handle error return 0; +} + +static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) { + pg_t * pagep; + frame_id_t och; + + *pagepp = NULL; + och = mp->clockHand; + + do { + pagep = mp->pages + mp->clockHand; + mp->clockHand = (mp->clockHand + 1) % mp->npages; + + if (pagep->pinRef == 0) { + if (pagep->rbit == 1) { + pagep->rbit = 0; + } else { + break; + } + } + + if (mp->clockHand == och) { + return; + } + } while (1); + + *pagepp = pagep; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index 0c7613da63..d951040d8d 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -45,11 +45,12 @@ typedef struct { TD_DLIST(TDB_MPFILE); } mpf_bucket_t; struct TDB_MPOOL { - int64_t cachesize; - pgsize_t pgsize; - int32_t npages; - pg_t * pages; - pg_list_t freeList; + int64_t cachesize; + pgsize_t pgsize; + int32_t npages; + pg_t * pages; + pg_list_t freeList; + frame_id_t clockHand; struct { int32_t nbucket; pg_list_t *hashtab; From 314e1e1af423094e1ae68abd1e5c1a974dd119d8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 21 Jan 2022 10:14:39 +0000 Subject: [PATCH 28/32] more tdb --- source/libs/tdb/CMakeLists.txt | 2 +- source/libs/tdb/src/db/tdb_mpool.c | 2 +- source/libs/tdb/src/inc/tdb_mpool.h | 4 +++- source/libs/tdb/test/CMakeLists.txt | 4 ++++ source/libs/tdb/test/tdbMPoolTest.cpp | 31 +++++++++++++++++++++++++++ source/libs/tdb/test/tdbTest.cpp | 4 ++-- 6 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 source/libs/tdb/test/tdbMPoolTest.cpp diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index bb8904117c..3cb5a65572 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -18,5 +18,5 @@ target_link_libraries( ) if(${BUILD_TEST}) - # add_subdirectory(test) + add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c index a6df79c9b3..9b67f405a9 100644 --- a/source/libs/tdb/src/db/tdb_mpool.c +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -225,7 +225,7 @@ int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { return 0; } -int tdbMPoolFilePutPage(TDB_MPOOL *mpf, pgno_t pgno, void *addr) { +int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { // TODO return 0; } diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h index d951040d8d..37c82f3833 100644 --- a/source/libs/tdb/src/inc/tdb_mpool.h +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -76,6 +76,7 @@ struct TDB_MPFILE { // TDB_MPOOL int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); int tdbMPoolClose(TDB_MPOOL *mp); +int tdbMPoolSync(TDB_MPOOL *mp); // TDB_MPFILE int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); @@ -83,7 +84,8 @@ int tdbMPoolFileClose(TDB_MPFILE *mpf); int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); -int tdbMPoolFilePutPage(TDB_MPOOL *mpf, pgno_t pgno, void *addr); +int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFileSync(TDB_MPFILE *mpf); #ifdef __cplusplus } diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index 2d77c1f4e9..7fbfaf5506 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -1,3 +1,7 @@ +# tdbMPoolTest +add_executable(tdbMPoolTest "tdbMPoolTest.cpp") +target_link_libraries(tdbMPoolTest tdb gtest gtest_main) + # tdbTest add_executable(tdbTest "tdbTest.cpp") target_link_libraries(tdbTest tdb gtest gtest_main) \ No newline at end of file diff --git a/source/libs/tdb/test/tdbMPoolTest.cpp b/source/libs/tdb/test/tdbMPoolTest.cpp new file mode 100644 index 0000000000..17381759fb --- /dev/null +++ b/source/libs/tdb/test/tdbMPoolTest.cpp @@ -0,0 +1,31 @@ +#include "gtest/gtest.h" + +#include + +#include "tdb_mpool.h" + +TEST(tdb_mpool_test, test1) { + TDB_MPOOL * mp; + TDB_MPFILE *mpf; + pgno_t pgno; + void * pgdata; + + // open mp + tdbMPoolOpen(&mp, 16384, 4096); + + // open mpf + tdbMPoolFileOpen(&mpf, "test.db", mp); + +#define TEST1_TOTAL_PAGES 100 + for (int i = 0; i < TEST1_TOTAL_PAGES; i++) { + tdbMPoolFileNewPage(mpf, &pgno, pgdata); + + *(pgno_t *)pgdata = i; + } + + // close mpf + tdbMPoolFileClose(mpf); + + // close mp + tdbMPoolClose(mp); +} diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index f27e17f1ca..113bb2560f 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -3,8 +3,8 @@ #include "tdb.h" TEST(tdb_api_test, tdb_create_open_close_db_test) { - int ret; - TDB *dbp; + // int ret; + // TDB *dbp; // tdbCreateDB(&dbp, TDB_BTREE_T); From 6632640d6701de5ec18cfca33d24046033d95f17 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 24 Jan 2022 13:35:48 +0800 Subject: [PATCH 29/32] more tdb --- source/libs/tdb/src/db/tdb_db.c | 76 +++----------------------------- source/libs/tdb/src/inc/tdb_db.h | 20 ++++----- 2 files changed, 15 insertions(+), 81 deletions(-) diff --git a/source/libs/tdb/src/db/tdb_db.c b/source/libs/tdb/src/db/tdb_db.c index 9fac9887d0..3675844535 100644 --- a/source/libs/tdb/src/db/tdb_db.c +++ b/source/libs/tdb/src/db/tdb_db.c @@ -13,76 +13,14 @@ * along with this program. If not, see . */ -#if 0 -#include "tdbDB.h" -#include "tdb.h" +#include "tdb_db.h" -TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type) { - TDB* dbp; - int ret; - - dbp = calloc(1, sizeof(*dbp)); - if (dbp == NULL) { - return -1; - } - - dbp->pageSize = TDB_DEFAULT_PGSIZE; - dbp->type = type; - - switch (type) { - case TDB_BTREE_T: - // ret = tdbInitBtreeDB(dbp); - // if (ret < 0) goto _err; - break; - case TDB_HASH_T: - // ret = tdbInitHashDB(dbp); - // if (ret < 0) goto _err; - break; - case TDB_HEAP_T: - // ret = tdbInitHeapDB(dbp); - // if (ret < 0) goto _err; - break; - default: - break; - } - - *dbpp = dbp; - return 0; - -_err: - if (dbp) { - free(dbp); - } - *dbpp = NULL; - return 0; -} - -TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags) { - int ret = 0; - - if ((dbp->fname = strdup(fname)) == NULL) { - ret = -1; - return ret; - } - - // Create the backup file if the file not exists - - // Open the file as a sub-db or a master-db - if (dbname) { - if ((dbp->dbname = strdup(dbname)) == NULL) { - ret = -1; - return ret; - } - // TODO: Open the DB as a SUB-DB in this file - } else { - // TODO: Open the DB as a MASTER-DB in this file - } - - return ret; -} - -TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) { +int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags) { // TODO return 0; } -#endif \ No newline at end of file + +int tdbClose(TDB *dbp, uint32_t flags) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdb_db.h b/source/libs/tdb/src/inc/tdb_db.h index 7607580d3e..5f0529462b 100644 --- a/source/libs/tdb/src/inc/tdb_db.h +++ b/source/libs/tdb/src/inc/tdb_db.h @@ -16,32 +16,28 @@ #ifndef _TD_TDB_DB_H_ #define _TD_TDB_DB_H_ -#include "tdb.h" -#include "tdbBtree.h" +#include "tdb_mpool.h" #ifdef __cplusplus extern "C" { #endif -typedef struct { - int fd; -} TDB_FH; +typedef struct TDB TDB; struct TDB { - pgsize_t pageSize; - tdb_db_t type; - char * fname; - char * dbname; + char * fname; + char * dbname; + TDB_MPFILE *mpf; // union { // TDB_BTREE *btree; // TDB_HASH * hash; // TDB_HEAP * heap; // } dbam; // db access method - - // TDB_FH * fhp; // The backup file handle - // TDB_MPOOL *mph; // The memory pool handle }; +int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags); +int tdbClose(TDB *dbp, uint32_t flags); + #ifdef __cplusplus } #endif From c19e7779df118091db8221b91eb75edd6f878376 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 24 Jan 2022 06:27:28 +0000 Subject: [PATCH 30/32] fix memory leak --- source/dnode/vnode/src/meta/metaBDBImpl.c | 2 ++ source/dnode/vnode/src/vnd/vnodeBufferPool.c | 1 + source/dnode/vnode/src/vnd/vnodeMgr.c | 1 + source/dnode/vnode/src/vnd/vnodeQuery.c | 3 +++ source/dnode/vnode/src/vnd/vnodeWrite.c | 9 ++++++--- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index d8ee48ff23..b9f7649913 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -588,6 +588,8 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { metaDecodeTbInfo(pBuf, &tbCfg); if (tbCfg.type == META_SUPER_TABLE) { continue; + } else if (tbCfg.type == META_CHILD_TABLE) { + kvRowFree(tbCfg.ctbCfg.pTag) } return tbCfg.name; } else { diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c index 8df6b42566..434498eef5 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufferPool.c @@ -71,6 +71,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { void vnodeCloseBufPool(SVnode *pVnode) { if (pVnode->pBufPool) { + tfree(pVnode->pBufPool->pMAF); vmaDestroy(pVnode->pBufPool->inuse); while (true) { diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index 730155b75a..6829d778f0 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -119,6 +119,7 @@ static void* loop(void* arg) { pthread_mutex_unlock(&(vnodeMgr.mutex)); (*(pTask->execute))(pTask->arg); + free(pTask); } return NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 888520dc56..436dbab357 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -145,6 +145,9 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { _exit: + free(pSW); + free(pTbCfg->name); + free(pTbCfg); rpcMsg.handle = pMsg->handle; rpcMsg.ahandle = pMsg->ahandle; rpcMsg.pCont = pTbMetaMsg; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ccddfd56d8..169dc32c1c 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -34,7 +34,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void * pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedU64(&pBuf, ver); @@ -53,7 +53,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq vCreateTbReq; SVCreateTbBatchReq vCreateTbBatchReq; - void *ptr = vnodeMalloc(pVnode, pMsg->contLen); + void * ptr = vnodeMalloc(pVnode, pMsg->contLen); if (ptr == NULL) { // TODO: handle error } @@ -76,6 +76,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } // TODO: maybe need to clear the requst struct + free(vCreateTbReq.stbCfg.pSchema); + free(vCreateTbReq.stbCfg.pTagSchema); + free(vCreateTbReq.name); break; case TDMT_VND_CREATE_TABLE: tSVCreateTbBatchReqDeserialize(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq); @@ -129,7 +132,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error } } - + return 0; } From 03c19aa887676bfd4a81ab69ac7c904c23f4cee6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 24 Jan 2022 06:44:19 +0000 Subject: [PATCH 31/32] fix memory leak --- source/dnode/vnode/src/meta/metaBDBImpl.c | 3 ++- source/dnode/vnode/src/tq/tq.c | 3 +++ source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c | 1 - source/dnode/vnode/src/vnd/vnodeQuery.c | 19 +++++++++++++++---- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index b9f7649913..c8c5759bd2 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -587,9 +587,10 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { pBuf = value.data; metaDecodeTbInfo(pBuf, &tbCfg); if (tbCfg.type == META_SUPER_TABLE) { + free(tbCfg.stbCfg.pTagSchema); continue; } else if (tbCfg.type == META_CHILD_TABLE) { - kvRowFree(tbCfg.ctbCfg.pTag) + kvRowFree(tbCfg.ctbCfg.pTag); } return tbCfg.name; } else { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 49bbb77797..243b8d1b66 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -81,6 +81,9 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl } void tqClose(STQ* pTq) { + if (pTq) { + free(pTq); + } // TODO } diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c index 5844ef0345..a2e3a0dc56 100644 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c @@ -114,4 +114,3 @@ static void vArenaNodeFree(SVArenaNode *pNode) { if (pNode) { free(pNode); } -} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 436dbab357..be30df6e3c 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -145,9 +145,15 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { _exit: + free(pSW->pSchema); free(pSW); free(pTbCfg->name); free(pTbCfg); + if (pTbCfg->type == META_SUPER_TABLE) { + free(pTbCfg->stbCfg.pTagSchema); + } else if (pTbCfg->type == META_SUPER_TABLE) { + kvRowFree(pTbCfg->ctbCfg.pTag); + } rpcMsg.handle = pMsg->handle; rpcMsg.ahandle = pMsg->ahandle; rpcMsg.pCont = pTbMetaMsg; @@ -159,8 +165,8 @@ _exit: return 0; } -static void freeItemHelper(void* pItem) { - char* p = *(char**)pItem; +static void freeItemHelper(void *pItem) { + char *p = *(char **)pItem; free(p); } @@ -190,14 +196,14 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { // TODO: temp debug, and should del when show tables command ok vInfo("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables); if (numOfTables > 10000) { - numOfTables = 10000; + numOfTables = 10000; } metaCloseTbCursor(pCur); int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; - //int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); + // int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); int32_t payloadLen = rowLen * numOfTables; // SVShowTablesFetchReq *pFetchReq = pMsg->pCont; @@ -225,6 +231,11 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { }; rpcSendResponse(&rpcMsg); + for (int i = 0; i < taosArrayGetSize(pArray); i++) { + name = *(char **)taosArrayGet(pArray, i); + free(name); + } + taosArrayDestroyEx(pArray, freeItemHelper); return 0; } From 748df84238d5f06e23253f95c68e3244db7a494f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 24 Jan 2022 06:57:00 +0000 Subject: [PATCH 32/32] fix memory leakage --- source/dnode/vnode/src/meta/metaBDBImpl.c | 1 + source/dnode/vnode/src/tq/tq.c | 1 + source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c | 1 + source/dnode/vnode/src/vnd/vnodeQuery.c | 5 +---- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index c8c5759bd2..dfdc144750 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -587,6 +587,7 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { pBuf = value.data; metaDecodeTbInfo(pBuf, &tbCfg); if (tbCfg.type == META_SUPER_TABLE) { + free(tbCfg.name); free(tbCfg.stbCfg.pTagSchema); continue; } else if (tbCfg.type == META_CHILD_TABLE) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 243b8d1b66..f3ebd5b284 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -82,6 +82,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl void tqClose(STQ* pTq) { if (pTq) { + tfree(pTq->path); free(pTq); } // TODO diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c index a2e3a0dc56..14b9a5124f 100644 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c @@ -114,3 +114,4 @@ static void vArenaNodeFree(SVArenaNode *pNode) { if (pNode) { free(pNode); } +} diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index be30df6e3c..9c89e36903 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -217,6 +217,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { STR_TO_VARSTR(p, n); p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); + free(n); } pFetchRsp->numOfRows = htonl(numOfTables); @@ -231,10 +232,6 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { }; rpcSendResponse(&rpcMsg); - for (int i = 0; i < taosArrayGetSize(pArray); i++) { - name = *(char **)taosArrayGet(pArray, i); - free(name); - } taosArrayDestroyEx(pArray, freeItemHelper); return 0;