From dca02a8566bb931178894502bbe35a0168abe102 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Jan 2022 12:40:15 +0000 Subject: [PATCH] more tkv --- source/libs/tkv/CMakeLists.txt | 1 + source/libs/tkv/inc/tkvBufPool.h | 39 ++++++++++++++ source/libs/tkv/inc/tkvDef.h | 3 ++ .../libs/tkv/inc/{tDiskMgr.h => tkvDiskMgr.h} | 14 ++--- source/libs/tkv/inc/{tPage.h => tkvPage.h} | 20 ++++--- source/libs/tkv/src/tDiskMgr.c | 18 +++---- source/libs/tkv/src/tkvBufPool.c | 54 +++++++++++++++++++ 7 files changed, 127 insertions(+), 22 deletions(-) create mode 100644 source/libs/tkv/inc/tkvBufPool.h rename source/libs/tkv/inc/{tDiskMgr.h => tkvDiskMgr.h} (67%) rename source/libs/tkv/inc/{tPage.h => tkvPage.h} (71%) create mode 100644 source/libs/tkv/src/tkvBufPool.c diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tkv/CMakeLists.txt index 0620e12f55..ec3259d1f2 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tkv/CMakeLists.txt @@ -8,4 +8,5 @@ target_include_directories( target_link_libraries( tkv PUBLIC os + PUBLIC util ) \ No newline at end of file diff --git a/source/libs/tkv/inc/tkvBufPool.h b/source/libs/tkv/inc/tkvBufPool.h new file mode 100644 index 0000000000..ec8d177a9a --- /dev/null +++ b/source/libs/tkv/inc/tkvBufPool.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_BUF_POOL_H_ +#define _TD_TKV_BUF_POOL_H_ + +#include "tkvPage.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct STkvBufPool STkvBufPool; + +int tbpOpen(STkvBufPool **ppTkvBufPool); +int tbpClose(STkvBufPool *pTkvBufPool); +STkvPage *tbpNewPage(STkvBufPool *pTkvBufPool); +int tbpDelPage(STkvBufPool *pTkvBufPool); +STkvPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); +int tbpUnpinPage(STkvBufPool *pTkvBufPool, pgid_t pgid); +void tbpFlushPages(STkvBufPool *pTkvBufPool); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_BUF_POOL_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/inc/tkvDef.h b/source/libs/tkv/inc/tkvDef.h index 6f1072abd5..a80c395364 100644 --- a/source/libs/tkv/inc/tkvDef.h +++ b/source/libs/tkv/inc/tkvDef.h @@ -26,6 +26,9 @@ extern "C" { typedef int32_t pgid_t; #define TKV_IVLD_PGID ((pgid_t)-1) +// framd_id_t +typedef int32_t frame_id_t; + #ifdef __cplusplus } #endif diff --git a/source/libs/tkv/inc/tDiskMgr.h b/source/libs/tkv/inc/tkvDiskMgr.h similarity index 67% rename from source/libs/tkv/inc/tDiskMgr.h rename to source/libs/tkv/inc/tkvDiskMgr.h index 65ab4d7a3d..2ebe98ace2 100644 --- a/source/libs/tkv/inc/tDiskMgr.h +++ b/source/libs/tkv/inc/tkvDiskMgr.h @@ -24,14 +24,14 @@ extern "C" { #include "tkvDef.h" -typedef struct SDiskMgr SDiskMgr; +typedef struct STkvDiskMgr STkvDiskMgr; -int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); -int tdmClose(SDiskMgr *pDiskMgr); -int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData); -int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); -int tdmFlush(SDiskMgr *pDiskMgr); -pgid_t tdmAllocPage(SDiskMgr *pDiskMgr); +int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); +int tdmClose(STkvDiskMgr *pDiskMgr); +int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData); +int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); +int tdmFlush(STkvDiskMgr *pDiskMgr); +pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr); #ifdef __cplusplus } diff --git a/source/libs/tkv/inc/tPage.h b/source/libs/tkv/inc/tkvPage.h similarity index 71% rename from source/libs/tkv/inc/tPage.h rename to source/libs/tkv/inc/tkvPage.h index 0546f6184f..d596d215cd 100644 --- a/source/libs/tkv/inc/tPage.h +++ b/source/libs/tkv/inc/tkvPage.h @@ -17,22 +17,30 @@ #define _TD_TKV_PAGE_H_ #include "os.h" +#include "tkvDef.h" #ifdef __cplusplus extern "C" { #endif +typedef struct STkvPage { + pgid_t pgid; + int32_t pinCount; + bool idDirty; + char* pData; +} STkvPage; + typedef struct { uint16_t dbver; uint16_t pgsize; uint32_t cksm; -} SPgHdr; +} STkvPgHdr; -typedef struct { - SPgHdr chdr; - uint16_t used; // number of used slots - uint16_t loffset; // the offset of the starting location of the last slot used -} SSlottedPgHdr; +// typedef struct { +// SPgHdr chdr; +// uint16_t used; // number of used slots +// uint16_t loffset; // the offset of the starting location of the last slot used +// } SSlottedPgHdr; #ifdef __cplusplus } diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tkv/src/tDiskMgr.c index 5774555ae0..fa8f6062d8 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tkv/src/tDiskMgr.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ -#include "tDiskMgr.h" +#include "tkvDiskMgr.h" -struct SDiskMgr { +struct STkvDiskMgr { char * fname; uint16_t pgsize; FileFd fd; @@ -24,8 +24,8 @@ struct SDiskMgr { #define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) -int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { - SDiskMgr *pDiskMgr; +int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { + STkvDiskMgr *pDiskMgr; pDiskMgr = malloc(sizeof(*pDiskMgr)); if (pDiskMgr == NULL) { @@ -50,25 +50,25 @@ int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { return 0; } -int tdmClose(SDiskMgr *pDiskMgr) { +int tdmClose(STkvDiskMgr *pDiskMgr) { close(pDiskMgr->fd); free(pDiskMgr->fname); free(pDiskMgr); return 0; } -int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { +int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); return 0; } -int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { +int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); return 0; } -int tdmFlush(SDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } +int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } -int32_t tdmAllocPage(SDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file +int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file diff --git a/source/libs/tkv/src/tkvBufPool.c b/source/libs/tkv/src/tkvBufPool.c new file mode 100644 index 0000000000..86bfa0ba3e --- /dev/null +++ b/source/libs/tkv/src/tkvBufPool.c @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "thash.h" +#include "tlist.h" + +#include "tkvBufPool.h" +#include "tkvDiskMgr.h" +#include "tkvPage.h" + +struct SFrameIdWrapper { + TD_SLIST_NODE(SFrameIdWrapper); + frame_id_t id; +}; + +struct STkvBufPool { + STkvPage* pages; + STkvDiskMgr* pDiskMgr; + SHashObj* pgTb; // page_id_t --> frame_id_t + TD_SLIST(SFrameIdWrapper) freeList; + pthread_mutex_t mutex; +}; + +typedef struct STkvLRUReplacer { +} STkvLRUReplacer; + +typedef struct STkvLFUReplacer { +} STkvLFUReplacer; + +typedef struct STkvCLKReplacer { +} STkvCLKReplacer; + +typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t; + +typedef struct STkvReplacer { + tkv_replacer_t type; + union { + STkvLRUReplacer lruRep; + STkvLFUReplacer lfuRep; + STkvCLKReplacer clkRep; + }; +} STkvReplacer; \ No newline at end of file