From a4b3bf6c1eec191e95f5499bddd2016c3f03527e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 10 Nov 2021 21:25:55 +0800 Subject: [PATCH 01/11] refact --- include/server/vnode/tsdb/tsdb.h | 13 +++++++++---- source/dnode/vnode/impl/inc/vnodeDef.h | 2 ++ source/dnode/vnode/impl/inc/vnodeMemAllocator.h | 6 ++++++ source/dnode/vnode/impl/src/vnodeMemAllocator.c | 15 ++++++++++++++- source/dnode/vnode/tsdb/inc/tsdbDef.h | 8 ++++++-- .../dnode/vnode/tsdb/inc/tsdbMemTable.h | 13 ++++--------- source/dnode/vnode/tsdb/src/tsdbMemTable.c | 14 ++++++++++++++ 7 files changed, 55 insertions(+), 16 deletions(-) rename include/server/vnode/tsdb/impl/tsdbImpl.h => source/dnode/vnode/tsdb/inc/tsdbMemTable.h (81%) create mode 100644 source/dnode/vnode/tsdb/src/tsdbMemTable.c diff --git a/include/server/vnode/tsdb/tsdb.h b/include/server/vnode/tsdb/tsdb.h index 949ac679ae..e92205378a 100644 --- a/include/server/vnode/tsdb/tsdb.h +++ b/include/server/vnode/tsdb/tsdb.h @@ -16,15 +16,14 @@ #ifndef _TD_TSDB_H_ #define _TD_TSDB_H_ -#include "impl/tsdbImpl.h" - #ifdef __cplusplus extern "C" { #endif // TYPES EXPOSED -typedef struct STsdb STsdb; -typedef struct STsdbOptions STsdbOptions; +typedef struct STsdb STsdb; +typedef struct STsdbOptions STsdbOptions; +typedef struct STsdbMemAllocator STsdbMemAllocator; // STsdb STsdb *tsdbOpen(const char *path, const STsdbOptions *); @@ -35,6 +34,12 @@ void tsdbRemove(const char *path); int tsdbOptionsInit(STsdbOptions *); void tsdbOptionsClear(STsdbOptions *); +/* ------------------------ STRUCT DEFINITIONS ------------------------ */ +struct STsdbOptions { + uint64_t lruCacheSize; + /* TODO */ +}; + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index 3ad7de722e..5ffbd8dcff 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -19,6 +19,7 @@ #include "mallocator.h" #include "sync.h" #include "tlockfree.h" +#include "wal.h" #include "vnode.h" #include "vnodeAllocatorPool.h" @@ -41,6 +42,7 @@ struct SVnode { SMeta* pMeta; STsdb* pTsdb; STQ* pTq; + SWal* pWal; SVnodeSync* pSync; SVnodeFS* pFs; }; diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index 7e9bd21fe3..9184eb416b 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -16,10 +16,16 @@ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_ +#include "mallocator.h" +#include "vnode.h" + #ifdef __cplusplus extern "C" { #endif +SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode); +void vnodeDestroyMemAllocator(SMemAllocator *pma); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/impl/src/vnodeMemAllocator.c b/source/dnode/vnode/impl/src/vnodeMemAllocator.c index 3771a14113..902014eb47 100644 --- a/source/dnode/vnode/impl/src/vnodeMemAllocator.c +++ b/source/dnode/vnode/impl/src/vnodeMemAllocator.c @@ -15,6 +15,17 @@ #include "vnodeDef.h" +SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode) { + SMemAllocator *pma = NULL; + /* TODO */ + return pma; +} + +void vnodeDestroyMemAllocator(SMemAllocator *pma) { + // TODO +} + +#if 0 #define VNODE_HEAP_ALLOCATOR 0 #define VNODE_ARENA_ALLOCATOR 1 @@ -97,4 +108,6 @@ void vnodeUnrefMemAllocator(SMemAllocator *pma) { /* ------------------------ Heap Allocator IMPL ------------------------ */ -/* ------------------------ Arena Allocator IMPL ------------------------ */ \ No newline at end of file +/* ------------------------ Arena Allocator IMPL ------------------------ */ + +#endif \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/inc/tsdbDef.h b/source/dnode/vnode/tsdb/inc/tsdbDef.h index ebe0d6b4b0..0a17387ba7 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbDef.h +++ b/source/dnode/vnode/tsdb/inc/tsdbDef.h @@ -16,7 +16,10 @@ #ifndef _TD_TSDB_DEF_H_ #define _TD_TSDB_DEF_H_ +#include "mallocator.h" + #include "tsdb.h" +#include "tsdbMemTable.h" #include "tsdbOptions.h" #ifdef __cplusplus @@ -24,8 +27,9 @@ extern "C" { #endif struct STsdb { - char * path; - STsdbOptions options; + char * path; + STsdbOptions options; + STsdbMemAllocator *pTMA; }; #ifdef __cplusplus diff --git a/include/server/vnode/tsdb/impl/tsdbImpl.h b/source/dnode/vnode/tsdb/inc/tsdbMemTable.h similarity index 81% rename from include/server/vnode/tsdb/impl/tsdbImpl.h rename to source/dnode/vnode/tsdb/inc/tsdbMemTable.h index 04d2ec43c9..15d868de9f 100644 --- a/include/server/vnode/tsdb/impl/tsdbImpl.h +++ b/source/dnode/vnode/tsdb/inc/tsdbMemTable.h @@ -13,22 +13,17 @@ * along with this program. If not, see . */ -#ifndef _TD_TSDB_IMPL_H_ -#define _TD_TSDB_IMPL_H_ - -#include "os.h" +#ifndef _TD_TSDB_MEM_TABLE_H_ +#define _TD_TSDB_MEM_TABLE_H_ #ifdef __cplusplus extern "C" { #endif -struct STsdbOptions { - size_t lruCacheSize; - /* TODO */ -}; +typedef struct SMemTable SMemTable; #ifdef __cplusplus } #endif -#endif /*_TD_TSDB_IMPL_H_*/ \ No newline at end of file +#endif /*_TD_TSDB_MEM_TABLE_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbMemTable.c b/source/dnode/vnode/tsdb/src/tsdbMemTable.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/dnode/vnode/tsdb/src/tsdbMemTable.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file From a1893dd80b3797a9c358b1069ff25741b4312989 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 10 Nov 2021 21:43:29 +0800 Subject: [PATCH 02/11] refact --- include/libs/{lru/lru.h => cache/cache.h} | 6 +++--- source/libs/CMakeLists.txt | 2 +- source/libs/cache/CMakeLists.txt | 7 +++++++ source/libs/{lru/inc/lruInt.h => cache/inc/cacheDef.h} | 6 +++--- source/libs/{lru/src/lru.c => cache/src/cache.c} | 0 .../{lru/test/lruTests.cpp => cache/test/cacheTests.cpp} | 0 source/libs/lru/CMakeLists.txt | 7 ------- 7 files changed, 14 insertions(+), 14 deletions(-) rename include/libs/{lru/lru.h => cache/cache.h} (91%) create mode 100644 source/libs/cache/CMakeLists.txt rename source/libs/{lru/inc/lruInt.h => cache/inc/cacheDef.h} (90%) rename source/libs/{lru/src/lru.c => cache/src/cache.c} (100%) rename source/libs/{lru/test/lruTests.cpp => cache/test/cacheTests.cpp} (100%) delete mode 100644 source/libs/lru/CMakeLists.txt diff --git a/include/libs/lru/lru.h b/include/libs/cache/cache.h similarity index 91% rename from include/libs/lru/lru.h rename to include/libs/cache/cache.h index c82e8ed746..6a2587ee96 100644 --- a/include/libs/lru/lru.h +++ b/include/libs/cache/cache.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_LRU_H_ -#define _TD_LRU_H_ +#ifndef _TD_CACHE_H_ +#define _TD_CACHE_H_ #ifdef __cplusplus extern "C" { @@ -24,4 +24,4 @@ extern "C" { } #endif -#endif /*_TD_LRU_H_*/ \ No newline at end of file +#endif /*_TD_CACHE_H_*/ \ No newline at end of file diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index eeaac61e21..007bb1e967 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -5,7 +5,7 @@ add_subdirectory(index) add_subdirectory(wal) add_subdirectory(parser) add_subdirectory(scheduler) -add_subdirectory(lru) +add_subdirectory(cache) add_subdirectory(catalog) add_subdirectory(executor) add_subdirectory(planner) diff --git a/source/libs/cache/CMakeLists.txt b/source/libs/cache/CMakeLists.txt new file mode 100644 index 0000000000..5ba59ef160 --- /dev/null +++ b/source/libs/cache/CMakeLists.txt @@ -0,0 +1,7 @@ +aux_source_directory(src CACHE_SRC) +add_library(cache ${CACHE_SRC}) +target_include_directories( + cache + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/cache" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) \ No newline at end of file diff --git a/source/libs/lru/inc/lruInt.h b/source/libs/cache/inc/cacheDef.h similarity index 90% rename from source/libs/lru/inc/lruInt.h rename to source/libs/cache/inc/cacheDef.h index e0da680104..2e0dbfcdb6 100644 --- a/source/libs/lru/inc/lruInt.h +++ b/source/libs/cache/inc/cacheDef.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_LRU_INT_H_ -#define _TD_LRU_INT_H_ +#ifndef _TD_CACHE_DEF_H_ +#define _TD_CACHE_DEF_H_ #ifdef __cplusplus extern "C" { @@ -24,4 +24,4 @@ extern "C" { } #endif -#endif /*_TD_LRU_INT_H_*/ \ No newline at end of file +#endif /*_TD_CACHE_DEF_H_*/ \ No newline at end of file diff --git a/source/libs/lru/src/lru.c b/source/libs/cache/src/cache.c similarity index 100% rename from source/libs/lru/src/lru.c rename to source/libs/cache/src/cache.c diff --git a/source/libs/lru/test/lruTests.cpp b/source/libs/cache/test/cacheTests.cpp similarity index 100% rename from source/libs/lru/test/lruTests.cpp rename to source/libs/cache/test/cacheTests.cpp diff --git a/source/libs/lru/CMakeLists.txt b/source/libs/lru/CMakeLists.txt deleted file mode 100644 index b65615dbbf..0000000000 --- a/source/libs/lru/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -aux_source_directory(src LRU_SRC) -add_library(lru ${LRU_SRC}) -target_include_directories( - lru - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/lru" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) \ No newline at end of file From 7277784852b9a4bed0a493a1f28d6e3059a1e215 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 11 Nov 2021 11:41:16 +0800 Subject: [PATCH 03/11] refact --- include/server/vnode/vnode.h | 73 +++++++++++++++++++++--- source/dnode/vnode/impl/src/vnodeWrite.c | 8 +-- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/include/server/vnode/vnode.h b/include/server/vnode/vnode.h index 29e14a673a..1b1e1c1b0f 100644 --- a/include/server/vnode/vnode.h +++ b/include/server/vnode/vnode.h @@ -17,9 +17,10 @@ #define _TD_VNODE_H_ #include "os.h" -#include "trequest.h" +#include "trpc.h" #include "meta.h" +#include "tarray.h" #include "tq.h" #include "tsdb.h" @@ -32,17 +33,71 @@ typedef struct SVnode SVnode; typedef struct SVnodeOptions SVnodeOptions; /* ------------------------ SVnode ------------------------ */ +/** + * @brief Open a VNODE. + * + * @param path path of the vnode + * @param pVnodeOptions options of the vnode + * @return SVnode* The vnode object + */ SVnode *vnodeOpen(const char *path, const SVnodeOptions *pVnodeOptions); -void vnodeClose(SVnode *pVnode); -void vnodeDestroy(const char *path); -int vnodeProcessWriteReqs(SVnode *pVnode, SReqBatch *pReqBatch); -int vnodeApplyWriteRequest(SVnode *pVnode, const SRequest *pRequest); -int vnodeProcessReadReq(SVnode *pVnode, SRequest *pReq); -int vnodeProcessSyncReq(SVnode *pVnode, SRequest *pReq); + +/** + * @brief Close a VNODE + * + * @param pVnode The vnode object + */ +void vnodeClose(SVnode *pVnode); + +/** + * @brief Destroy a VNODE. + * + * @param path Path of the VNODE. + */ +void vnodeDestroy(const char *path); + +/** + * @brief Process an array of write messages. + * + * @param pVnode The vnode object. + * @param pMsgs The array of SRpcMsg + * @return int 0 for success, -1 for failure + */ +int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs); + +/** + * @brief Apply a write request message. + * + * @param pVnode The vnode object. + * @param pMsg The request message + * @return int 0 for success, -1 for failure + */ +int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); + +/** + * @brief Process the sync request + * + * @param pVnode + * @param pMsg + * @param pRsp + * @return int + */ +int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); /* ------------------------ SVnodeOptions ------------------------ */ -void vnodeOptionsInit(SVnodeOptions *); -void vnodeOptionsClear(SVnodeOptions *); +/** + * @brief Initialize VNODE options. + * + * @param pOptions The options object to be initialized. It should not be NULL. + */ +void vnodeOptionsInit(SVnodeOptions *pOptions); + +/** + * @brief Clear VNODE options. + * + * @param pOptions Options to clear. + */ +void vnodeOptionsClear(SVnodeOptions *pOptions); /* ------------------------ STRUCT DEFINITIONS ------------------------ */ struct SVnodeOptions { diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index f3204cfd37..3d200b28d5 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -15,24 +15,24 @@ #include "vnodeDef.h" -int vnodeProcessWriteReqs(SVnode *pVnode, SReqBatch *pReqBatch) { +int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { /* TODO */ return 0; } -int vnodeApplyWriteRequest(SVnode *pVnode, const SRequest *pRequest) { +int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int reqType; /* TODO */ size_t reqSize; /* TODO */ uint64_t reqVersion = 0; /* TODO */ int code = 0; // Copy the request to vnode buffer - SRequest *pReq = mMalloc(pVnode->inuse, reqSize); + void *pReq = mMalloc(pVnode->inuse, reqSize); if (pReq == NULL) { // TODO: handle error } - memcpy(pReq, pRequest, reqSize); + memcpy(pReq, pMsg, reqSize); // Push the request to TQ so consumers can consume tqPushMsg(pVnode->pTq, pReq, 0); From 15d9a46600db7362be4e2cd8486226ad9e0de7d2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 12 Nov 2021 10:53:52 +0800 Subject: [PATCH 04/11] more --- include/server/vnode/meta/meta.h | 20 +- include/server/vnode/vnode.h | 1 + include/util/mallocator.h | 8 +- include/util/tlist.h | 3 +- ...vnodeAllocatorPool.h => vnodeBufferPool.h} | 22 +- source/dnode/vnode/impl/inc/vnodeDef.h | 23 +- source/dnode/vnode/impl/inc/vnodeRequest.h | 14 ++ .../dnode/vnode/impl/src/vnodeAllocatorPool.c | 37 --- source/dnode/vnode/impl/src/vnodeBufferPool.c | 211 ++++++++++++++++++ source/dnode/vnode/impl/src/vnodeMain.c | 7 - source/dnode/vnode/impl/src/vnodeWrite.c | 5 + source/util/src/tlist.c | 12 +- 12 files changed, 274 insertions(+), 89 deletions(-) rename source/dnode/vnode/impl/inc/{vnodeAllocatorPool.h => vnodeBufferPool.h} (59%) delete mode 100644 source/dnode/vnode/impl/src/vnodeAllocatorPool.c create mode 100644 source/dnode/vnode/impl/src/vnodeBufferPool.c diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index b94ffc7a5d..421f96ef5f 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -28,24 +28,24 @@ typedef struct SMetaOptions SMetaOptions; typedef struct STbOptions STbOptions; // SMeta operations -SMeta *metaOpen(const char *path, const SMetaOptions *); -void metaClose(SMeta *); +SMeta *metaOpen(const char *path, const SMetaOptions *pOptions); +void metaClose(SMeta *pMeta); void metaRemove(const char *path); -int metaCreateTable(SMeta *pMeta, const STbOptions *); +int metaCreateTable(SMeta *pMeta, const STbOptions *pTbOptions); int metaDropTable(SMeta *pMeta, tb_uid_t uid); -int metaCommit(SMeta *); +int metaCommit(SMeta *pMeta); // Options -void metaOptionsInit(SMetaOptions *); -void metaOptionsClear(SMetaOptions *); +void metaOptionsInit(SMetaOptions *pOptions); +void metaOptionsClear(SMetaOptions *pOptions); // STableOpts #define META_TABLE_OPTS_DECLARE(name) STableOpts name = {0} -void metaNormalTableOptsInit(STbOptions *, const char *name, const STSchema *pSchema); -void metaSuperTableOptsInit(STbOptions *, const char *name, tb_uid_t uid, const STSchema *pSchema, +void metaNormalTableOptsInit(STbOptions *pTbOptions, const char *name, const STSchema *pSchema); +void metaSuperTableOptsInit(STbOptions *pTbOptions, const char *name, tb_uid_t uid, const STSchema *pSchema, const STSchema *pTagSchema); -void metaChildTableOptsInit(STbOptions *, const char *name, tb_uid_t suid, const SKVRow tags); -void metaTableOptsClear(STbOptions *); +void metaChildTableOptsInit(STbOptions *pTbOptions, const char *name, tb_uid_t suid, const SKVRow tags); +void metaTableOptsClear(STbOptions *pTbOptions); #ifdef __cplusplus } diff --git a/include/server/vnode/vnode.h b/include/server/vnode/vnode.h index 1b1e1c1b0f..51cd643caf 100644 --- a/include/server/vnode/vnode.h +++ b/include/server/vnode/vnode.h @@ -70,6 +70,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs); * * @param pVnode The vnode object. * @param pMsg The request message + * @param pRsp The response message * @return int 0 for success, -1 for failure */ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); diff --git a/include/util/mallocator.h b/include/util/mallocator.h index a4705bdd2c..204f344cb7 100644 --- a/include/util/mallocator.h +++ b/include/util/mallocator.h @@ -25,20 +25,14 @@ extern "C" { typedef struct SMemAllocator SMemAllocator; #define MALLOCATOR_APIS \ + void *impl; \ void *(*malloc)(SMemAllocator *, size_t size); \ void *(*calloc)(SMemAllocator *, size_t nmemb, size_t size); \ void *(*realloc)(SMemAllocator *, void *ptr, size_t size); \ void (*free)(SMemAllocator *, void *ptr); \ size_t (*usage)(SMemAllocator *); -// Interfaces to implement -typedef struct { - MALLOCATOR_APIS -} SMemAllocatorIf; - struct SMemAllocator { - void * impl; - size_t usize; MALLOCATOR_APIS }; diff --git a/include/util/tlist.h b/include/util/tlist.h index 8246205e9b..e803e96605 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -46,9 +46,10 @@ typedef struct { #define isListEmpty(l) ((l)->numOfEles == 0) #define listNodeFree(n) free(n) +void tdListInit(SList *list, int eleSize); +void tdListEmpty(SList *list); SList * tdListNew(int eleSize); void * tdListFree(SList *list); -void tdListEmpty(SList *list); void tdListPrependNode(SList *list, SListNode *node); void tdListAppendNode(SList *list, SListNode *node); int tdListPrepend(SList *list, void *data); diff --git a/source/dnode/vnode/impl/inc/vnodeAllocatorPool.h b/source/dnode/vnode/impl/inc/vnodeBufferPool.h similarity index 59% rename from source/dnode/vnode/impl/inc/vnodeAllocatorPool.h rename to source/dnode/vnode/impl/inc/vnodeBufferPool.h index f02329fa87..5c65b4f614 100644 --- a/source/dnode/vnode/impl/inc/vnodeAllocatorPool.h +++ b/source/dnode/vnode/impl/inc/vnodeBufferPool.h @@ -13,27 +13,27 @@ * along with this program. If not, see . */ -#ifndef _TD_VNODE_ALLOCATOR_POOL_H_ -#define _TD_VNODE_ALLOCATOR_POOL_H_ +#ifndef _TD_VNODE_BUFFER_POOL_H_ +#define _TD_VNODE_BUFFER_POOL_H_ +#include "tlist.h" #include "vnode.h" #ifdef __cplusplus extern "C" { #endif -typedef struct { - int nexta; - int enda; - SMemAllocator *free[3]; - SMemAllocator *used[3]; -} SVAllocatorPool; +typedef struct SVBufPool SVBufPool; -int vnodeOpenAllocatorPool(SVnode *pVnode); -void vnodeCloseAllocatorPool(SVnode *pVnode); +int vnodeOpenBufPool(SVnode *pVnode); +void vnodeCloseBufPool(SVnode *pVnode); +SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode); +void vnodeDestroyMemAllocator(SMemAllocator *pma); +void vnodeRefMemAllocator(SMemAllocator *pma); +void vnodeUnrefMemAllocator(SMemAllocator *pma); #ifdef __cplusplus } #endif -#endif /*_TD_VNODE_ALLOCATOR_POOL_H_*/ \ No newline at end of file +#endif /*_TD_VNODE_BUFFER_POOL_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index 5ffbd8dcff..9cf9210cf3 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -22,7 +22,7 @@ #include "wal.h" #include "vnode.h" -#include "vnodeAllocatorPool.h" +#include "vnodeBufferPool.h" #include "vnodeCommit.h" #include "vnodeFileSystem.h" #include "vnodeOptions.h" @@ -34,17 +34,16 @@ extern "C" { #endif struct SVnode { - char* path; - SVnodeOptions options; - SVState state; - SVAllocatorPool* pool; - SMemAllocator* inuse; - SMeta* pMeta; - STsdb* pTsdb; - STQ* pTq; - SWal* pWal; - SVnodeSync* pSync; - SVnodeFS* pFs; + char* path; + SVnodeOptions options; + SVState state; + SVBufPool* pBufPool; + SMeta* pMeta; + STsdb* pTsdb; + STQ* pTq; + SWal* pWal; + SVnodeSync* pSync; + SVnodeFS* pFs; }; #ifdef __cplusplus diff --git a/source/dnode/vnode/impl/inc/vnodeRequest.h b/source/dnode/vnode/impl/inc/vnodeRequest.h index af909fb636..788918f105 100644 --- a/source/dnode/vnode/impl/inc/vnodeRequest.h +++ b/source/dnode/vnode/impl/inc/vnodeRequest.h @@ -20,6 +20,20 @@ extern "C" { #endif +typedef struct SVnodeReq SVnodeReq; +typedef struct SVnodeRsp SVnodeRsp; + +typedef enum { +} EVReqT; + +struct SVnodeReq { + /* TODO */ +}; + +struct SVnodeRsp { + /* TODO */ +}; + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/impl/src/vnodeAllocatorPool.c b/source/dnode/vnode/impl/src/vnodeAllocatorPool.c deleted file mode 100644 index b5617210a0..0000000000 --- a/source/dnode/vnode/impl/src/vnodeAllocatorPool.c +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" - -int vnodeOpenAllocatorPool(SVnode *pVnode) { - // TODO - return 0; -} - -void vnodeCloseAllocatorPool(SVnode *pVnode) { - if (pVnode->pool) { - } -} - -/* ------------------------ STATIC METHODS ------------------------ */ -static SVAllocatorPool *vapCreate() { - SVAllocatorPool *pPool = NULL; - /* TODO */ - return pPool; -} - -static void vapDestroy() { - // TODO -} \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c new file mode 100644 index 0000000000..d5005cf98c --- /dev/null +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeDef.h" + +/* ------------------------ STRUCTURES ------------------------ */ +struct SVBufPool { + SList free; + SList incycle; + SListNode *inuse; +}; + +typedef enum { E_V_HEAP_ALLOCATOR = 0, E_V_ARENA_ALLOCATOR } EVMemAllocatorT; + +typedef struct { +} SVHeapAllocator; + +typedef struct SVArenaNode { + struct SVArenaNode *prev; + uint64_t size; + void * ptr; + char data[]; +} SVArenaNode; + +typedef struct { + SVArenaNode *inuse; + SVArenaNode node; +} SVArenaAllocator; + +typedef struct { + uint64_t capacity; + EVMemAllocatorT type; + T_REF_DECLARE() + union { + SVHeapAllocator vha; + SVArenaAllocator vaa; + }; +} SVMemAllocator; + +static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); +static void vBufPoolFreeNode(SListNode *pNode); +static int vArenaAllocatorInit(SVArenaAllocator *pvaa); +static void vArenaAllocatorClear(SVArenaAllocator *pvaa); +static int vHeapAllocatorInit(SVHeapAllocator *pvha); +static void vHeapAllocatorClear(SVHeapAllocator *pvha); + +int vnodeOpenBufPool(SVnode *pVnode) { + uint64_t capacity; + EVMemAllocatorT type = E_V_ARENA_ALLOCATOR; + + if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { + /* TODO */ + return -1; + } + + tdListInit(&(pVnode->pBufPool->free), 0); + tdListInit(&(pVnode->pBufPool->incycle), 0); + + capacity = pVnode->options.wsize / 3; + if (pVnode->options.isHeapAllocator) { + type = E_V_HEAP_ALLOCATOR; + } + + for (int i = 0; i < 3; i++) { + SListNode *pNode = vBufPoolNewNode(capacity, type); + if (pNode == NULL) { + vnodeCloseBufPool(pVnode); + return -1; + } + + tdListAppendNode(&(pVnode->pBufPool->free), pNode); + } + + pVnode->pBufPool->inuse = tdListPopHead(&(pVnode->pBufPool->free)); + + return 0; +} + +void vnodeCloseBufPool(SVnode *pVnode) { + SListNode *pNode; + if (pVnode->pBufPool) { + // Clear free list + while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) != NULL) { + vBufPoolFreeNode(pNode); + } + + // Clear incycle list + while ((pNode = tdListPopHead(&(pVnode->pBufPool->incycle))) != NULL) { + vBufPoolFreeNode(pNode); + } + + // Free inuse node + vBufPoolFreeNode(pVnode->pBufPool->inuse); + + free(pVnode->pBufPool); + pVnode->pBufPool = NULL; + } +} + +SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode) { + SMemAllocator *pma; + + pma = (SMemAllocator *)calloc(1, sizeof(*pma)); + if (pma == NULL) { + /* TODO */ + return NULL; + } + + pma->impl = pVnode; + if (pVnode->options.isHeapAllocator) { + /* TODO */ + pma->malloc = NULL; + pma->calloc = NULL; + pma->realloc = NULL; + pma->free = NULL; + pma->usage = NULL; + } else { + /* TODO */ + pma->malloc = NULL; + pma->calloc = NULL; + pma->realloc = NULL; + pma->free = NULL; + pma->usage = NULL; + } + + return pma; +} + +void vnodeDestroyMemAllocator(SMemAllocator *pma) { tfree(pma); } + +void vnodeRefMemAllocator(SMemAllocator *pma) { + SVnode * pVnode = (SVnode *)pma->impl; + SVMemAllocator *pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); + + T_REF_INC(pvma); +} + +void vnodeUnrefMemAllocator(SMemAllocator *pma) { + SVnode * pVnode = (SVnode *)pma->impl; + SVMemAllocator *pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); + + if (T_REF_DEC(pvma) == 0) { + /* TODO */ + } +} + +/* ------------------------ STATIC METHODS ------------------------ */ +static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { + SListNode * pNode; + SVMemAllocator *pvma; + + pNode = (SListNode *)calloc(1, sizeof(*pNode) + sizeof(SVMemAllocator)); + if (pNode == NULL) { + return NULL; + } + + pvma = (SVMemAllocator *)(pNode->data); + pvma->capacity = capacity; + pvma->type = type; + + switch (type) { + case E_V_HEAP_ALLOCATOR: + vHeapAllocatorInit(&(pvma->vha)); + break; + case E_V_ARENA_ALLOCATOR: + vArenaAllocatorInit(&(pvma->vaa)); + break; + default: + ASSERT(0); + } + + return pNode; +} + +static void vBufPoolFreeNode(SListNode *pNode) { + if (pNode) { + free(pNode); + } +} + +// --------------- For arena allocator +static int vArenaAllocatorInit(SVArenaAllocator *pvaa) { + // TODO + return 0; +} + +static void vArenaAllocatorClear(SVArenaAllocator *pvaa) { + // TODO +} + +// --------------- For heap allocator +static int vHeapAllocatorInit(SVHeapAllocator *pvha) { + // TODO + return 0; +} + +static void vHeapAllocatorClear(SVHeapAllocator *pvha) { + // TODO +} \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index 6c0fe9c974..a0c1d38ea9 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -87,12 +87,6 @@ static void vnodeFree(SVnode *pVnode) { static int vnodeOpenImpl(SVnode *pVnode) { char dir[TSDB_FILENAME_LEN]; - // Open allocator pool - if (vnodeOpenAllocatorPool(pVnode) < 0) { - // TODO: handle error - return -1; - } - // Open meta sprintf(dir, "%s/meta", pVnode->path); pVnode->pMeta = metaOpen(dir, &(pVnode->options.metaOptions)); @@ -117,7 +111,6 @@ static int vnodeOpenImpl(SVnode *pVnode) { static void vnodeCloseImpl(SVnode *pVnode) { if (pVnode) { - vnodeCloseAllocatorPool(pVnode); // TODO: Close TQ tsdbClose(pVnode->pTsdb); metaClose(pVnode->pMeta); diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index 3d200b28d5..810ac570bc 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -21,6 +21,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { } int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +#if 0 int reqType; /* TODO */ size_t reqSize; /* TODO */ uint64_t reqVersion = 0; /* TODO */ @@ -45,7 +46,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { case TSDB_MSG_TYPE_DROP_TABLE: code = metaDropTable(pVnode->pMeta, 0 /* TODO */); break; + case TSDB_MSG_TYPE_SUBMIT: /* TODO */ + break; default: break; } @@ -57,6 +60,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } return code; +#endif + return 0; } /* ------------------------ STATIC METHODS ------------------------ */ \ No newline at end of file diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index c5b4dbad10..6756af226f 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -13,16 +13,20 @@ * along with this program. If not, see . */ -#include "os.h" #include "tlist.h" +#include "os.h" + +void tdListInit(SList *list, int eleSize) { + list->eleSize = eleSize; + list->numOfEles = 0; + list->head = list->tail = NULL; +} SList *tdListNew(int eleSize) { SList *list = (SList *)malloc(sizeof(SList)); if (list == NULL) return NULL; - list->eleSize = eleSize; - list->numOfEles = 0; - list->head = list->tail = NULL; + tdListInit(list, eleSize); return list; } From 2834b3a6efcfd113bbb8f8550f03d5ccff0aef27 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 12 Nov 2021 11:00:05 +0800 Subject: [PATCH 05/11] more --- include/dnode/vnode/tsdb/impl/tsdbImpl.h | 29 ------------------------ 1 file changed, 29 deletions(-) delete mode 100644 include/dnode/vnode/tsdb/impl/tsdbImpl.h diff --git a/include/dnode/vnode/tsdb/impl/tsdbImpl.h b/include/dnode/vnode/tsdb/impl/tsdbImpl.h deleted file mode 100644 index 15d868de9f..0000000000 --- a/include/dnode/vnode/tsdb/impl/tsdbImpl.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_MEM_TABLE_H_ -#define _TD_TSDB_MEM_TABLE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SMemTable SMemTable; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TSDB_MEM_TABLE_H_*/ \ No newline at end of file From a37a59e1f128bdd5c2613a228594a17452ade2d7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 12 Nov 2021 13:29:48 +0800 Subject: [PATCH 06/11] refact --- include/util/mallocator.h | 33 ++++++++-------------- source/dnode/vnode/tsdb/inc/tsdbMemTable.h | 7 ++++- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/include/util/mallocator.h b/include/util/mallocator.h index 204f344cb7..6a10a26156 100644 --- a/include/util/mallocator.h +++ b/include/util/mallocator.h @@ -24,31 +24,20 @@ extern "C" { typedef struct SMemAllocator SMemAllocator; -#define MALLOCATOR_APIS \ - void *impl; \ - void *(*malloc)(SMemAllocator *, size_t size); \ - void *(*calloc)(SMemAllocator *, size_t nmemb, size_t size); \ - void *(*realloc)(SMemAllocator *, void *ptr, size_t size); \ - void (*free)(SMemAllocator *, void *ptr); \ - size_t (*usage)(SMemAllocator *); - struct SMemAllocator { - MALLOCATOR_APIS + void *impl; + void *(*malloc)(SMemAllocator *, uint64_t size); + void *(*calloc)(SMemAllocator *, uint64_t nmemb, uint64_t size); + void *(*realloc)(SMemAllocator *, void *ptr, uint64_t size); + void (*free)(SMemAllocator *, void *ptr); + uint64_t (*usage)(SMemAllocator *); }; -// heap allocator -SMemAllocator *tdCreateHeapAllocator(); -void tdDestroyHeapAllocator(SMemAllocator *pMemAllocator); - -// arena allocator -SMemAllocator *tdCreateArenaAllocator(size_t size); -void tdDestroyArenaAllocator(SMemAllocator *); - -#define mMalloc(pMemAllocator, size) (*(pMemAllocator->malloc))(pMemAllocator, size) -#define mCalloc(pMemAllocator, nmemb, size) (*(pMemAllocator->calloc))(pMemAllocator, nmemb, size) -#define mRealloc(pMemAllocator, ptr, size) (*(pMemAllocator->realloc))(pMemAllocator, ptr, size) -#define mFree(pMemAllocator, ptr) (*(pMemAllocator->free))(pMemAllocator, ptr) -#define mUsage(pMemAllocator) (*(pMemAllocator->usage))(pMemAllocator) +typedef struct { + void *impl; + SMemAllocator *(*create)(); + void (*destroy)(SMemAllocator *); +} SMemAllocatorFactory; #ifdef __cplusplus } diff --git a/source/dnode/vnode/tsdb/inc/tsdbMemTable.h b/source/dnode/vnode/tsdb/inc/tsdbMemTable.h index 15d868de9f..6be90ef53c 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbMemTable.h +++ b/source/dnode/vnode/tsdb/inc/tsdbMemTable.h @@ -16,11 +16,16 @@ #ifndef _TD_TSDB_MEM_TABLE_H_ #define _TD_TSDB_MEM_TABLE_H_ +#include "tsdb.h" + #ifdef __cplusplus extern "C" { #endif -typedef struct SMemTable SMemTable; +typedef struct SMemTable { + /* TODO */ + SMemAllocator *pma; +} SMemTable; #ifdef __cplusplus } From 6010516f58e09853b2c73f819e6169d4dd72ad5b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 12 Nov 2021 13:35:10 +0800 Subject: [PATCH 07/11] refact --- source/dnode/vnode/impl/inc/vnodeBufferPool.h | 8 ++------ source/dnode/vnode/meta/inc/metaDef.h | 15 +++++++++------ source/dnode/vnode/tsdb/inc/tsdbDef.h | 6 +++--- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/impl/inc/vnodeBufferPool.h index 5c65b4f614..3033862779 100644 --- a/source/dnode/vnode/impl/inc/vnodeBufferPool.h +++ b/source/dnode/vnode/impl/inc/vnodeBufferPool.h @@ -25,12 +25,8 @@ extern "C" { typedef struct SVBufPool SVBufPool; -int vnodeOpenBufPool(SVnode *pVnode); -void vnodeCloseBufPool(SVnode *pVnode); -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode); -void vnodeDestroyMemAllocator(SMemAllocator *pma); -void vnodeRefMemAllocator(SMemAllocator *pma); -void vnodeUnrefMemAllocator(SMemAllocator *pma); +int vnodeOpenBufPool(SVnode *pVnode); +void vnodeCloseBufPool(SVnode *pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/meta/inc/metaDef.h b/source/dnode/vnode/meta/inc/metaDef.h index 562476a439..b0d31de1b4 100644 --- a/source/dnode/vnode/meta/inc/metaDef.h +++ b/source/dnode/vnode/meta/inc/metaDef.h @@ -16,6 +16,8 @@ #ifndef _TD_META_DEF_H_ #define _TD_META_DEF_H_ +#include "mallocator.h" + #include "meta.h" #include "metaCache.h" #include "metaDB.h" @@ -30,12 +32,13 @@ extern "C" { #endif struct SMeta { - char* path; // path of current meta - SMetaOptions options; // meta option - meta_db_t* pDB; // raw data db - meta_index_t* pIdx; // tag index - meta_cache_t* pCache; // LRU cache - STbUidGenerator uidGnrt; // meta table UID generator + char* path; + SMetaOptions options; + meta_db_t* pDB; + meta_index_t* pIdx; + meta_cache_t* pCache; + STbUidGenerator uidGnrt; + SMemAllocatorFactory* pmaf; }; #ifdef __cplusplus diff --git a/source/dnode/vnode/tsdb/inc/tsdbDef.h b/source/dnode/vnode/tsdb/inc/tsdbDef.h index 0a17387ba7..ca3d0319c1 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbDef.h +++ b/source/dnode/vnode/tsdb/inc/tsdbDef.h @@ -27,9 +27,9 @@ extern "C" { #endif struct STsdb { - char * path; - STsdbOptions options; - STsdbMemAllocator *pTMA; + char * path; + STsdbOptions options; + SMemAllocatorFactory *pmaf; }; #ifdef __cplusplus From baac49c75b246ed7d369ad79e6b5a66c7a9cb601 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 12 Nov 2021 17:50:46 +0800 Subject: [PATCH 08/11] more --- include/dnode/vnode/vnode.h | 2 +- include/util/mallocator.h | 1 + source/dnode/vnode/impl/src/vnodeMain.c | 7 ++++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 51cd643caf..52470d60a9 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -45,7 +45,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeOptions *pVnodeOptions); /** * @brief Close a VNODE * - * @param pVnode The vnode object + * @param pVnode The vnode object to close */ void vnodeClose(SVnode *pVnode); diff --git a/include/util/mallocator.h b/include/util/mallocator.h index 6a10a26156..fd66811f38 100644 --- a/include/util/mallocator.h +++ b/include/util/mallocator.h @@ -25,6 +25,7 @@ extern "C" { typedef struct SMemAllocator SMemAllocator; struct SMemAllocator { + char name[16]; void *impl; void *(*malloc)(SMemAllocator *, uint64_t size); void *(*calloc)(SMemAllocator *, uint64_t nmemb, uint64_t size); diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index a0c1d38ea9..c70b692c30 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -87,6 +87,11 @@ static void vnodeFree(SVnode *pVnode) { static int vnodeOpenImpl(SVnode *pVnode) { char dir[TSDB_FILENAME_LEN]; + if (vnodeOpenBufPool(pVnode) < 0) { + // TODO: handle error + return -1; + } + // Open meta sprintf(dir, "%s/meta", pVnode->path); pVnode->pMeta = metaOpen(dir, &(pVnode->options.metaOptions)); @@ -111,7 +116,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { static void vnodeCloseImpl(SVnode *pVnode) { if (pVnode) { - // TODO: Close TQ + vnodeCloseBufPool(pVnode); tsdbClose(pVnode->pTsdb); metaClose(pVnode->pMeta); } From 75c708c9e6fdf207dd9a6a3be33561ef2836f51d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Nov 2021 10:27:06 +0800 Subject: [PATCH 09/11] make doxygen generate call hierarchy --- docs/Doxyfile.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 207157c415..4fa01d27c8 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -491,7 +491,7 @@ EXTRACT_PACKAGE = NO # included in the documentation. # The default value is: NO. -EXTRACT_STATIC = NO +EXTRACT_STATIC = YES # If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined # locally in source files will be included in the documentation. If set to NO, @@ -2416,7 +2416,7 @@ INCLUDED_BY_GRAPH = YES # The default value is: NO. # This tag requires that the tag HAVE_DOT is set to YES. -CALL_GRAPH = NO +CALL_GRAPH = YES # If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller # dependency graph for every global function or class method. @@ -2428,7 +2428,7 @@ CALL_GRAPH = NO # The default value is: NO. # This tag requires that the tag HAVE_DOT is set to YES. -CALLER_GRAPH = NO +CALLER_GRAPH = YES # If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical # hierarchy of all classes instead of a textual one. From 094ce843211817744e8b0572744c64feaa1882d9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Nov 2021 10:30:31 +0800 Subject: [PATCH 10/11] generate tree view --- docs/Doxyfile.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 4fa01d27c8..139300ea80 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -1526,7 +1526,7 @@ ECLIPSE_DOC_ID = org.doxygen.Project # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -DISABLE_INDEX = NO +DISABLE_INDEX = YES # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag @@ -1543,7 +1543,7 @@ DISABLE_INDEX = NO # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_TREEVIEW = NO +GENERATE_TREEVIEW = YES # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. From e12b2d043e223069e5319bb3c47bf0fba33fb1af Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Nov 2021 11:41:53 +0800 Subject: [PATCH 11/11] more --- source/dnode/vnode/impl/src/vnodeBufferPool.c | 145 ++++++++---------- 1 file changed, 61 insertions(+), 84 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index d5005cf98c..eba71b5ba3 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -16,15 +16,23 @@ #include "vnodeDef.h" /* ------------------------ STRUCTURES ------------------------ */ +#define VNODE_BUF_POOL_SHARDS 3 + struct SVBufPool { SList free; SList incycle; SListNode *inuse; }; -typedef enum { E_V_HEAP_ALLOCATOR = 0, E_V_ARENA_ALLOCATOR } EVMemAllocatorT; +typedef enum { + // Heap allocator + E_V_HEAP_ALLOCATOR = 0, + // Arena allocator + E_V_ARENA_ALLOCATOR +} EVMemAllocatorT; typedef struct { + /* TODO */ } SVHeapAllocator; typedef struct SVArenaNode { @@ -35,14 +43,16 @@ typedef struct SVArenaNode { } SVArenaNode; typedef struct { + uint64_t ssize; // step size + uint64_t lsize; // limit size SVArenaNode *inuse; SVArenaNode node; } SVArenaAllocator; typedef struct { + T_REF_DECLARE() uint64_t capacity; EVMemAllocatorT type; - T_REF_DECLARE() union { SVHeapAllocator vha; SVArenaAllocator vaa; @@ -51,10 +61,6 @@ typedef struct { static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); static void vBufPoolFreeNode(SListNode *pNode); -static int vArenaAllocatorInit(SVArenaAllocator *pvaa); -static void vArenaAllocatorClear(SVArenaAllocator *pvaa); -static int vHeapAllocatorInit(SVHeapAllocator *pvha); -static void vHeapAllocatorClear(SVHeapAllocator *pvha); int vnodeOpenBufPool(SVnode *pVnode) { uint64_t capacity; @@ -68,12 +74,12 @@ int vnodeOpenBufPool(SVnode *pVnode) { tdListInit(&(pVnode->pBufPool->free), 0); tdListInit(&(pVnode->pBufPool->incycle), 0); - capacity = pVnode->options.wsize / 3; + capacity = pVnode->options.wsize / VNODE_BUF_POOL_SHARDS; if (pVnode->options.isHeapAllocator) { type = E_V_HEAP_ALLOCATOR; } - for (int i = 0; i < 3; i++) { + for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) { SListNode *pNode = vBufPoolNewNode(capacity, type); if (pNode == NULL) { vnodeCloseBufPool(pVnode); @@ -83,8 +89,6 @@ int vnodeOpenBufPool(SVnode *pVnode) { tdListAppendNode(&(pVnode->pBufPool->free), pNode); } - pVnode->pBufPool->inuse = tdListPopHead(&(pVnode->pBufPool->free)); - return 0; } @@ -102,67 +106,49 @@ void vnodeCloseBufPool(SVnode *pVnode) { } // Free inuse node - vBufPoolFreeNode(pVnode->pBufPool->inuse); + if (pVnode->pBufPool->inuse) { + vBufPoolFreeNode(pVnode->pBufPool->inuse); + } free(pVnode->pBufPool); pVnode->pBufPool = NULL; } } -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode) { - SMemAllocator *pma; - - pma = (SMemAllocator *)calloc(1, sizeof(*pma)); - if (pma == NULL) { - /* TODO */ - return NULL; - } - - pma->impl = pVnode; - if (pVnode->options.isHeapAllocator) { - /* TODO */ - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } else { - /* TODO */ - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } - - return pma; -} - -void vnodeDestroyMemAllocator(SMemAllocator *pma) { tfree(pma); } - -void vnodeRefMemAllocator(SMemAllocator *pma) { - SVnode * pVnode = (SVnode *)pma->impl; - SVMemAllocator *pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - - T_REF_INC(pvma); -} - -void vnodeUnrefMemAllocator(SMemAllocator *pma) { - SVnode * pVnode = (SVnode *)pma->impl; - SVMemAllocator *pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - - if (T_REF_DEC(pvma) == 0) { - /* TODO */ - } -} - /* ------------------------ STATIC METHODS ------------------------ */ +static void vArenaAllocatorInit(SVArenaAllocator *pvaa, uint64_t capacity, uint64_t ssize, uint64_t lsize) { /* TODO */ + pvaa->ssize = ssize; + pvaa->lsize = lsize; + pvaa->inuse = &pvaa->node; + + pvaa->node.prev = NULL; + pvaa->node.size = capacity; + pvaa->node.ptr = pvaa->node.data; +} + +static void vArenaAllocatorClear(SVArenaAllocator *pvaa) { /* TODO */ + while (pvaa->inuse != &(pvaa->node)) { + SVArenaNode *pANode = pvaa->inuse; + pvaa->inuse = pANode->prev; + free(pANode); + } +} + static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { SListNode * pNode; SVMemAllocator *pvma; + uint64_t msize; + uint64_t ssize = 0; // TODO + uint64_t lsize = 0; // TODO - pNode = (SListNode *)calloc(1, sizeof(*pNode) + sizeof(SVMemAllocator)); + msize = sizeof(SListNode) + sizeof(SVMemAllocator); + if (type == E_V_ARENA_ALLOCATOR) { + msize += capacity; + } + + pNode = (SListNode *)calloc(1, msize); if (pNode == NULL) { + // TODO: handle error return NULL; } @@ -171,11 +157,11 @@ static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { pvma->type = type; switch (type) { - case E_V_HEAP_ALLOCATOR: - vHeapAllocatorInit(&(pvma->vha)); - break; case E_V_ARENA_ALLOCATOR: - vArenaAllocatorInit(&(pvma->vaa)); + vArenaAllocatorInit(&(pvma->vaa), capacity, ssize, lsize); + break; + case E_V_HEAP_ALLOCATOR: + // vHeapAllocatorInit(&(pvma->vha)); break; default: ASSERT(0); @@ -185,27 +171,18 @@ static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { } static void vBufPoolFreeNode(SListNode *pNode) { - if (pNode) { - free(pNode); + SVMemAllocator *pvma = (SVMemAllocator *)(pNode->data); + + switch (pvma->type) { + case E_V_ARENA_ALLOCATOR: + vArenaAllocatorClear(&(pvma->vaa)); + break; + case E_V_HEAP_ALLOCATOR: + // vHeapAllocatorClear(&(pvma->vha)); + break; + default: + break; } -} -// --------------- For arena allocator -static int vArenaAllocatorInit(SVArenaAllocator *pvaa) { - // TODO - return 0; -} - -static void vArenaAllocatorClear(SVArenaAllocator *pvaa) { - // TODO -} - -// --------------- For heap allocator -static int vHeapAllocatorInit(SVHeapAllocator *pvha) { - // TODO - return 0; -} - -static void vHeapAllocatorClear(SVHeapAllocator *pvha) { - // TODO + free(pNode); } \ No newline at end of file