From 98059c65e39bea413b827a70b48f1aed525e1fd9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 10 Dec 2021 11:21:07 +0800 Subject: [PATCH 01/16] more --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ba8543e518..14449987d9 100644 --- a/.gitignore +++ b/.gitignore @@ -101,3 +101,4 @@ TAGS contrib/* !contrib/CMakeLists.txt !contrib/test +.devcontainer/ \ No newline at end of file From 22a2d4299271f08c81f3ae903f3b90b8e372173f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 11 Dec 2021 07:00:52 +0000 Subject: [PATCH 02/16] more --- .devcontainer/Dockerfile | 9 ++++++++ .devcontainer/devcontainer.json | 32 ++++++++++++++++++++++++++ .gitignore | 3 +-- source/dnode/vnode/impl/CMakeLists.txt | 6 ++--- 4 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..fc9a51af65 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp/.devcontainer/base.Dockerfile + +# [Choice] Debian / Ubuntu version (use Debian 11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 +ARG VARIANT="bullseye" +FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT} + +# [Optional] Uncomment this section to install additional packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..9b752d091d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp +{ + "name": "C++", + "build": { + "dockerfile": "Dockerfile", + // Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 + // Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon + "args": { "VARIANT": "ubuntu-21.04" } + }, + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + + // Set *default* container specific settings.json values on container create. + "settings": {}, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "austin.code-gnu-global", + "visualstudioexptteam.vscodeintel" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "gcc -v", + + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} diff --git a/.gitignore b/.gitignore index 14449987d9..ab4fbda71f 100644 --- a/.gitignore +++ b/.gitignore @@ -100,5 +100,4 @@ TAGS contrib/* !contrib/CMakeLists.txt -!contrib/test -.devcontainer/ \ No newline at end of file +!contrib/test \ No newline at end of file diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index d6d267c4d4..6972605afd 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -18,6 +18,6 @@ target_link_libraries( ) # test -#if(${BUILD_TEST}) -# add_subdirectory(test) -#endif(${BUILD_TEST}) \ No newline at end of file +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file From ae18e4c390487507668f1b7e20204e9a47df1337 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 11 Dec 2021 17:28:31 +0800 Subject: [PATCH 03/16] more work --- CMakeLists.txt | 2 +- source/dnode/vnode/impl/inc/vnodeBufferPool.h | 1 + source/dnode/vnode/impl/inc/vnodeCommit.h | 4 ++-- source/dnode/vnode/impl/src/vnodeBufferPool.c | 22 +++++++++++++++++-- source/dnode/vnode/impl/src/vnodeCommit.c | 2 -- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b626977588..9762a466b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake") set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib") include(${CMAKE_SUPPORT_DIR}/cmake.options) -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3") # contrib add_subdirectory(contrib) diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/impl/inc/vnodeBufferPool.h index d64dc93847..bfc4de9e12 100644 --- a/source/dnode/vnode/impl/inc/vnodeBufferPool.h +++ b/source/dnode/vnode/impl/inc/vnodeBufferPool.h @@ -28,6 +28,7 @@ typedef struct SVBufPool SVBufPool; int vnodeOpenBufPool(SVnode *pVnode); void vnodeCloseBufPool(SVnode *pVnode); void *vnodeMalloc(SVnode *pVnode, uint64_t size); +bool vnodeBufPoolIsFull(SVnode *pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/inc/vnodeCommit.h b/source/dnode/vnode/impl/inc/vnodeCommit.h index c8ff4947aa..a60e8feac2 100644 --- a/source/dnode/vnode/impl/inc/vnodeCommit.h +++ b/source/dnode/vnode/impl/inc/vnodeCommit.h @@ -22,8 +22,8 @@ extern "C" { #endif -bool vnodeShouldCommit(SVnode *pVnode); -int vnodeAsyncCommit(SVnode *pVnode); +#define vnodeShouldCommit vnodeBufPoolIsFull +int vnodeAsyncCommit(SVnode *pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 00203ed9b6..7415033f02 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -148,6 +148,24 @@ void *vnodeMalloc(SVnode *pVnode, uint64_t size) { return vBufPoolMalloc(pvma, size); } +bool vnodeBufPoolIsFull(SVnode *pVnode) { + SVBufPool * pBufPool = pVnode->pBufPool; + SVMemAllocator *pvma; + + if (pBufPool->inuse == NULL) return false; + + pvma = (SVMemAllocator *)(pBufPool->inuse->data); + if (pvma->type == E_V_HEAP_ALLOCATOR) { + ASSERT(0); + } else { + SVArenaNode *pNode = pvma->vaa.inuse; + bool ret = + (pNode != &(pvma->vaa.node)) || ((pNode->size - POINTER_DISTANCE(pNode->ptr, pNode->data)) <= pvma->vaa.lsize); + + return ret; + } +} + /* ------------------------ STATIC METHODS ------------------------ */ static void vArenaAllocatorInit(SVArenaAllocator *pvaa, uint64_t capacity, uint64_t ssize, uint64_t lsize) { /* TODO */ pvaa->ssize = ssize; @@ -171,8 +189,8 @@ static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { SListNode * pNode; SVMemAllocator *pvma; uint64_t msize; - uint64_t ssize = 0; // TODO - uint64_t lsize = 0; // TODO + uint64_t ssize = 4096; // TODO + uint64_t lsize = 1024; // TODO msize = sizeof(SListNode) + sizeof(SVMemAllocator); if (type == E_V_ARENA_ALLOCATOR) { diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/impl/src/vnodeCommit.c index 18a0c6d91d..cac7999f59 100644 --- a/source/dnode/vnode/impl/src/vnodeCommit.c +++ b/source/dnode/vnode/impl/src/vnodeCommit.c @@ -18,8 +18,6 @@ static int vnodeStartCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode); -bool vnodeShouldCommit(SVnode *pVnode) { return false; } - int vnodeAsyncCommit(SVnode *pVnode) { #if 0 if (vnodeStartCommit(pVnode) < 0) { From 556c7912a652ae3d43be76cccf2816c1f6e5a436 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 11 Dec 2021 22:41:16 +0800 Subject: [PATCH 04/16] integrate WAL --- source/dnode/vnode/impl/src/vnodeCfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/src/vnodeCfg.c b/source/dnode/vnode/impl/src/vnodeCfg.c index 01facba888..7aac9ca6a7 100644 --- a/source/dnode/vnode/impl/src/vnodeCfg.c +++ b/source/dnode/vnode/impl/src/vnodeCfg.c @@ -15,7 +15,7 @@ #include "vnodeDef.h" -const SVnodeCfg defaultVnodeOptions = {0}; /* TODO */ +const SVnodeCfg defaultVnodeOptions = {.wsize = 16*1024*1024, .walCfg = {.walLevel = TAOS_WAL_WRITE}}; /* TODO */ void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */ vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions); From 9f9e2ebd4c10955e9852645e9a4634d8393ddcc6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 10:47:52 +0800 Subject: [PATCH 05/16] add module init --- include/util/tmacro.h | 44 +++++++++++++++++++++++++ source/dnode/vnode/impl/src/vnodeMain.c | 13 +++++++- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 include/util/tmacro.h diff --git a/include/util/tmacro.h b/include/util/tmacro.h new file mode 100644 index 0000000000..74056cfe07 --- /dev/null +++ b/include/util/tmacro.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_MACRO_H_ +#define _TD_UTIL_MACRO_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Module init/clear MACRO definitions +#define TD_MOD_UNINITIALIZED 0 +#define TD_MOD_INITIALIZED 1 + +#define TD_MOD_UNCLEARD 0 +#define TD_MOD_CLEARD 1 + +#define TD_DEF_MOD_INIT_FLAG(MOD) static int8_t MOD##InitFlag = TD_MOD_UNINITIALIZED +#define TD_DEF_MOD_CLEAR_FLAG(MOD) static int8_t MOD##ClearFlag = TD_MOD_UNCLEARD + +#define TD_CHECK_AND_SET_MODE_INIT(MOD) \ + atomic_val_compare_exchange_8(&(MOD##InitFlag), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED) + +#define TD_CHECK_AND_SET_MOD_CLEAR(MOD) atomic_val_compare_exchange_8(&(MOD##ClearFlag), TD_MOD_UNCLEARD, TD_MOD_CLEARD) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_MACRO_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index ab33b58858..9b94b4a361 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tmacro.h" #include "vnodeDef.h" static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg); @@ -20,8 +21,14 @@ static void vnodeFree(SVnode *pVnode); static int vnodeOpenImpl(SVnode *pVnode); static void vnodeCloseImpl(SVnode *pVnode); +TD_DEF_MOD_INIT_FLAG(vnode); +TD_DEF_MOD_CLEAR_FLAG(vnode); + int vnodeInit() { - // TODO + if (TD_CHECK_AND_SET_MODE_INIT(vnode) == TD_MOD_INITIALIZED) { + return 0; + } + if (walInit() < 0) { return -1; } @@ -30,6 +37,10 @@ int vnodeInit() { } void vnodeClear() { + if (TD_CHECK_AND_SET_MOD_CLEAR(vnode) == TD_MOD_CLEARD) { + return; + } + walCleanUp(); } From 103a6225901a7dfa1b10aa4f5e45951b38eecb71 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 16:31:39 +0800 Subject: [PATCH 06/16] more --- include/dnode/vnode/vnode.h | 2 + include/util/tdlist.h | 132 ++++++++++++++++ source/dnode/vnode/impl/inc/vnodeDef.h | 4 +- .../dnode/vnode/impl/inc/vnodeMemAllocator.h | 28 +++- .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 70 +++++++++ source/dnode/vnode/impl/src/vnodeBufferPool.c | 141 +++++++++--------- .../dnode/vnode/impl/src/vnodeMemAllocator.c | 113 -------------- 7 files changed, 300 insertions(+), 190 deletions(-) create mode 100644 include/util/tdlist.h create mode 100644 source/dnode/vnode/impl/src/vnodeArenaMAImpl.c delete mode 100644 source/dnode/vnode/impl/src/vnodeMemAllocator.c diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 30531ad738..007ce83812 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -36,6 +36,8 @@ typedef struct SVnodeCfg { struct { /** write buffer size */ uint64_t wsize; + uint64_t ssize; + uint64_t lsize; /** use heap allocator or arena allocator */ bool isHeapAllocator; }; diff --git a/include/util/tdlist.h b/include/util/tdlist.h new file mode 100644 index 0000000000..7ebd8f5d09 --- /dev/null +++ b/include/util/tdlist.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_TDLIST_H_ +#define _TD_UTIL_TDLIST_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define TD_LIST_NODE(S) \ + struct { \ + S *prev_; \ + S *next_; \ + } + +#define TD_LIST(S) \ + struct { \ + S * head_; \ + S * tail_; \ + int neles_; \ + } + +#define tlistInit(l) \ + (l)->head_ = (l)->tail_ = NULL; \ + (l)->neles_ = 0; + +#define tlistHead(l) (l)->head_ +#define tlistTail(l) (l)->tail_ +#define tlistNEles(l) (l)->neles_ + +#define tlistAppend(l, n) \ + if ((l)->head_ == NULL) { \ + (n)->prev_ = (n)->next_ = NULL; \ + (l)->head_ = (l)->tail_ = (n); \ + } else { \ + (n)->prev_ = (l)->tail_; \ + (n)->next_ = NULL; \ + (l)->tail_->next_ = (n); \ + (l)->tail_ = (n); \ + } \ + (l)->neles_ += 1; + +#define tlistPrepend(l, n) \ + if ((l)->head_ == NULL) { \ + (n)->prev_ = (n)->next_ = NULL; \ + (l)->head_ = (l)->tail_ = (n); \ + } else { \ + (n)->prev_ = NULL; \ + (n)->next_ = (l)->head_; \ + (l)->head_->prev_ = (n); \ + (l)->head_ = (n); \ + } \ + (l)->neles_ += 1; + +#define tlistPop(l, n) \ + ({ \ + if ((n)) { \ + if ((l)->head_ == (n)) { \ + (l)->head_ = (n)->next_; \ + } \ + if ((l)->tail_ == (n)) { \ + (l)->tail_ = (n)->prev_; \ + } \ + if ((n)->prev_ != NULL) { \ + (n)->prev_->next_ = (n)->next_; \ + } \ + if ((n)->next_ != NULL) { \ + (n)->next_->prev_ = (n)->prev_; \ + } \ + (l)->neles_ -= 1; \ + (n)->prev_ = (n)->next_ = NULL; \ + } \ + (n); \ + }) + +#define tlistPopHead(l) tlistPop(l, (l)->head_) + +#define tlistPopTail(l) tlistPop(l, (l)->tail_) + +#define tlistIterInit(it, l, dir) + +// List iterator +#define TD_LIST_FITER 0 +#define TD_LIST_BITER 1 +#define TD_LIST_ITER(S) \ + struct { \ + int it_dir_; \ + S * it_next_; \ + S * it_ptr_; \ + TD_LIST(S) * it_list_; \ + } + +#define tlistIterInit(it, l, dir) \ + (it)->it_dir_ = (dir); \ + (it)->it_list_ = l; \ + if ((dir) == TD_LIST_FITER) { \ + (it)->it_next_ = (l)->head_; \ + } else { \ + (it)->it_next_ = (l)->tail_; \ + } + +#define tlistIterNext(it) \ + ({ \ + (it)->it_ptr_ = (it)->it_next_; \ + if ((it)->it_next_ != NULL) { \ + if ((it)->it_dir_ == TD_LIST_FITER) { \ + (it)->it_next_ = (it)->it_next_->next_; \ + } else { \ + (it)->it_next_ = (it)->it_next_->prev_; \ + } \ + } \ + (it)->it_ptr_; \ + }) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_TDLIST_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index c92de433c3..e3a3fac6b9 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -18,15 +18,17 @@ #include "mallocator.h" #include "sync.h" +#include "tcoding.h" +#include "tdlist.h" #include "tlockfree.h" #include "wal.h" -#include "tcoding.h" #include "vnode.h" #include "vnodeBufferPool.h" #include "vnodeCfg.h" #include "vnodeCommit.h" #include "vnodeFS.h" +#include "vnodeMemAllocator.h" #include "vnodeRequest.h" #include "vnodeStateMgr.h" #include "vnodeSync.h" diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index 9184eb416b..bd014c6ff6 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -16,15 +16,35 @@ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_ -#include "mallocator.h" -#include "vnode.h" +#include "os.h" #ifdef __cplusplus extern "C" { #endif -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode); -void vnodeDestroyMemAllocator(SMemAllocator *pma); +typedef struct SVArenaNode SVArenaNode; +typedef struct SVMemAllocator SVMemAllocator; + +struct SVArenaNode { + TD_LIST_NODE(SVArenaNode); + uint64_t nsize; // current node size + void * ptr; + char data[]; +}; + +struct SVMemAllocator { + TD_LIST_NODE(SVMemAllocator); + uint64_t capacity; + uint64_t ssize; + uint64_t lsize; + TD_LIST(SVArenaNode) nlist; +}; + +SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize); +void vmaDestroy(SVMemAllocator *pVMA); +void vmaReset(SVMemAllocator *pVMA); +void * vmaMalloc(SVMemAllocator *pVMA, uint64_t size); +void vmaFree(SVMemAllocator *pVMA, void *ptr); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c new file mode 100644 index 0000000000..e0c098b24b --- /dev/null +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeDef.h" + +static SVArenaNode *vArenaNodeNew(uint64_t capacity); +static void vArenaNodeFree(SVArenaNode *pNode); + +SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { + SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA)); + if (pVMA == NULL) { + // TODO: handle error + return NULL; + } + + pVMA->capacity = capacity; + pVMA->ssize = ssize; + pVMA->lsize = lsize; + tlistInit(&(pVMA->nlist)); + + SVArenaNode *pNode = vArenaNodeNew(capacity); + if (pNode == NULL) { + // TODO + return NULL; + } + + tlistAppend(&(pVMA->nlist), pNode); + + return pVMA; +} + +void vmaDestroy(SVMemAllocator *pVMA) { + // TODO +} + +void vmaReset(SVMemAllocator *pVMA) { + // TODO +} + +void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { + // TODO + return NULL; +} + +void vmaFree(SVMemAllocator *pVMA, void *ptr) { + // TODO +} + +/* ------------------------ STATIC METHODS ------------------------ */ +static SVArenaNode *vArenaNodeNew(uint64_t capacity) { + SVArenaNode *pNode = NULL; + // TODO + return pNode; +} + +static void vArenaNodeFree(SVArenaNode *pNode) { + // TODO +} \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 7415033f02..6a6c97a717 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -19,14 +19,76 @@ #define VNODE_BUF_POOL_SHARDS 3 struct SVBufPool { - // buffer pool impl - SList free; - SList incycle; - SListNode *inuse; + TD_LIST(SVMemAllocator) free; + TD_LIST(SVMemAllocator) incycle; + SVMemAllocator *inuse; // MAF for submodules - SMemAllocatorFactory maf; + // SMemAllocatorFactory maf; }; +int vnodeOpenBufPool(SVnode *pVnode) { + uint64_t capacity; + // EVMemAllocatorT type = E_V_ARENA_ALLOCATOR; + + if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { + /* TODO */ + return -1; + } + + tlistInit(&(pVnode->pBufPool->free)); + tlistInit(&(pVnode->pBufPool->incycle)); + + pVnode->pBufPool->inuse = NULL; + + // TODO + capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS; + + for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) { + SVMemAllocator *pVMA = vmaCreate(capacity, pVnode->config.ssize, pVnode->config.lsize); + if (pVMA == NULL) { + // TODO: handle error + return -1; + } + + tlistAppend(&(pVnode->pBufPool->free), pVMA); + } + + return 0; +} + +void vnodeCloseBufPool(SVnode *pVnode) { + if (pVnode->pBufPool) { + vmaDestroy(pVnode->pBufPool->inuse); + + while (true) { + SVMemAllocator *pVMA = tlistPopHead(&(pVnode->pBufPool->incycle)); + if (pVMA == NULL) break; + vmaDestroy(pVMA); + } + + while (true) { + SVMemAllocator *pVMA = tlistPopHead(&(pVnode->pBufPool->free)); + if (pVMA == NULL) break; + vmaDestroy(pVMA); + } + + free(pVnode->pBufPool); + pVnode->pBufPool = NULL; + } +} + +void *vnodeMalloc(SVnode *pVnode, uint64_t size) { + // TODO + return NULL; +} + +bool vnodeBufPoolIsFull(SVnode *pVnode) { + // TODO + return false; +} + +#if 0 + typedef enum { // Heap allocator E_V_HEAP_ALLOCATOR = 0, @@ -57,15 +119,6 @@ typedef struct { SListNode *pNode; } SVMAWrapper; -typedef struct { - T_REF_DECLARE() - uint64_t capacity; - EVMemAllocatorT type; - union { - SVHeapAllocator vha; - SVArenaAllocator vaa; - }; -} SVMemAllocator; static SListNode * vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); static void vBufPoolFreeNode(SListNode *pNode); @@ -73,63 +126,6 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf); static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); -int vnodeOpenBufPool(SVnode *pVnode) { - uint64_t capacity; - EVMemAllocatorT type = E_V_ARENA_ALLOCATOR; - - if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { - /* TODO */ - return -1; - } - - tdListInit(&(pVnode->pBufPool->free), 0); - tdListInit(&(pVnode->pBufPool->incycle), 0); - - capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS; - if (pVnode->config.isHeapAllocator) { - type = E_V_HEAP_ALLOCATOR; - } - - for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) { - SListNode *pNode = vBufPoolNewNode(capacity, type); - if (pNode == NULL) { - vnodeCloseBufPool(pVnode); - return -1; - } - - tdListAppendNode(&(pVnode->pBufPool->free), pNode); - } - - pVnode->pBufPool->maf.impl = pVnode; - pVnode->pBufPool->maf.create = vBufPoolCreateMA; - pVnode->pBufPool->maf.destroy = vBufPoolDestroyMA; - - return 0; -} - -void vnodeCloseBufPool(SVnode *pVnode) { - SListNode *pNode; - if (pVnode->pBufPool) { - // Clear free list - while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) != NULL) { - vBufPoolFreeNode(pNode); - } - - // Clear incycle list - while ((pNode = tdListPopHead(&(pVnode->pBufPool->incycle))) != NULL) { - vBufPoolFreeNode(pNode); - } - - // Free inuse node - if (pVnode->pBufPool->inuse) { - vBufPoolFreeNode(pVnode->pBufPool->inuse); - } - - free(pVnode->pBufPool); - pVnode->pBufPool = NULL; - } -} - void *vnodeMalloc(SVnode *pVnode, uint64_t size) { void *ptr; @@ -335,4 +331,5 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma) { tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode)); // tsem_post(); todo: sem_post } -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeMemAllocator.c b/source/dnode/vnode/impl/src/vnodeMemAllocator.c deleted file mode 100644 index 902014eb47..0000000000 --- a/source/dnode/vnode/impl/src/vnodeMemAllocator.c +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" - -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode) { - SMemAllocator *pma = NULL; - /* TODO */ - return pma; -} - -void vnodeDestroyMemAllocator(SMemAllocator *pma) { - // TODO -} - -#if 0 -#define VNODE_HEAP_ALLOCATOR 0 -#define VNODE_ARENA_ALLOCATOR 1 - -typedef struct { - uint64_t tsize; - uint64_t used; -} SVHeapAllocator; - -typedef struct SVArenaNode { - struct SVArenaNode *prev; - void * nptr; - char data[]; -} SVArenaNode; - -typedef struct { - SVArenaNode *inuse; - SVArenaNode node; -} SVArenaAllocator; - -typedef struct { - int8_t type; - uint64_t tsize; - T_REF_DECLARE() - union { - SVHeapAllocator vha; - SVArenaAllocator vaa; - }; -} SVMemAllocator; - -SMemAllocator *vnodeCreateMemAllocator(int8_t type, uint64_t tsize, uint64_t ssize /* step size only for arena */) { - SMemAllocator * pma; - uint64_t msize; - SVMemAllocator *pva; - - msize = sizeof(*pma) + sizeof(SVMemAllocator); - if (type == VNODE_ARENA_ALLOCATOR) { - msize += tsize; - } - - pma = (SMemAllocator *)calloc(1, msize); - if (pma == NULL) { - return NULL; - } - - pma->impl = POINTER_SHIFT(pma, sizeof(*pma)); - pva = (SVMemAllocator *)(pma->impl); - pva->type = type; - pva->tsize = tsize; - - if (type == VNODE_HEAP_ALLOCATOR) { - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } else if (type == VNODE_ARENA_ALLOCATOR) { - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } else { - ASSERT(0); - } - - return pma; -} - -void vnodeDestroyMemAllocator(SMemAllocator *pma) { - // TODO -} - -void vnodeRefMemAllocator(SMemAllocator *pma) { - // TODO -} - -void vnodeUnrefMemAllocator(SMemAllocator *pma) { - // TODO -} - -/* ------------------------ Heap Allocator IMPL ------------------------ */ - -/* ------------------------ Arena Allocator IMPL ------------------------ */ - -#endif \ No newline at end of file From f748f0710a20a1584dd4ae84b848f35c302c99b6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 17:19:30 +0800 Subject: [PATCH 07/16] more --- CMakeLists.txt | 6 +++++- source/dnode/vnode/impl/test/CMakeLists.txt | 7 ++++++- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 11 +++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9762a466b1..1a8cb1d710 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,9 +20,13 @@ add_library(api INTERFACE) target_include_directories(api INTERFACE "include/client") # src +if(${BUILD_TEST}) + include(CTest) + enable_testing() +endif(${BUILD_TEST}) add_subdirectory(source) # docs add_subdirectory(docs) -# tests (TODO) +# tests (TODO) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/CMakeLists.txt b/source/dnode/vnode/impl/test/CMakeLists.txt index 83506a4fde..e1226331e9 100644 --- a/source/dnode/vnode/impl/test/CMakeLists.txt +++ b/source/dnode/vnode/impl/test/CMakeLists.txt @@ -4,4 +4,9 @@ target_sources(vnodeApiTests PRIVATE "vnodeApiTests.cpp" ) -target_link_libraries(vnodeApiTests vnode gtest gtest_main) \ No newline at end of file +target_link_libraries(vnodeApiTests vnode gtest gtest_main) + +add_test( + NAME vnode_api_tests + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests + ) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index df784181b7..ac2ccbc132 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -1,3 +1,14 @@ +/** + * @file vnodeApiTests.cpp + * @author hzcheng (hzcheng@taosdata.com) + * @brief VNODE module API tests + * @version 0.1 + * @date 2021-12-13 + * + * @copyright Copyright (c) 2021 + * + */ + #include #include From 8b6eba666fff5d8a519e2d839148f7e55b16f9a1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 17:57:14 +0800 Subject: [PATCH 08/16] more --- include/util/tdlist.h | 2 - .../dnode/vnode/impl/inc/vnodeMemAllocator.h | 3 +- .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 65 ++++++++++++++++--- 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 7ebd8f5d09..757b404ad6 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -90,8 +90,6 @@ extern "C" { #define tlistPopTail(l) tlistPop(l, (l)->tail_) -#define tlistIterInit(it, l, dir) - // List iterator #define TD_LIST_FITER 0 #define TD_LIST_BITER 1 diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index bd014c6ff6..df8b367d25 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -27,7 +27,7 @@ typedef struct SVMemAllocator SVMemAllocator; struct SVArenaNode { TD_LIST_NODE(SVArenaNode); - uint64_t nsize; // current node size + uint64_t size; // current node size void * ptr; char data[]; }; @@ -45,6 +45,7 @@ void vmaDestroy(SVMemAllocator *pVMA); void vmaReset(SVMemAllocator *pVMA); void * vmaMalloc(SVMemAllocator *pVMA, uint64_t size); void vmaFree(SVMemAllocator *pVMA, void *ptr); +bool vmaIsFull(SVMemAllocator *pVMA); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index e0c098b24b..532c2dad8e 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -21,7 +21,6 @@ static void vArenaNodeFree(SVArenaNode *pNode); SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA)); if (pVMA == NULL) { - // TODO: handle error return NULL; } @@ -32,7 +31,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { SVArenaNode *pNode = vArenaNodeNew(capacity); if (pNode == NULL) { - // TODO + free(pVMA); return NULL; } @@ -42,29 +41,79 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { } void vmaDestroy(SVMemAllocator *pVMA) { - // TODO + if (pVMA) { + while (true) { + SVArenaNode *pNode = tlistPopTail(&(pVMA->nlist)); + + if (pNode) { + vArenaNodeFree(pNode); + } else { + break; + } + } + + free(pVMA); + } } void vmaReset(SVMemAllocator *pVMA) { - // TODO + while (tlistNEles(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = tlistPopTail(&(pVMA->nlist)); + vArenaNodeFree(pNode); + } + + SVArenaNode *pNode = tlistHead(&(pVMA->nlist)); + pNode->ptr = pNode->data; } void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { - // TODO - return NULL; + SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + void * ptr; + + if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { + uint64_t capacity = MAX(pVMA->ssize, size); + pNode = vArenaNodeNew(capacity); + if (pNode == NULL) { + // TODO: handle error + return NULL; + } + + tlistAppend(&(pVMA->nlist), pNode); + } + + ptr = pNode->ptr; + pNode->ptr = POINTER_SHIFT(ptr, size); + + return ptr; } void vmaFree(SVMemAllocator *pVMA, void *ptr) { // TODO } +bool vmaIsFull(SVMemAllocator *pVMA) { + SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + + return (tlistNEles(&(pVMA->nlist)) > 1) || (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); +} + /* ------------------------ STATIC METHODS ------------------------ */ static SVArenaNode *vArenaNodeNew(uint64_t capacity) { SVArenaNode *pNode = NULL; - // TODO + + pNode = (SVArenaNode *)malloc(sizeof(*pNode) + capacity); + if (pNode == NULL) { + return NULL; + } + + pNode->size = capacity; + pNode->ptr = pNode->data; + return pNode; } static void vArenaNodeFree(SVArenaNode *pNode) { - // TODO + if (pNode) { + free(pNode); + } } \ No newline at end of file From b5b484a79cf6f427e5da3a4a5ab35dd0bfc6dc72 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 18:06:32 +0800 Subject: [PATCH 09/16] more --- source/dnode/vnode/impl/src/vnodeBufferPool.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 6a6c97a717..7738b81bb2 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -78,13 +78,24 @@ void vnodeCloseBufPool(SVnode *pVnode) { } void *vnodeMalloc(SVnode *pVnode, uint64_t size) { - // TODO - return NULL; + SVBufPool *pBufPool = pVnode->pBufPool; + + if (pBufPool->inuse == NULL) { + while (true) { + // TODO: add sem_wait and sem_post + pBufPool->inuse = tlistPopHead(&(pBufPool->free)); + if (pBufPool->inuse) { + break; + } + } + } + + return vmaMalloc(pBufPool->inuse, size); } bool vnodeBufPoolIsFull(SVnode *pVnode) { - // TODO - return false; + if (pVnode->pBufPool->inuse == NULL) return false; + return vmaIsFull(pVnode->pBufPool->inuse); } #if 0 From 0eff683d18caa4906f7b4516140b22223aa2b522 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 18:14:45 +0800 Subject: [PATCH 10/16] more work --- source/dnode/vnode/impl/src/vnodeBufferPool.c | 54 ------------------- 1 file changed, 54 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 7738b81bb2..fe15e9ebff 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -137,61 +137,7 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf); static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); -void *vnodeMalloc(SVnode *pVnode, uint64_t size) { - void *ptr; - - if (pVnode->pBufPool->inuse == NULL) { - SListNode *pNode; - while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) == NULL) { - // todo - // tsem_wait(); - ASSERT(0); - } - - pVnode->pBufPool->inuse = pNode; - } - - SVMemAllocator *pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - return vBufPoolMalloc(pvma, size); -} - -bool vnodeBufPoolIsFull(SVnode *pVnode) { - SVBufPool * pBufPool = pVnode->pBufPool; - SVMemAllocator *pvma; - - if (pBufPool->inuse == NULL) return false; - - pvma = (SVMemAllocator *)(pBufPool->inuse->data); - if (pvma->type == E_V_HEAP_ALLOCATOR) { - ASSERT(0); - } else { - SVArenaNode *pNode = pvma->vaa.inuse; - bool ret = - (pNode != &(pvma->vaa.node)) || ((pNode->size - POINTER_DISTANCE(pNode->ptr, pNode->data)) <= pvma->vaa.lsize); - - return ret; - } -} - /* ------------------------ STATIC METHODS ------------------------ */ -static void vArenaAllocatorInit(SVArenaAllocator *pvaa, uint64_t capacity, uint64_t ssize, uint64_t lsize) { /* TODO */ - pvaa->ssize = ssize; - pvaa->lsize = lsize; - pvaa->inuse = &pvaa->node; - - pvaa->node.prev = NULL; - pvaa->node.size = capacity; - pvaa->node.ptr = pvaa->node.data; -} - -static void vArenaAllocatorClear(SVArenaAllocator *pvaa) { /* TODO */ - while (pvaa->inuse != &(pvaa->node)) { - SVArenaNode *pANode = pvaa->inuse; - pvaa->inuse = pANode->prev; - free(pANode); - } -} - static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { SListNode * pNode; SVMemAllocator *pvma; From 04c83f497d52dfb4386ac517c6f45701668dad54 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 20:41:03 +0800 Subject: [PATCH 11/16] more --- include/util/tdlist.h | 39 +++++++------------ .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 6 ++- source/dnode/vnode/impl/src/vnodeBufferPool.c | 9 +++-- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 2 +- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 757b404ad6..0f341801e0 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -65,30 +65,21 @@ extern "C" { } \ (l)->neles_ += 1; -#define tlistPop(l, n) \ - ({ \ - if ((n)) { \ - if ((l)->head_ == (n)) { \ - (l)->head_ = (n)->next_; \ - } \ - if ((l)->tail_ == (n)) { \ - (l)->tail_ = (n)->prev_; \ - } \ - if ((n)->prev_ != NULL) { \ - (n)->prev_->next_ = (n)->next_; \ - } \ - if ((n)->next_ != NULL) { \ - (n)->next_->prev_ = (n)->prev_; \ - } \ - (l)->neles_ -= 1; \ - (n)->prev_ = (n)->next_ = NULL; \ - } \ - (n); \ - }) - -#define tlistPopHead(l) tlistPop(l, (l)->head_) - -#define tlistPopTail(l) tlistPop(l, (l)->tail_) +#define tlistPop(l, n) \ + if ((l)->head_ == (n)) { \ + (l)->head_ = (n)->next_; \ + } \ + if ((l)->tail_ == (n)) { \ + (l)->tail_ = (n)->prev_; \ + } \ + if ((n)->prev_ != NULL) { \ + (n)->prev_->next_ = (n)->next_; \ + } \ + if ((n)->next_ != NULL) { \ + (n)->next_->prev_ = (n)->prev_; \ + } \ + (l)->neles_ -= 1; \ + (n)->prev_ = (n)->next_ = NULL; // List iterator #define TD_LIST_FITER 0 diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index 532c2dad8e..748808a9fd 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -43,9 +43,10 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { void vmaDestroy(SVMemAllocator *pVMA) { if (pVMA) { while (true) { - SVArenaNode *pNode = tlistPopTail(&(pVMA->nlist)); + SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); if (pNode) { + tlistPop(&(pVMA->nlist), pNode); vArenaNodeFree(pNode); } else { break; @@ -58,7 +59,8 @@ void vmaDestroy(SVMemAllocator *pVMA) { void vmaReset(SVMemAllocator *pVMA) { while (tlistNEles(&(pVMA->nlist)) > 1) { - SVArenaNode *pNode = tlistPopTail(&(pVMA->nlist)); + SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + tlistPop(&(pVMA->nlist), pNode); vArenaNodeFree(pNode); } diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index fe15e9ebff..d5fcdf91e3 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -61,14 +61,16 @@ void vnodeCloseBufPool(SVnode *pVnode) { vmaDestroy(pVnode->pBufPool->inuse); while (true) { - SVMemAllocator *pVMA = tlistPopHead(&(pVnode->pBufPool->incycle)); + SVMemAllocator *pVMA = tlistHead(&(pVnode->pBufPool->incycle)); if (pVMA == NULL) break; + tlistPop(&(pVnode->pBufPool->incycle), pVMA); vmaDestroy(pVMA); } while (true) { - SVMemAllocator *pVMA = tlistPopHead(&(pVnode->pBufPool->free)); + SVMemAllocator *pVMA = tlistHead(&(pVnode->pBufPool->free)); if (pVMA == NULL) break; + tlistPop(&(pVnode->pBufPool->free), pVMA); vmaDestroy(pVMA); } @@ -83,8 +85,9 @@ void *vnodeMalloc(SVnode *pVnode, uint64_t size) { if (pBufPool->inuse == NULL) { while (true) { // TODO: add sem_wait and sem_post - pBufPool->inuse = tlistPopHead(&(pBufPool->free)); + pBufPool->inuse = tlistHead(&(pBufPool->free)); if (pBufPool->inuse) { + tlistPop(&(pBufPool->free), pBufPool->inuse); break; } } diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index ac2ccbc132..ec7e40dc0d 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -131,7 +131,7 @@ TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { { // Create some child tables - int ntables = 1000000; + int ntables = 1000; int batch = 10; for (int i = 0; i < ntables / batch; i++) { SArray *pMsgs = (SArray *)taosArrayInit(batch, sizeof(SRpcMsg *)); From 2c053411122fa037578f2c4bf78015d3f11a2990 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 20:44:55 +0800 Subject: [PATCH 12/16] more --- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index ec7e40dc0d..ac2ccbc132 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -131,7 +131,7 @@ TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { { // Create some child tables - int ntables = 1000; + int ntables = 1000000; int batch = 10; for (int i = 0; i < ntables / batch; i++) { SArray *pMsgs = (SArray *)taosArrayInit(batch, sizeof(SRpcMsg *)); From d66cba75e1e4aba185cb69629b6d564fe5bf696e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 21:06:33 +0800 Subject: [PATCH 13/16] more --- include/util/tdlist.h | 49 +++++++++++++------ .../dnode/vnode/impl/inc/vnodeMemAllocator.h | 6 +-- source/dnode/vnode/impl/src/vnodeBufferPool.c | 4 +- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 0f341801e0..4be1779c74 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -20,17 +20,36 @@ extern "C" { #endif -#define TD_LIST_NODE(S) \ - struct { \ - S *prev_; \ - S *next_; \ +// Single linked list +#define TD_SLIST_NODE(TYPE) \ + struct { \ + struct type *sl_next_; \ } -#define TD_LIST(S) \ - struct { \ - S * head_; \ - S * tail_; \ - int neles_; \ +#define TD_SLIST(TYPE) \ + struct { \ + struct TYPE *sl_head_; \ + } + +#define TD_SLIST_NODE_NEXT(sln) (sln)->sl_next_ + +#define tSListInit(sl) \ + do { \ + (sl)->sl_head_ = NULL; \ + } while (0) + +// Double linked list +#define TD_DLIST_NODE(S) \ + struct { \ + S *prev_; \ + S *next_; \ + } + +#define TD_DLIST(S) \ + struct { \ + S * head_; \ + S * tail_; \ + int neles_; \ } #define tlistInit(l) \ @@ -84,12 +103,12 @@ extern "C" { // List iterator #define TD_LIST_FITER 0 #define TD_LIST_BITER 1 -#define TD_LIST_ITER(S) \ - struct { \ - int it_dir_; \ - S * it_next_; \ - S * it_ptr_; \ - TD_LIST(S) * it_list_; \ +#define TD_LIST_ITER(S) \ + struct { \ + int it_dir_; \ + S * it_next_; \ + S * it_ptr_; \ + TD_DLIST(S) * it_list_; \ } #define tlistIterInit(it, l, dir) \ diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index df8b367d25..6b9fddaa02 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -26,18 +26,18 @@ typedef struct SVArenaNode SVArenaNode; typedef struct SVMemAllocator SVMemAllocator; struct SVArenaNode { - TD_LIST_NODE(SVArenaNode); + TD_DLIST_NODE(SVArenaNode); uint64_t size; // current node size void * ptr; char data[]; }; struct SVMemAllocator { - TD_LIST_NODE(SVMemAllocator); + TD_DLIST_NODE(SVMemAllocator); uint64_t capacity; uint64_t ssize; uint64_t lsize; - TD_LIST(SVArenaNode) nlist; + TD_DLIST(SVArenaNode) nlist; }; SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize); diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index d5fcdf91e3..347fe44e26 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -19,8 +19,8 @@ #define VNODE_BUF_POOL_SHARDS 3 struct SVBufPool { - TD_LIST(SVMemAllocator) free; - TD_LIST(SVMemAllocator) incycle; + TD_DLIST(SVMemAllocator) free; + TD_DLIST(SVMemAllocator) incycle; SVMemAllocator *inuse; // MAF for submodules // SMemAllocatorFactory maf; From aaf9ca18c51ade1843dee8ab46fd38dfb8cf4ec9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 21:18:43 +0800 Subject: [PATCH 14/16] more --- include/util/tdlist.h | 29 ++++++++++++------- .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 2 +- source/dnode/vnode/impl/src/vnodeBufferPool.c | 4 +-- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 4be1779c74..2be9ba3895 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -31,28 +31,35 @@ extern "C" { struct TYPE *sl_head_; \ } -#define TD_SLIST_NODE_NEXT(sln) (sln)->sl_next_ +#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) #define tSListInit(sl) \ do { \ (sl)->sl_head_ = NULL; \ } while (0) +#define tSListPrepend(sl, sln) \ + do { \ + TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ + TD_SLIST_HEAD(sl) = (sln); \ + } while (0); + // Double linked list -#define TD_DLIST_NODE(S) \ - struct { \ - S *prev_; \ - S *next_; \ +#define TD_DLIST_NODE(TYPE) \ + struct { \ + TYPE *prev_; \ + TYPE *next_; \ } -#define TD_DLIST(S) \ - struct { \ - S * head_; \ - S * tail_; \ - int neles_; \ +#define TD_DLIST(TYPE) \ + struct { \ + TYPE *head_; \ + TYPE *tail_; \ + int neles_; \ } -#define tlistInit(l) \ +#define tDListInit(l) \ (l)->head_ = (l)->tail_ = NULL; \ (l)->neles_ = 0; diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index 748808a9fd..b5424894ca 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -27,7 +27,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { pVMA->capacity = capacity; pVMA->ssize = ssize; pVMA->lsize = lsize; - tlistInit(&(pVMA->nlist)); + tDListInit(&(pVMA->nlist)); SVArenaNode *pNode = vArenaNodeNew(capacity); if (pNode == NULL) { diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 347fe44e26..084a7d18a7 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -35,8 +35,8 @@ int vnodeOpenBufPool(SVnode *pVnode) { return -1; } - tlistInit(&(pVnode->pBufPool->free)); - tlistInit(&(pVnode->pBufPool->incycle)); + tDListInit(&(pVnode->pBufPool->free)); + tDListInit(&(pVnode->pBufPool->incycle)); pVnode->pBufPool->inuse = NULL; From c0e308f275ddcd22ae8a1346823a95b8baec3c66 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 21:50:34 +0800 Subject: [PATCH 15/16] more --- include/util/tdlist.h | 154 ++++++++++-------- .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 23 +-- source/dnode/vnode/impl/src/vnodeBufferPool.c | 14 +- 3 files changed, 107 insertions(+), 84 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 2be9ba3895..8360d2713f 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -26,87 +26,108 @@ extern "C" { struct type *sl_next_; \ } -#define TD_SLIST(TYPE) \ - struct { \ - struct TYPE *sl_head_; \ +#define TD_SLIST(TYPE) \ + struct { \ + struct TYPE *sl_dl_head_; \ + int sl_dl_neles_; \ } -#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_HEAD(sl) ((sl)->sl_dl_head_) +#define TD_SLIST_NELES(sl) ((sl)->sl_dl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) -#define tSListInit(sl) \ - do { \ - (sl)->sl_head_ = NULL; \ +#define tSListInit(sl) \ + do { \ + (sl)->sl_dl_head_ = NULL; \ + (sl)->sl_dl_neles_ = 0; \ } while (0) -#define tSListPrepend(sl, sln) \ +#define tSListPush(sl, sln) \ do { \ TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ TD_SLIST_HEAD(sl) = (sln); \ - } while (0); + TD_SLIST_NELES(sl) += 1; \ + } while (0) + +#define tSListPop(sl) \ + do { \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ + TD_SLIST_NELES(sl) -= 1; \ + } while (0) // Double linked list #define TD_DLIST_NODE(TYPE) \ struct { \ - TYPE *prev_; \ - TYPE *next_; \ + TYPE *dl_prev_; \ + TYPE *dl_next_; \ } -#define TD_DLIST(TYPE) \ - struct { \ - TYPE *head_; \ - TYPE *tail_; \ - int neles_; \ +#define TD_DLIST(TYPE) \ + struct { \ + struct TYPE *dl_head_; \ + struct TYPE *dl_tail_; \ + int dl_neles_; \ } -#define tDListInit(l) \ - (l)->head_ = (l)->tail_ = NULL; \ - (l)->neles_ = 0; +#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) +#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) +#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) +#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) -#define tlistHead(l) (l)->head_ -#define tlistTail(l) (l)->tail_ -#define tlistNEles(l) (l)->neles_ +#define tDListInit(dl) \ + do { \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \ + TD_DLIST_NELES(dl) = 0; \ + } while (0) -#define tlistAppend(l, n) \ - if ((l)->head_ == NULL) { \ - (n)->prev_ = (n)->next_ = NULL; \ - (l)->head_ = (l)->tail_ = (n); \ - } else { \ - (n)->prev_ = (l)->tail_; \ - (n)->next_ = NULL; \ - (l)->tail_->next_ = (n); \ - (l)->tail_ = (n); \ - } \ - (l)->neles_ += 1; +#define tDListAppend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln); \ + TD_DLIST_TAIL(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) -#define tlistPrepend(l, n) \ - if ((l)->head_ == NULL) { \ - (n)->prev_ = (n)->next_ = NULL; \ - (l)->head_ = (l)->tail_ = (n); \ - } else { \ - (n)->prev_ = NULL; \ - (n)->next_ = (l)->head_; \ - (l)->head_->prev_ = (n); \ - (l)->head_ = (n); \ - } \ - (l)->neles_ += 1; +#define tDListPrepend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = NULL; \ + TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln); \ + TD_DLIST_HEAD(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) -#define tlistPop(l, n) \ - if ((l)->head_ == (n)) { \ - (l)->head_ = (n)->next_; \ - } \ - if ((l)->tail_ == (n)) { \ - (l)->tail_ = (n)->prev_; \ - } \ - if ((n)->prev_ != NULL) { \ - (n)->prev_->next_ = (n)->next_; \ - } \ - if ((n)->next_ != NULL) { \ - (n)->next_->prev_ = (n)->prev_; \ - } \ - (l)->neles_ -= 1; \ - (n)->prev_ = (n)->next_ = NULL; +#define tDListPop(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == (dln)) { \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_TAIL(dl) == (dln)) { \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln); \ + } \ + if (TD_DLIST_NODE_PREV(dln) != NULL) { \ + TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_NODE_NEXT(dln) != NULL) { \ + TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \ + } \ + TD_DLIST_NELES(dl) -= 1; \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + } while (0) +#if 0 // List iterator #define TD_LIST_FITER 0 #define TD_LIST_BITER 1 @@ -118,13 +139,13 @@ extern "C" { TD_DLIST(S) * it_list_; \ } -#define tlistIterInit(it, l, dir) \ - (it)->it_dir_ = (dir); \ - (it)->it_list_ = l; \ - if ((dir) == TD_LIST_FITER) { \ - (it)->it_next_ = (l)->head_; \ - } else { \ - (it)->it_next_ = (l)->tail_; \ +#define tlistIterInit(it, l, dir) \ + (it)->it_dir_ = (dir); \ + (it)->it_list_ = l; \ + if ((dir) == TD_LIST_FITER) { \ + (it)->it_next_ = (l)->dl_head_; \ + } else { \ + (it)->it_next_ = (l)->dl_tail_; \ } #define tlistIterNext(it) \ @@ -139,6 +160,7 @@ extern "C" { } \ (it)->it_ptr_; \ }) +#endif #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index b5424894ca..5d2b404314 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -35,7 +35,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { return NULL; } - tlistAppend(&(pVMA->nlist), pNode); + tDListAppend(&(pVMA->nlist), pNode); return pVMA; } @@ -43,10 +43,10 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { void vmaDestroy(SVMemAllocator *pVMA) { if (pVMA) { while (true) { - SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); if (pNode) { - tlistPop(&(pVMA->nlist), pNode); + tDListPop(&(pVMA->nlist), pNode); vArenaNodeFree(pNode); } else { break; @@ -58,18 +58,18 @@ void vmaDestroy(SVMemAllocator *pVMA) { } void vmaReset(SVMemAllocator *pVMA) { - while (tlistNEles(&(pVMA->nlist)) > 1) { - SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); - tlistPop(&(pVMA->nlist), pNode); + while (TD_DLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); + tDListPop(&(pVMA->nlist), pNode); vArenaNodeFree(pNode); } - SVArenaNode *pNode = tlistHead(&(pVMA->nlist)); + SVArenaNode *pNode = TD_DLIST_HEAD(&(pVMA->nlist)); pNode->ptr = pNode->data; } void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { - SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); void * ptr; if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { @@ -80,7 +80,7 @@ void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { return NULL; } - tlistAppend(&(pVMA->nlist), pNode); + tDListAppend(&(pVMA->nlist), pNode); } ptr = pNode->ptr; @@ -94,9 +94,10 @@ void vmaFree(SVMemAllocator *pVMA, void *ptr) { } bool vmaIsFull(SVMemAllocator *pVMA) { - SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); - return (tlistNEles(&(pVMA->nlist)) > 1) || (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); + return (TD_DLIST_NELES(&(pVMA->nlist)) > 1) || + (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); } /* ------------------------ STATIC METHODS ------------------------ */ diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 084a7d18a7..1db15c3990 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -50,7 +50,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { return -1; } - tlistAppend(&(pVnode->pBufPool->free), pVMA); + tDListAppend(&(pVnode->pBufPool->free), pVMA); } return 0; @@ -61,16 +61,16 @@ void vnodeCloseBufPool(SVnode *pVnode) { vmaDestroy(pVnode->pBufPool->inuse); while (true) { - SVMemAllocator *pVMA = tlistHead(&(pVnode->pBufPool->incycle)); + SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle)); if (pVMA == NULL) break; - tlistPop(&(pVnode->pBufPool->incycle), pVMA); + tDListPop(&(pVnode->pBufPool->incycle), pVMA); vmaDestroy(pVMA); } while (true) { - SVMemAllocator *pVMA = tlistHead(&(pVnode->pBufPool->free)); + SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free)); if (pVMA == NULL) break; - tlistPop(&(pVnode->pBufPool->free), pVMA); + tDListPop(&(pVnode->pBufPool->free), pVMA); vmaDestroy(pVMA); } @@ -85,9 +85,9 @@ void *vnodeMalloc(SVnode *pVnode, uint64_t size) { if (pBufPool->inuse == NULL) { while (true) { // TODO: add sem_wait and sem_post - pBufPool->inuse = tlistHead(&(pBufPool->free)); + pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free)); if (pBufPool->inuse) { - tlistPop(&(pBufPool->free), pBufPool->inuse); + tDListPop(&(pBufPool->free), pBufPool->inuse); break; } } From 4503901f3e42eaab22b257ed2741de5cbb7a055d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 22:21:28 +0800 Subject: [PATCH 16/16] more --- include/util/tdlist.h | 22 +++++------ .../dnode/vnode/impl/inc/vnodeMemAllocator.h | 11 +++--- .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 37 ++++++++----------- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 8360d2713f..a19f3bebec 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -23,23 +23,23 @@ extern "C" { // Single linked list #define TD_SLIST_NODE(TYPE) \ struct { \ - struct type *sl_next_; \ + struct TYPE *sl_next_; \ } -#define TD_SLIST(TYPE) \ - struct { \ - struct TYPE *sl_dl_head_; \ - int sl_dl_neles_; \ +#define TD_SLIST(TYPE) \ + struct { \ + struct TYPE *sl_head_; \ + int sl_neles_; \ } -#define TD_SLIST_HEAD(sl) ((sl)->sl_dl_head_) -#define TD_SLIST_NELES(sl) ((sl)->sl_dl_neles_) +#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) -#define tSListInit(sl) \ - do { \ - (sl)->sl_dl_head_ = NULL; \ - (sl)->sl_dl_neles_ = 0; \ +#define tSListInit(sl) \ + do { \ + (sl)->sl_head_ = NULL; \ + (sl)->sl_neles_ = 0; \ } while (0) #define tSListPush(sl, sln) \ diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index 6b9fddaa02..c8c58e9f69 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -26,7 +26,7 @@ typedef struct SVArenaNode SVArenaNode; typedef struct SVMemAllocator SVMemAllocator; struct SVArenaNode { - TD_DLIST_NODE(SVArenaNode); + TD_SLIST_NODE(SVArenaNode); uint64_t size; // current node size void * ptr; char data[]; @@ -34,10 +34,11 @@ struct SVArenaNode { struct SVMemAllocator { TD_DLIST_NODE(SVMemAllocator); - uint64_t capacity; - uint64_t ssize; - uint64_t lsize; - TD_DLIST(SVArenaNode) nlist; + uint64_t capacity; + uint64_t ssize; + uint64_t lsize; + SVArenaNode *pNode; + TD_SLIST(SVArenaNode) nlist; }; SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize); diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index 5d2b404314..99d4781df9 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -27,30 +27,25 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { pVMA->capacity = capacity; pVMA->ssize = ssize; pVMA->lsize = lsize; - tDListInit(&(pVMA->nlist)); + tSListInit(&(pVMA->nlist)); - SVArenaNode *pNode = vArenaNodeNew(capacity); - if (pNode == NULL) { + pVMA->pNode = vArenaNodeNew(capacity); + if (pVMA->pNode == NULL) { free(pVMA); return NULL; } - tDListAppend(&(pVMA->nlist), pNode); + tSListPush(&(pVMA->nlist), pVMA->pNode); return pVMA; } void vmaDestroy(SVMemAllocator *pVMA) { if (pVMA) { - while (true) { - SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); - - if (pNode) { - tDListPop(&(pVMA->nlist), pNode); - vArenaNodeFree(pNode); - } else { - break; - } + while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + tSListPop(&(pVMA->nlist)); + vArenaNodeFree(pNode); } free(pVMA); @@ -58,18 +53,18 @@ void vmaDestroy(SVMemAllocator *pVMA) { } void vmaReset(SVMemAllocator *pVMA) { - while (TD_DLIST_NELES(&(pVMA->nlist)) > 1) { - SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); - tDListPop(&(pVMA->nlist), pNode); + while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + tSListPop(&(pVMA->nlist)); vArenaNodeFree(pNode); } - SVArenaNode *pNode = TD_DLIST_HEAD(&(pVMA->nlist)); + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); pNode->ptr = pNode->data; } void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { - SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); void * ptr; if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { @@ -80,7 +75,7 @@ void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { return NULL; } - tDListAppend(&(pVMA->nlist), pNode); + tSListPush(&(pVMA->nlist), pNode); } ptr = pNode->ptr; @@ -94,9 +89,9 @@ void vmaFree(SVMemAllocator *pVMA, void *ptr) { } bool vmaIsFull(SVMemAllocator *pVMA) { - SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); - return (TD_DLIST_NELES(&(pVMA->nlist)) > 1) || + return (TD_SLIST_NELES(&(pVMA->nlist)) > 1) || (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); }