diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..fc9a51af65 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp/.devcontainer/base.Dockerfile + +# [Choice] Debian / Ubuntu version (use Debian 11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 +ARG VARIANT="bullseye" +FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT} + +# [Optional] Uncomment this section to install additional packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..9b752d091d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp +{ + "name": "C++", + "build": { + "dockerfile": "Dockerfile", + // Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 + // Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon + "args": { "VARIANT": "ubuntu-21.04" } + }, + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + + // Set *default* container specific settings.json values on container create. + "settings": {}, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "austin.code-gnu-global", + "visualstudioexptteam.vscodeintel" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "gcc -v", + + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} diff --git a/.gitignore b/.gitignore index c651127992..2308ea7896 100644 --- a/.gitignore +++ b/.gitignore @@ -101,4 +101,4 @@ TAGS contrib/* !contrib/CMakeLists.txt -!contrib/test +!contrib/test \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index b626977588..1a8cb1d710 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake") set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib") include(${CMAKE_SUPPORT_DIR}/cmake.options) -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3") # contrib add_subdirectory(contrib) @@ -20,9 +20,13 @@ add_library(api INTERFACE) target_include_directories(api INTERFACE "include/client") # src +if(${BUILD_TEST}) + include(CTest) + enable_testing() +endif(${BUILD_TEST}) add_subdirectory(source) # docs add_subdirectory(docs) -# tests (TODO) +# tests (TODO) \ No newline at end of file diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 30531ad738..007ce83812 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -36,6 +36,8 @@ typedef struct SVnodeCfg { struct { /** write buffer size */ uint64_t wsize; + uint64_t ssize; + uint64_t lsize; /** use heap allocator or arena allocator */ bool isHeapAllocator; }; diff --git a/include/util/tdlist.h b/include/util/tdlist.h new file mode 100644 index 0000000000..a19f3bebec --- /dev/null +++ b/include/util/tdlist.h @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_TDLIST_H_ +#define _TD_UTIL_TDLIST_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// Single linked list +#define TD_SLIST_NODE(TYPE) \ + struct { \ + struct TYPE *sl_next_; \ + } + +#define TD_SLIST(TYPE) \ + struct { \ + struct TYPE *sl_head_; \ + int sl_neles_; \ + } + +#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) +#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) + +#define tSListInit(sl) \ + do { \ + (sl)->sl_head_ = NULL; \ + (sl)->sl_neles_ = 0; \ + } while (0) + +#define tSListPush(sl, sln) \ + do { \ + TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ + TD_SLIST_HEAD(sl) = (sln); \ + TD_SLIST_NELES(sl) += 1; \ + } while (0) + +#define tSListPop(sl) \ + do { \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ + TD_SLIST_NELES(sl) -= 1; \ + } while (0) + +// Double linked list +#define TD_DLIST_NODE(TYPE) \ + struct { \ + TYPE *dl_prev_; \ + TYPE *dl_next_; \ + } + +#define TD_DLIST(TYPE) \ + struct { \ + struct TYPE *dl_head_; \ + struct TYPE *dl_tail_; \ + int dl_neles_; \ + } + +#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) +#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) +#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) +#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) + +#define tDListInit(dl) \ + do { \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \ + TD_DLIST_NELES(dl) = 0; \ + } while (0) + +#define tDListAppend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln); \ + TD_DLIST_TAIL(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + +#define tDListPrepend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = NULL; \ + TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln); \ + TD_DLIST_HEAD(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + +#define tDListPop(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == (dln)) { \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_TAIL(dl) == (dln)) { \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln); \ + } \ + if (TD_DLIST_NODE_PREV(dln) != NULL) { \ + TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_NODE_NEXT(dln) != NULL) { \ + TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \ + } \ + TD_DLIST_NELES(dl) -= 1; \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + } while (0) + +#if 0 +// List iterator +#define TD_LIST_FITER 0 +#define TD_LIST_BITER 1 +#define TD_LIST_ITER(S) \ + struct { \ + int it_dir_; \ + S * it_next_; \ + S * it_ptr_; \ + TD_DLIST(S) * it_list_; \ + } + +#define tlistIterInit(it, l, dir) \ + (it)->it_dir_ = (dir); \ + (it)->it_list_ = l; \ + if ((dir) == TD_LIST_FITER) { \ + (it)->it_next_ = (l)->dl_head_; \ + } else { \ + (it)->it_next_ = (l)->dl_tail_; \ + } + +#define tlistIterNext(it) \ + ({ \ + (it)->it_ptr_ = (it)->it_next_; \ + if ((it)->it_next_ != NULL) { \ + if ((it)->it_dir_ == TD_LIST_FITER) { \ + (it)->it_next_ = (it)->it_next_->next_; \ + } else { \ + (it)->it_next_ = (it)->it_next_->prev_; \ + } \ + } \ + (it)->it_ptr_; \ + }) +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_TDLIST_H_*/ \ No newline at end of file diff --git a/include/util/tmacro.h b/include/util/tmacro.h new file mode 100644 index 0000000000..74056cfe07 --- /dev/null +++ b/include/util/tmacro.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_MACRO_H_ +#define _TD_UTIL_MACRO_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Module init/clear MACRO definitions +#define TD_MOD_UNINITIALIZED 0 +#define TD_MOD_INITIALIZED 1 + +#define TD_MOD_UNCLEARD 0 +#define TD_MOD_CLEARD 1 + +#define TD_DEF_MOD_INIT_FLAG(MOD) static int8_t MOD##InitFlag = TD_MOD_UNINITIALIZED +#define TD_DEF_MOD_CLEAR_FLAG(MOD) static int8_t MOD##ClearFlag = TD_MOD_UNCLEARD + +#define TD_CHECK_AND_SET_MODE_INIT(MOD) \ + atomic_val_compare_exchange_8(&(MOD##InitFlag), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED) + +#define TD_CHECK_AND_SET_MOD_CLEAR(MOD) atomic_val_compare_exchange_8(&(MOD##ClearFlag), TD_MOD_UNCLEARD, TD_MOD_CLEARD) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_MACRO_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index d6d267c4d4..6972605afd 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -18,6 +18,6 @@ target_link_libraries( ) # test -#if(${BUILD_TEST}) -# add_subdirectory(test) -#endif(${BUILD_TEST}) \ No newline at end of file +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/impl/inc/vnodeBufferPool.h index d64dc93847..bfc4de9e12 100644 --- a/source/dnode/vnode/impl/inc/vnodeBufferPool.h +++ b/source/dnode/vnode/impl/inc/vnodeBufferPool.h @@ -28,6 +28,7 @@ typedef struct SVBufPool SVBufPool; int vnodeOpenBufPool(SVnode *pVnode); void vnodeCloseBufPool(SVnode *pVnode); void *vnodeMalloc(SVnode *pVnode, uint64_t size); +bool vnodeBufPoolIsFull(SVnode *pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/inc/vnodeCommit.h b/source/dnode/vnode/impl/inc/vnodeCommit.h index c8ff4947aa..a60e8feac2 100644 --- a/source/dnode/vnode/impl/inc/vnodeCommit.h +++ b/source/dnode/vnode/impl/inc/vnodeCommit.h @@ -22,8 +22,8 @@ extern "C" { #endif -bool vnodeShouldCommit(SVnode *pVnode); -int vnodeAsyncCommit(SVnode *pVnode); +#define vnodeShouldCommit vnodeBufPoolIsFull +int vnodeAsyncCommit(SVnode *pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index c92de433c3..e3a3fac6b9 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -18,15 +18,17 @@ #include "mallocator.h" #include "sync.h" +#include "tcoding.h" +#include "tdlist.h" #include "tlockfree.h" #include "wal.h" -#include "tcoding.h" #include "vnode.h" #include "vnodeBufferPool.h" #include "vnodeCfg.h" #include "vnodeCommit.h" #include "vnodeFS.h" +#include "vnodeMemAllocator.h" #include "vnodeRequest.h" #include "vnodeStateMgr.h" #include "vnodeSync.h" diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index 9184eb416b..c8c58e9f69 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -16,15 +16,37 @@ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_ -#include "mallocator.h" -#include "vnode.h" +#include "os.h" #ifdef __cplusplus extern "C" { #endif -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode); -void vnodeDestroyMemAllocator(SMemAllocator *pma); +typedef struct SVArenaNode SVArenaNode; +typedef struct SVMemAllocator SVMemAllocator; + +struct SVArenaNode { + TD_SLIST_NODE(SVArenaNode); + uint64_t size; // current node size + void * ptr; + char data[]; +}; + +struct SVMemAllocator { + TD_DLIST_NODE(SVMemAllocator); + uint64_t capacity; + uint64_t ssize; + uint64_t lsize; + SVArenaNode *pNode; + TD_SLIST(SVArenaNode) nlist; +}; + +SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize); +void vmaDestroy(SVMemAllocator *pVMA); +void vmaReset(SVMemAllocator *pVMA); +void * vmaMalloc(SVMemAllocator *pVMA, uint64_t size); +void vmaFree(SVMemAllocator *pVMA, void *ptr); +bool vmaIsFull(SVMemAllocator *pVMA); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c new file mode 100644 index 0000000000..99d4781df9 --- /dev/null +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeDef.h" + +static SVArenaNode *vArenaNodeNew(uint64_t capacity); +static void vArenaNodeFree(SVArenaNode *pNode); + +SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { + SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA)); + if (pVMA == NULL) { + return NULL; + } + + pVMA->capacity = capacity; + pVMA->ssize = ssize; + pVMA->lsize = lsize; + tSListInit(&(pVMA->nlist)); + + pVMA->pNode = vArenaNodeNew(capacity); + if (pVMA->pNode == NULL) { + free(pVMA); + return NULL; + } + + tSListPush(&(pVMA->nlist), pVMA->pNode); + + return pVMA; +} + +void vmaDestroy(SVMemAllocator *pVMA) { + if (pVMA) { + while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + tSListPop(&(pVMA->nlist)); + vArenaNodeFree(pNode); + } + + free(pVMA); + } +} + +void vmaReset(SVMemAllocator *pVMA) { + while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + tSListPop(&(pVMA->nlist)); + vArenaNodeFree(pNode); + } + + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + pNode->ptr = pNode->data; +} + +void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + void * ptr; + + if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { + uint64_t capacity = MAX(pVMA->ssize, size); + pNode = vArenaNodeNew(capacity); + if (pNode == NULL) { + // TODO: handle error + return NULL; + } + + tSListPush(&(pVMA->nlist), pNode); + } + + ptr = pNode->ptr; + pNode->ptr = POINTER_SHIFT(ptr, size); + + return ptr; +} + +void vmaFree(SVMemAllocator *pVMA, void *ptr) { + // TODO +} + +bool vmaIsFull(SVMemAllocator *pVMA) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + + return (TD_SLIST_NELES(&(pVMA->nlist)) > 1) || + (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); +} + +/* ------------------------ STATIC METHODS ------------------------ */ +static SVArenaNode *vArenaNodeNew(uint64_t capacity) { + SVArenaNode *pNode = NULL; + + pNode = (SVArenaNode *)malloc(sizeof(*pNode) + capacity); + if (pNode == NULL) { + return NULL; + } + + pNode->size = capacity; + pNode->ptr = pNode->data; + + return pNode; +} + +static void vArenaNodeFree(SVArenaNode *pNode) { + if (pNode) { + free(pNode); + } +} \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 00203ed9b6..1db15c3990 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -19,14 +19,90 @@ #define VNODE_BUF_POOL_SHARDS 3 struct SVBufPool { - // buffer pool impl - SList free; - SList incycle; - SListNode *inuse; + TD_DLIST(SVMemAllocator) free; + TD_DLIST(SVMemAllocator) incycle; + SVMemAllocator *inuse; // MAF for submodules - SMemAllocatorFactory maf; + // SMemAllocatorFactory maf; }; +int vnodeOpenBufPool(SVnode *pVnode) { + uint64_t capacity; + // EVMemAllocatorT type = E_V_ARENA_ALLOCATOR; + + if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { + /* TODO */ + return -1; + } + + tDListInit(&(pVnode->pBufPool->free)); + tDListInit(&(pVnode->pBufPool->incycle)); + + pVnode->pBufPool->inuse = NULL; + + // TODO + capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS; + + for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) { + SVMemAllocator *pVMA = vmaCreate(capacity, pVnode->config.ssize, pVnode->config.lsize); + if (pVMA == NULL) { + // TODO: handle error + return -1; + } + + tDListAppend(&(pVnode->pBufPool->free), pVMA); + } + + return 0; +} + +void vnodeCloseBufPool(SVnode *pVnode) { + if (pVnode->pBufPool) { + vmaDestroy(pVnode->pBufPool->inuse); + + while (true) { + SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle)); + if (pVMA == NULL) break; + tDListPop(&(pVnode->pBufPool->incycle), pVMA); + vmaDestroy(pVMA); + } + + while (true) { + SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free)); + if (pVMA == NULL) break; + tDListPop(&(pVnode->pBufPool->free), pVMA); + vmaDestroy(pVMA); + } + + free(pVnode->pBufPool); + pVnode->pBufPool = NULL; + } +} + +void *vnodeMalloc(SVnode *pVnode, uint64_t size) { + SVBufPool *pBufPool = pVnode->pBufPool; + + if (pBufPool->inuse == NULL) { + while (true) { + // TODO: add sem_wait and sem_post + pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free)); + if (pBufPool->inuse) { + tDListPop(&(pBufPool->free), pBufPool->inuse); + break; + } + } + } + + return vmaMalloc(pBufPool->inuse, size); +} + +bool vnodeBufPoolIsFull(SVnode *pVnode) { + if (pVnode->pBufPool->inuse == NULL) return false; + return vmaIsFull(pVnode->pBufPool->inuse); +} + +#if 0 + typedef enum { // Heap allocator E_V_HEAP_ALLOCATOR = 0, @@ -57,15 +133,6 @@ typedef struct { SListNode *pNode; } SVMAWrapper; -typedef struct { - T_REF_DECLARE() - uint64_t capacity; - EVMemAllocatorT type; - union { - SVHeapAllocator vha; - SVArenaAllocator vaa; - }; -} SVMemAllocator; static SListNode * vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); static void vBufPoolFreeNode(SListNode *pNode); @@ -73,106 +140,13 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf); static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); -int vnodeOpenBufPool(SVnode *pVnode) { - uint64_t capacity; - EVMemAllocatorT type = E_V_ARENA_ALLOCATOR; - - if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { - /* TODO */ - return -1; - } - - tdListInit(&(pVnode->pBufPool->free), 0); - tdListInit(&(pVnode->pBufPool->incycle), 0); - - capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS; - if (pVnode->config.isHeapAllocator) { - type = E_V_HEAP_ALLOCATOR; - } - - for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) { - SListNode *pNode = vBufPoolNewNode(capacity, type); - if (pNode == NULL) { - vnodeCloseBufPool(pVnode); - return -1; - } - - tdListAppendNode(&(pVnode->pBufPool->free), pNode); - } - - pVnode->pBufPool->maf.impl = pVnode; - pVnode->pBufPool->maf.create = vBufPoolCreateMA; - pVnode->pBufPool->maf.destroy = vBufPoolDestroyMA; - - return 0; -} - -void vnodeCloseBufPool(SVnode *pVnode) { - SListNode *pNode; - if (pVnode->pBufPool) { - // Clear free list - while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) != NULL) { - vBufPoolFreeNode(pNode); - } - - // Clear incycle list - while ((pNode = tdListPopHead(&(pVnode->pBufPool->incycle))) != NULL) { - vBufPoolFreeNode(pNode); - } - - // Free inuse node - if (pVnode->pBufPool->inuse) { - vBufPoolFreeNode(pVnode->pBufPool->inuse); - } - - free(pVnode->pBufPool); - pVnode->pBufPool = NULL; - } -} - -void *vnodeMalloc(SVnode *pVnode, uint64_t size) { - void *ptr; - - if (pVnode->pBufPool->inuse == NULL) { - SListNode *pNode; - while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) == NULL) { - // todo - // tsem_wait(); - ASSERT(0); - } - - pVnode->pBufPool->inuse = pNode; - } - - SVMemAllocator *pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - return vBufPoolMalloc(pvma, size); -} - /* ------------------------ STATIC METHODS ------------------------ */ -static void vArenaAllocatorInit(SVArenaAllocator *pvaa, uint64_t capacity, uint64_t ssize, uint64_t lsize) { /* TODO */ - pvaa->ssize = ssize; - pvaa->lsize = lsize; - pvaa->inuse = &pvaa->node; - - pvaa->node.prev = NULL; - pvaa->node.size = capacity; - pvaa->node.ptr = pvaa->node.data; -} - -static void vArenaAllocatorClear(SVArenaAllocator *pvaa) { /* TODO */ - while (pvaa->inuse != &(pvaa->node)) { - SVArenaNode *pANode = pvaa->inuse; - pvaa->inuse = pANode->prev; - free(pANode); - } -} - static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { SListNode * pNode; SVMemAllocator *pvma; uint64_t msize; - uint64_t ssize = 0; // TODO - uint64_t lsize = 0; // TODO + uint64_t ssize = 4096; // TODO + uint64_t lsize = 1024; // TODO msize = sizeof(SListNode) + sizeof(SVMemAllocator); if (type == E_V_ARENA_ALLOCATOR) { @@ -317,4 +291,5 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma) { tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode)); // tsem_post(); todo: sem_post } -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeCfg.c b/source/dnode/vnode/impl/src/vnodeCfg.c index 01facba888..7aac9ca6a7 100644 --- a/source/dnode/vnode/impl/src/vnodeCfg.c +++ b/source/dnode/vnode/impl/src/vnodeCfg.c @@ -15,7 +15,7 @@ #include "vnodeDef.h" -const SVnodeCfg defaultVnodeOptions = {0}; /* TODO */ +const SVnodeCfg defaultVnodeOptions = {.wsize = 16*1024*1024, .walCfg = {.walLevel = TAOS_WAL_WRITE}}; /* TODO */ void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */ vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions); diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/impl/src/vnodeCommit.c index 18a0c6d91d..cac7999f59 100644 --- a/source/dnode/vnode/impl/src/vnodeCommit.c +++ b/source/dnode/vnode/impl/src/vnodeCommit.c @@ -18,8 +18,6 @@ static int vnodeStartCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode); -bool vnodeShouldCommit(SVnode *pVnode) { return false; } - int vnodeAsyncCommit(SVnode *pVnode) { #if 0 if (vnodeStartCommit(pVnode) < 0) { diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index ab33b58858..9b94b4a361 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tmacro.h" #include "vnodeDef.h" static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg); @@ -20,8 +21,14 @@ static void vnodeFree(SVnode *pVnode); static int vnodeOpenImpl(SVnode *pVnode); static void vnodeCloseImpl(SVnode *pVnode); +TD_DEF_MOD_INIT_FLAG(vnode); +TD_DEF_MOD_CLEAR_FLAG(vnode); + int vnodeInit() { - // TODO + if (TD_CHECK_AND_SET_MODE_INIT(vnode) == TD_MOD_INITIALIZED) { + return 0; + } + if (walInit() < 0) { return -1; } @@ -30,6 +37,10 @@ int vnodeInit() { } void vnodeClear() { + if (TD_CHECK_AND_SET_MOD_CLEAR(vnode) == TD_MOD_CLEARD) { + return; + } + walCleanUp(); } diff --git a/source/dnode/vnode/impl/src/vnodeMemAllocator.c b/source/dnode/vnode/impl/src/vnodeMemAllocator.c deleted file mode 100644 index 902014eb47..0000000000 --- a/source/dnode/vnode/impl/src/vnodeMemAllocator.c +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" - -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode) { - SMemAllocator *pma = NULL; - /* TODO */ - return pma; -} - -void vnodeDestroyMemAllocator(SMemAllocator *pma) { - // TODO -} - -#if 0 -#define VNODE_HEAP_ALLOCATOR 0 -#define VNODE_ARENA_ALLOCATOR 1 - -typedef struct { - uint64_t tsize; - uint64_t used; -} SVHeapAllocator; - -typedef struct SVArenaNode { - struct SVArenaNode *prev; - void * nptr; - char data[]; -} SVArenaNode; - -typedef struct { - SVArenaNode *inuse; - SVArenaNode node; -} SVArenaAllocator; - -typedef struct { - int8_t type; - uint64_t tsize; - T_REF_DECLARE() - union { - SVHeapAllocator vha; - SVArenaAllocator vaa; - }; -} SVMemAllocator; - -SMemAllocator *vnodeCreateMemAllocator(int8_t type, uint64_t tsize, uint64_t ssize /* step size only for arena */) { - SMemAllocator * pma; - uint64_t msize; - SVMemAllocator *pva; - - msize = sizeof(*pma) + sizeof(SVMemAllocator); - if (type == VNODE_ARENA_ALLOCATOR) { - msize += tsize; - } - - pma = (SMemAllocator *)calloc(1, msize); - if (pma == NULL) { - return NULL; - } - - pma->impl = POINTER_SHIFT(pma, sizeof(*pma)); - pva = (SVMemAllocator *)(pma->impl); - pva->type = type; - pva->tsize = tsize; - - if (type == VNODE_HEAP_ALLOCATOR) { - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } else if (type == VNODE_ARENA_ALLOCATOR) { - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } else { - ASSERT(0); - } - - return pma; -} - -void vnodeDestroyMemAllocator(SMemAllocator *pma) { - // TODO -} - -void vnodeRefMemAllocator(SMemAllocator *pma) { - // TODO -} - -void vnodeUnrefMemAllocator(SMemAllocator *pma) { - // TODO -} - -/* ------------------------ Heap Allocator IMPL ------------------------ */ - -/* ------------------------ Arena Allocator IMPL ------------------------ */ - -#endif \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/CMakeLists.txt b/source/dnode/vnode/impl/test/CMakeLists.txt index 83506a4fde..e1226331e9 100644 --- a/source/dnode/vnode/impl/test/CMakeLists.txt +++ b/source/dnode/vnode/impl/test/CMakeLists.txt @@ -4,4 +4,9 @@ target_sources(vnodeApiTests PRIVATE "vnodeApiTests.cpp" ) -target_link_libraries(vnodeApiTests vnode gtest gtest_main) \ No newline at end of file +target_link_libraries(vnodeApiTests vnode gtest gtest_main) + +add_test( + NAME vnode_api_tests + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests + ) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index df784181b7..ac2ccbc132 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -1,3 +1,14 @@ +/** + * @file vnodeApiTests.cpp + * @author hzcheng (hzcheng@taosdata.com) + * @brief VNODE module API tests + * @version 0.1 + * @date 2021-12-13 + * + * @copyright Copyright (c) 2021 + * + */ + #include #include