xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/vnode' into 3.0

This commit is contained in:
Hongze Cheng 2021-12-13 22:50:18 +08:00
parent 0d596ea37d 4503901f3e
commit 7fe040235d
20 changed files with 530 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.devcontainer/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp/.devcontainer/base.Dockerfile
# [Choice] Debian / Ubuntu version (use Debian 11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04
ARG VARIANT="bullseye"
FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT}
# [Optional] Uncomment this section to install additional packages.
# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
# && apt-get -y install --no-install-recommends <your-package-list-here>

32
.devcontainer/devcontainer.json Normal file
View File

@ -0,0 +1,32 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp
{
"name": "C++",
"build": {
"dockerfile": "Dockerfile",
// Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04
// Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon
"args": { "VARIANT": "ubuntu-21.04" }
},
"runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
// Set *default* container specific settings.json values on container create.
"settings": {},
// Add the IDs of extensions you want installed when the container is created.
"extensions": [
"ms-vscode.cpptools",
"ms-vscode.cmake-tools",
"austin.code-gnu-global",
"visualstudioexptteam.vscodeintel"
],
// Use 'forwardPorts' to make a list of ports inside the container available locally.
// "forwardPorts": [],
// Use 'postCreateCommand' to run commands after the container is created.
// "postCreateCommand": "gcc -v",
// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
"remoteUser": "vscode"
}

2
.gitignore vendored
View File

@ -101,4 +101,4 @@ TAGS
contrib/* contrib/*
!contrib/CMakeLists.txt !contrib/CMakeLists.txt
!contrib/test !contrib/test

8
CMakeLists.txt
View File

@ -10,7 +10,7 @@ set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake")
set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib") set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib")
include(${CMAKE_SUPPORT_DIR}/cmake.options) include(${CMAKE_SUPPORT_DIR}/cmake.options)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3")
# contrib # contrib
add_subdirectory(contrib) add_subdirectory(contrib)
@ -20,9 +20,13 @@ add_library(api INTERFACE)
target_include_directories(api INTERFACE "include/client") target_include_directories(api INTERFACE "include/client")
# src # src
if(${BUILD_TEST})
include(CTest)
enable_testing()
endif(${BUILD_TEST})
add_subdirectory(source) add_subdirectory(source)
# docs # docs
add_subdirectory(docs) add_subdirectory(docs)
# tests (TODO) # tests (TODO)

2
include/dnode/vnode/vnode.h
View File

@ -36,6 +36,8 @@ typedef struct SVnodeCfg {
struct { struct {
/** write buffer size */ /** write buffer size */
uint64_t wsize; uint64_t wsize;
uint64_t ssize;
uint64_t lsize;
/** use heap allocator or arena allocator */ /** use heap allocator or arena allocator */
bool isHeapAllocator; bool isHeapAllocator;
}; };

169
include/util/tdlist.h Normal file
View File

@ -0,0 +1,169 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_TDLIST_H_
#define _TD_UTIL_TDLIST_H_
#ifdef __cplusplus
extern "C" {
#endif
// Single linked list
#define TD_SLIST_NODE(TYPE) \
struct { \
struct TYPE *sl_next_; \
}
#define TD_SLIST(TYPE) \
struct { \
struct TYPE *sl_head_; \
int sl_neles_; \
}
#define TD_SLIST_HEAD(sl) ((sl)->sl_head_)
#define TD_SLIST_NELES(sl) ((sl)->sl_neles_)
#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_)
#define tSListInit(sl) \
do { \
(sl)->sl_head_ = NULL; \
(sl)->sl_neles_ = 0; \
} while (0)
#define tSListPush(sl, sln) \
do { \
TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \
TD_SLIST_HEAD(sl) = (sln); \
TD_SLIST_NELES(sl) += 1; \
} while (0)
#define tSListPop(sl) \
do { \
TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \
TD_SLIST_NELES(sl) -= 1; \
} while (0)
// Double linked list
#define TD_DLIST_NODE(TYPE) \
struct { \
TYPE *dl_prev_; \
TYPE *dl_next_; \
}
#define TD_DLIST(TYPE) \
struct { \
struct TYPE *dl_head_; \
struct TYPE *dl_tail_; \
int dl_neles_; \
}
#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_)
#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_)
#define TD_DLIST_HEAD(dl) ((dl)->dl_head_)
#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_)
#define TD_DLIST_NELES(dl) ((dl)->dl_neles_)
#define tDListInit(dl) \
do { \
TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \
TD_DLIST_NELES(dl) = 0; \
} while (0)
#define tDListAppend(dl, dln) \
do { \
if (TD_DLIST_HEAD(dl) == NULL) { \
TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \
TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \
} else { \
TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl); \
TD_DLIST_NODE_NEXT(dln) = NULL; \
TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln); \
TD_DLIST_TAIL(dl) = (dln); \
} \
TD_DLIST_NELES(dl) += 1; \
} while (0)
#define tDListPrepend(dl, dln) \
do { \
if (TD_DLIST_HEAD(dl) == NULL) { \
TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \
TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \
} else { \
TD_DLIST_NODE_PREV(dln) = NULL; \
TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl); \
TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln); \
TD_DLIST_HEAD(dl) = (dln); \
} \
TD_DLIST_NELES(dl) += 1; \
} while (0)
#define tDListPop(dl, dln) \
do { \
if (TD_DLIST_HEAD(dl) == (dln)) { \
TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln); \
} \
if (TD_DLIST_TAIL(dl) == (dln)) { \
TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln); \
} \
if (TD_DLIST_NODE_PREV(dln) != NULL) { \
TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \
} \
if (TD_DLIST_NODE_NEXT(dln) != NULL) { \
TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \
} \
TD_DLIST_NELES(dl) -= 1; \
TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \
} while (0)
#if 0
// List iterator
#define TD_LIST_FITER 0
#define TD_LIST_BITER 1
#define TD_LIST_ITER(S) \
struct { \
int it_dir_; \
S * it_next_; \
S * it_ptr_; \
TD_DLIST(S) * it_list_; \
}
#define tlistIterInit(it, l, dir) \
(it)->it_dir_ = (dir); \
(it)->it_list_ = l; \
if ((dir) == TD_LIST_FITER) { \
(it)->it_next_ = (l)->dl_head_; \
} else { \
(it)->it_next_ = (l)->dl_tail_; \
}
#define tlistIterNext(it) \
({ \
(it)->it_ptr_ = (it)->it_next_; \
if ((it)->it_next_ != NULL) { \
if ((it)->it_dir_ == TD_LIST_FITER) { \
(it)->it_next_ = (it)->it_next_->next_; \
} else { \
(it)->it_next_ = (it)->it_next_->prev_; \
} \
} \
(it)->it_ptr_; \
})
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_TDLIST_H_*/

44
include/util/tmacro.h Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_MACRO_H_
#define _TD_UTIL_MACRO_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
// Module init/clear MACRO definitions
#define TD_MOD_UNINITIALIZED 0
#define TD_MOD_INITIALIZED 1
#define TD_MOD_UNCLEARD 0
#define TD_MOD_CLEARD 1
#define TD_DEF_MOD_INIT_FLAG(MOD) static int8_t MOD##InitFlag = TD_MOD_UNINITIALIZED
#define TD_DEF_MOD_CLEAR_FLAG(MOD) static int8_t MOD##ClearFlag = TD_MOD_UNCLEARD
#define TD_CHECK_AND_SET_MODE_INIT(MOD) \
atomic_val_compare_exchange_8(&(MOD##InitFlag), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED)
#define TD_CHECK_AND_SET_MOD_CLEAR(MOD) atomic_val_compare_exchange_8(&(MOD##ClearFlag), TD_MOD_UNCLEARD, TD_MOD_CLEARD)
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_MACRO_H_*/

6
source/dnode/vnode/impl/CMakeLists.txt
View File

@ -18,6 +18,6 @@ target_link_libraries(
) )
# test # test
#if(${BUILD_TEST}) if(${BUILD_TEST})
# add_subdirectory(test) add_subdirectory(test)
#endif(${BUILD_TEST}) endif(${BUILD_TEST})

1
source/dnode/vnode/impl/inc/vnodeBufferPool.h
View File

@ -28,6 +28,7 @@ typedef struct SVBufPool SVBufPool;
int vnodeOpenBufPool(SVnode *pVnode); int vnodeOpenBufPool(SVnode *pVnode);
void vnodeCloseBufPool(SVnode *pVnode); void vnodeCloseBufPool(SVnode *pVnode);
void *vnodeMalloc(SVnode *pVnode, uint64_t size); void *vnodeMalloc(SVnode *pVnode, uint64_t size);
bool vnodeBufPoolIsFull(SVnode *pVnode);
#ifdef __cplusplus #ifdef __cplusplus
} }

4
source/dnode/vnode/impl/inc/vnodeCommit.h
View File

@ -22,8 +22,8 @@
extern "C" { extern "C" {
#endif #endif
bool vnodeShouldCommit(SVnode *pVnode); #define vnodeShouldCommit vnodeBufPoolIsFull
int vnodeAsyncCommit(SVnode *pVnode); int vnodeAsyncCommit(SVnode *pVnode);
#ifdef __cplusplus #ifdef __cplusplus
} }

4
source/dnode/vnode/impl/inc/vnodeDef.h
View File

@ -18,15 +18,17 @@
#include "mallocator.h" #include "mallocator.h"
#include "sync.h" #include "sync.h"
#include "tcoding.h"
#include "tdlist.h"
#include "tlockfree.h" #include "tlockfree.h"
#include "wal.h" #include "wal.h"
#include "tcoding.h"
#include "vnode.h" #include "vnode.h"
#include "vnodeBufferPool.h" #include "vnodeBufferPool.h"
#include "vnodeCfg.h" #include "vnodeCfg.h"
#include "vnodeCommit.h" #include "vnodeCommit.h"
#include "vnodeFS.h" #include "vnodeFS.h"
#include "vnodeMemAllocator.h"
#include "vnodeRequest.h" #include "vnodeRequest.h"
#include "vnodeStateMgr.h" #include "vnodeStateMgr.h"
#include "vnodeSync.h" #include "vnodeSync.h"

30
source/dnode/vnode/impl/inc/vnodeMemAllocator.h
View File

@ -16,15 +16,37 @@
#ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_
#define _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_
#include "mallocator.h" #include "os.h"
#include "vnode.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode); typedef struct SVArenaNode SVArenaNode;
void vnodeDestroyMemAllocator(SMemAllocator *pma); typedef struct SVMemAllocator SVMemAllocator;
struct SVArenaNode {
TD_SLIST_NODE(SVArenaNode);
uint64_t size; // current node size
void * ptr;
char data[];
};
struct SVMemAllocator {
TD_DLIST_NODE(SVMemAllocator);
uint64_t capacity;
uint64_t ssize;
uint64_t lsize;
SVArenaNode *pNode;
TD_SLIST(SVArenaNode) nlist;
};
SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize);
void vmaDestroy(SVMemAllocator *pVMA);
void vmaReset(SVMemAllocator *pVMA);
void * vmaMalloc(SVMemAllocator *pVMA, uint64_t size);
void vmaFree(SVMemAllocator *pVMA, void *ptr);
bool vmaIsFull(SVMemAllocator *pVMA);
#ifdef __cplusplus #ifdef __cplusplus
} }

117
source/dnode/vnode/impl/src/vnodeArenaMAImpl.c Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeDef.h"
static SVArenaNode *vArenaNodeNew(uint64_t capacity);
static void vArenaNodeFree(SVArenaNode *pNode);
SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) {
SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA));
if (pVMA == NULL) {
return NULL;
}
pVMA->capacity = capacity;
pVMA->ssize = ssize;
pVMA->lsize = lsize;
tSListInit(&(pVMA->nlist));
pVMA->pNode = vArenaNodeNew(capacity);
if (pVMA->pNode == NULL) {
free(pVMA);
return NULL;
}
tSListPush(&(pVMA->nlist), pVMA->pNode);
return pVMA;
}
void vmaDestroy(SVMemAllocator *pVMA) {
if (pVMA) {
while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) {
SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
tSListPop(&(pVMA->nlist));
vArenaNodeFree(pNode);
}
free(pVMA);
}
}
void vmaReset(SVMemAllocator *pVMA) {
while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) {
SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
tSListPop(&(pVMA->nlist));
vArenaNodeFree(pNode);
}
SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
pNode->ptr = pNode->data;
}
void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) {
SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
void * ptr;
if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) {
uint64_t capacity = MAX(pVMA->ssize, size);
pNode = vArenaNodeNew(capacity);
if (pNode == NULL) {
// TODO: handle error
return NULL;
}
tSListPush(&(pVMA->nlist), pNode);
}
ptr = pNode->ptr;
pNode->ptr = POINTER_SHIFT(ptr, size);
return ptr;
}
void vmaFree(SVMemAllocator *pVMA, void *ptr) {
// TODO
}
bool vmaIsFull(SVMemAllocator *pVMA) {
SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist));
return (TD_SLIST_NELES(&(pVMA->nlist)) > 1) ||
(pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize);
}
/* ------------------------ STATIC METHODS ------------------------ */
static SVArenaNode *vArenaNodeNew(uint64_t capacity) {
SVArenaNode *pNode = NULL;
pNode = (SVArenaNode *)malloc(sizeof(*pNode) + capacity);
if (pNode == NULL) {
return NULL;
}
pNode->size = capacity;
pNode->ptr = pNode->data;
return pNode;
}
static void vArenaNodeFree(SVArenaNode *pNode) {
if (pNode) {
free(pNode);
}
}

195
source/dnode/vnode/impl/src/vnodeBufferPool.c
View File

@ -19,14 +19,90 @@
#define VNODE_BUF_POOL_SHARDS 3 #define VNODE_BUF_POOL_SHARDS 3
struct SVBufPool { struct SVBufPool {
// buffer pool impl TD_DLIST(SVMemAllocator) free;
SList free; TD_DLIST(SVMemAllocator) incycle;
SList incycle; SVMemAllocator *inuse;
SListNode *inuse;
// MAF for submodules // MAF for submodules
SMemAllocatorFactory maf; // SMemAllocatorFactory maf;
}; };
int vnodeOpenBufPool(SVnode *pVnode) {
uint64_t capacity;
// EVMemAllocatorT type = E_V_ARENA_ALLOCATOR;
if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) {
/* TODO */
return -1;
}
tDListInit(&(pVnode->pBufPool->free));
tDListInit(&(pVnode->pBufPool->incycle));
pVnode->pBufPool->inuse = NULL;
// TODO
capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS;
for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) {
SVMemAllocator *pVMA = vmaCreate(capacity, pVnode->config.ssize, pVnode->config.lsize);
if (pVMA == NULL) {
// TODO: handle error
return -1;
}
tDListAppend(&(pVnode->pBufPool->free), pVMA);
}
return 0;
}
void vnodeCloseBufPool(SVnode *pVnode) {
if (pVnode->pBufPool) {
vmaDestroy(pVnode->pBufPool->inuse);
while (true) {
SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle));
if (pVMA == NULL) break;
tDListPop(&(pVnode->pBufPool->incycle), pVMA);
vmaDestroy(pVMA);
}
while (true) {
SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free));
if (pVMA == NULL) break;
tDListPop(&(pVnode->pBufPool->free), pVMA);
vmaDestroy(pVMA);
}
free(pVnode->pBufPool);
pVnode->pBufPool = NULL;
}
}
void *vnodeMalloc(SVnode *pVnode, uint64_t size) {
SVBufPool *pBufPool = pVnode->pBufPool;
if (pBufPool->inuse == NULL) {
while (true) {
// TODO: add sem_wait and sem_post
pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free));
if (pBufPool->inuse) {
tDListPop(&(pBufPool->free), pBufPool->inuse);
break;
}
}
}
return vmaMalloc(pBufPool->inuse, size);
}
bool vnodeBufPoolIsFull(SVnode *pVnode) {
if (pVnode->pBufPool->inuse == NULL) return false;
return vmaIsFull(pVnode->pBufPool->inuse);
}
#if 0
typedef enum { typedef enum {
// Heap allocator // Heap allocator
E_V_HEAP_ALLOCATOR = 0, E_V_HEAP_ALLOCATOR = 0,
@ -57,15 +133,6 @@ typedef struct {
SListNode *pNode; SListNode *pNode;
} SVMAWrapper; } SVMAWrapper;
typedef struct {
T_REF_DECLARE()
uint64_t capacity;
EVMemAllocatorT type;
union {
SVHeapAllocator vha;
SVArenaAllocator vaa;
};
} SVMemAllocator;
static SListNode * vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); static SListNode * vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type);
static void vBufPoolFreeNode(SListNode *pNode); static void vBufPoolFreeNode(SListNode *pNode);
@ -73,106 +140,13 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf);
static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma);
static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size);
int vnodeOpenBufPool(SVnode *pVnode) {
uint64_t capacity;
EVMemAllocatorT type = E_V_ARENA_ALLOCATOR;
if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) {
/* TODO */
return -1;
}
tdListInit(&(pVnode->pBufPool->free), 0);
tdListInit(&(pVnode->pBufPool->incycle), 0);
capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS;
if (pVnode->config.isHeapAllocator) {
type = E_V_HEAP_ALLOCATOR;
}
for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) {
SListNode *pNode = vBufPoolNewNode(capacity, type);
if (pNode == NULL) {
vnodeCloseBufPool(pVnode);
return -1;
}
tdListAppendNode(&(pVnode->pBufPool->free), pNode);
}
pVnode->pBufPool->maf.impl = pVnode;
pVnode->pBufPool->maf.create = vBufPoolCreateMA;
pVnode->pBufPool->maf.destroy = vBufPoolDestroyMA;
return 0;
}
void vnodeCloseBufPool(SVnode *pVnode) {
SListNode *pNode;
if (pVnode->pBufPool) {
// Clear free list
while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) != NULL) {
vBufPoolFreeNode(pNode);
}
// Clear incycle list
while ((pNode = tdListPopHead(&(pVnode->pBufPool->incycle))) != NULL) {
vBufPoolFreeNode(pNode);
}
// Free inuse node
if (pVnode->pBufPool->inuse) {
vBufPoolFreeNode(pVnode->pBufPool->inuse);
}
free(pVnode->pBufPool);
pVnode->pBufPool = NULL;
}
}
void *vnodeMalloc(SVnode *pVnode, uint64_t size) {
void *ptr;
if (pVnode->pBufPool->inuse == NULL) {
SListNode *pNode;
while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) == NULL) {
// todo
// tsem_wait();
ASSERT(0);
}
pVnode->pBufPool->inuse = pNode;
}
SVMemAllocator *pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data);
return vBufPoolMalloc(pvma, size);
}
/* ------------------------ STATIC METHODS ------------------------ */ /* ------------------------ STATIC METHODS ------------------------ */
static void vArenaAllocatorInit(SVArenaAllocator *pvaa, uint64_t capacity, uint64_t ssize, uint64_t lsize) { /* TODO */
pvaa->ssize = ssize;
pvaa->lsize = lsize;
pvaa->inuse = &pvaa->node;
pvaa->node.prev = NULL;
pvaa->node.size = capacity;
pvaa->node.ptr = pvaa->node.data;
}
static void vArenaAllocatorClear(SVArenaAllocator *pvaa) { /* TODO */
while (pvaa->inuse != &(pvaa->node)) {
SVArenaNode *pANode = pvaa->inuse;
pvaa->inuse = pANode->prev;
free(pANode);
}
}
static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) {
SListNode * pNode; SListNode * pNode;
SVMemAllocator *pvma; SVMemAllocator *pvma;
uint64_t msize; uint64_t msize;
uint64_t ssize = 0; // TODO uint64_t ssize = 4096; // TODO
uint64_t lsize = 0; // TODO uint64_t lsize = 1024; // TODO
msize = sizeof(SListNode) + sizeof(SVMemAllocator); msize = sizeof(SListNode) + sizeof(SVMemAllocator);
if (type == E_V_ARENA_ALLOCATOR) { if (type == E_V_ARENA_ALLOCATOR) {
@ -317,4 +291,5 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma) {
tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode)); tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode));
// tsem_post(); todo: sem_post // tsem_post(); todo: sem_post
} }
} }
#endif

2
source/dnode/vnode/impl/src/vnodeCfg.c
View File

@ -15,7 +15,7 @@
#include "vnodeDef.h" #include "vnodeDef.h"
const SVnodeCfg defaultVnodeOptions = {0}; /* TODO */ const SVnodeCfg defaultVnodeOptions = {.wsize = 16*1024*1024, .walCfg = {.walLevel = TAOS_WAL_WRITE}}; /* TODO */
void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */ void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */
vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions); vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions);

2
source/dnode/vnode/impl/src/vnodeCommit.c
View File

@ -18,8 +18,6 @@
static int vnodeStartCommit(SVnode *pVnode); static int vnodeStartCommit(SVnode *pVnode);
static int vnodeEndCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode);
bool vnodeShouldCommit(SVnode *pVnode) { return false; }
int vnodeAsyncCommit(SVnode *pVnode) { int vnodeAsyncCommit(SVnode *pVnode) {
#if 0 #if 0
if (vnodeStartCommit(pVnode) < 0) { if (vnodeStartCommit(pVnode) < 0) {

13
source/dnode/vnode/impl/src/vnodeMain.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tmacro.h"
#include "vnodeDef.h" #include "vnodeDef.h"
static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg); static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg);
@ -20,8 +21,14 @@ static void vnodeFree(SVnode *pVnode);
static int vnodeOpenImpl(SVnode *pVnode); static int vnodeOpenImpl(SVnode *pVnode);
static void vnodeCloseImpl(SVnode *pVnode); static void vnodeCloseImpl(SVnode *pVnode);
TD_DEF_MOD_INIT_FLAG(vnode);
TD_DEF_MOD_CLEAR_FLAG(vnode);
int vnodeInit() { int vnodeInit() {
// TODO if (TD_CHECK_AND_SET_MODE_INIT(vnode) == TD_MOD_INITIALIZED) {
return 0;
}
if (walInit() < 0) { if (walInit() < 0) {
return -1; return -1;
} }
@ -30,6 +37,10 @@ int vnodeInit() {
} }
void vnodeClear() { void vnodeClear() {
if (TD_CHECK_AND_SET_MOD_CLEAR(vnode) == TD_MOD_CLEARD) {
return;
}
walCleanUp(); walCleanUp();
} }

113
source/dnode/vnode/impl/src/vnodeMemAllocator.c
View File

@ -1,113 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeDef.h"
SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode) {
SMemAllocator *pma = NULL;
/* TODO */
return pma;
}
void vnodeDestroyMemAllocator(SMemAllocator *pma) {
// TODO
}
#if 0
#define VNODE_HEAP_ALLOCATOR 0
#define VNODE_ARENA_ALLOCATOR 1
typedef struct {
uint64_t tsize;
uint64_t used;
} SVHeapAllocator;
typedef struct SVArenaNode {
struct SVArenaNode *prev;
void * nptr;
char data[];
} SVArenaNode;
typedef struct {
SVArenaNode *inuse;
SVArenaNode node;
} SVArenaAllocator;
typedef struct {
int8_t type;
uint64_t tsize;
T_REF_DECLARE()
union {
SVHeapAllocator vha;
SVArenaAllocator vaa;
};
} SVMemAllocator;
SMemAllocator *vnodeCreateMemAllocator(int8_t type, uint64_t tsize, uint64_t ssize /* step size only for arena */) {
SMemAllocator * pma;
uint64_t msize;
SVMemAllocator *pva;
msize = sizeof(*pma) + sizeof(SVMemAllocator);
if (type == VNODE_ARENA_ALLOCATOR) {
msize += tsize;
}
pma = (SMemAllocator *)calloc(1, msize);
if (pma == NULL) {
return NULL;
}
pma->impl = POINTER_SHIFT(pma, sizeof(*pma));
pva = (SVMemAllocator *)(pma->impl);
pva->type = type;
pva->tsize = tsize;
if (type == VNODE_HEAP_ALLOCATOR) {
pma->malloc = NULL;
pma->calloc = NULL;
pma->realloc = NULL;
pma->free = NULL;
pma->usage = NULL;
} else if (type == VNODE_ARENA_ALLOCATOR) {
pma->malloc = NULL;
pma->calloc = NULL;
pma->realloc = NULL;
pma->free = NULL;
pma->usage = NULL;
} else {
ASSERT(0);
}
return pma;
}
void vnodeDestroyMemAllocator(SMemAllocator *pma) {
// TODO
}
void vnodeRefMemAllocator(SMemAllocator *pma) {
// TODO
}
void vnodeUnrefMemAllocator(SMemAllocator *pma) {
// TODO
}
/* ------------------------ Heap Allocator IMPL ------------------------ */
/* ------------------------ Arena Allocator IMPL ------------------------ */
#endif

7
source/dnode/vnode/impl/test/CMakeLists.txt
View File

@ -4,4 +4,9 @@ target_sources(vnodeApiTests
PRIVATE PRIVATE
"vnodeApiTests.cpp" "vnodeApiTests.cpp"
) )
target_link_libraries(vnodeApiTests vnode gtest gtest_main) target_link_libraries(vnodeApiTests vnode gtest gtest_main)
add_test(
NAME vnode_api_tests
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests
)

11
source/dnode/vnode/impl/test/vnodeApiTests.cpp
View File

@ -1,3 +1,14 @@
/**
* @file vnodeApiTests.cpp
* @author hzcheng (hzcheng@taosdata.com)
* @brief VNODE module API tests
* @version 0.1
* @date 2021-12-13
*
* @copyright Copyright (c) 2021
*
*/
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <iostream> #include <iostream>