From 98059c65e39bea413b827a70b48f1aed525e1fd9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 10 Dec 2021 11:21:07 +0800 Subject: [PATCH 01/24] more --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ba8543e518..14449987d9 100644 --- a/.gitignore +++ b/.gitignore @@ -101,3 +101,4 @@ TAGS contrib/* !contrib/CMakeLists.txt !contrib/test +.devcontainer/ \ No newline at end of file From 22a2d4299271f08c81f3ae903f3b90b8e372173f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 11 Dec 2021 07:00:52 +0000 Subject: [PATCH 02/24] more --- .devcontainer/Dockerfile | 9 ++++++++ .devcontainer/devcontainer.json | 32 ++++++++++++++++++++++++++ .gitignore | 3 +-- source/dnode/vnode/impl/CMakeLists.txt | 6 ++--- 4 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..fc9a51af65 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp/.devcontainer/base.Dockerfile + +# [Choice] Debian / Ubuntu version (use Debian 11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 +ARG VARIANT="bullseye" +FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT} + +# [Optional] Uncomment this section to install additional packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..9b752d091d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp +{ + "name": "C++", + "build": { + "dockerfile": "Dockerfile", + // Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 + // Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon + "args": { "VARIANT": "ubuntu-21.04" } + }, + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + + // Set *default* container specific settings.json values on container create. + "settings": {}, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "austin.code-gnu-global", + "visualstudioexptteam.vscodeintel" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "gcc -v", + + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} diff --git a/.gitignore b/.gitignore index 14449987d9..ab4fbda71f 100644 --- a/.gitignore +++ b/.gitignore @@ -100,5 +100,4 @@ TAGS contrib/* !contrib/CMakeLists.txt -!contrib/test -.devcontainer/ \ No newline at end of file +!contrib/test \ No newline at end of file diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index d6d267c4d4..6972605afd 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -18,6 +18,6 @@ target_link_libraries( ) # test -#if(${BUILD_TEST}) -# add_subdirectory(test) -#endif(${BUILD_TEST}) \ No newline at end of file +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file From ae18e4c390487507668f1b7e20204e9a47df1337 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 11 Dec 2021 17:28:31 +0800 Subject: [PATCH 03/24] more work --- CMakeLists.txt | 2 +- source/dnode/vnode/impl/inc/vnodeBufferPool.h | 1 + source/dnode/vnode/impl/inc/vnodeCommit.h | 4 ++-- source/dnode/vnode/impl/src/vnodeBufferPool.c | 22 +++++++++++++++++-- source/dnode/vnode/impl/src/vnodeCommit.c | 2 -- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b626977588..9762a466b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake") set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib") include(${CMAKE_SUPPORT_DIR}/cmake.options) -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -gdwarf-2 -msse4.2 -mfma -g3") # contrib add_subdirectory(contrib) diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/impl/inc/vnodeBufferPool.h index d64dc93847..bfc4de9e12 100644 --- a/source/dnode/vnode/impl/inc/vnodeBufferPool.h +++ b/source/dnode/vnode/impl/inc/vnodeBufferPool.h @@ -28,6 +28,7 @@ typedef struct SVBufPool SVBufPool; int vnodeOpenBufPool(SVnode *pVnode); void vnodeCloseBufPool(SVnode *pVnode); void *vnodeMalloc(SVnode *pVnode, uint64_t size); +bool vnodeBufPoolIsFull(SVnode *pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/inc/vnodeCommit.h b/source/dnode/vnode/impl/inc/vnodeCommit.h index c8ff4947aa..a60e8feac2 100644 --- a/source/dnode/vnode/impl/inc/vnodeCommit.h +++ b/source/dnode/vnode/impl/inc/vnodeCommit.h @@ -22,8 +22,8 @@ extern "C" { #endif -bool vnodeShouldCommit(SVnode *pVnode); -int vnodeAsyncCommit(SVnode *pVnode); +#define vnodeShouldCommit vnodeBufPoolIsFull +int vnodeAsyncCommit(SVnode *pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 00203ed9b6..7415033f02 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -148,6 +148,24 @@ void *vnodeMalloc(SVnode *pVnode, uint64_t size) { return vBufPoolMalloc(pvma, size); } +bool vnodeBufPoolIsFull(SVnode *pVnode) { + SVBufPool * pBufPool = pVnode->pBufPool; + SVMemAllocator *pvma; + + if (pBufPool->inuse == NULL) return false; + + pvma = (SVMemAllocator *)(pBufPool->inuse->data); + if (pvma->type == E_V_HEAP_ALLOCATOR) { + ASSERT(0); + } else { + SVArenaNode *pNode = pvma->vaa.inuse; + bool ret = + (pNode != &(pvma->vaa.node)) || ((pNode->size - POINTER_DISTANCE(pNode->ptr, pNode->data)) <= pvma->vaa.lsize); + + return ret; + } +} + /* ------------------------ STATIC METHODS ------------------------ */ static void vArenaAllocatorInit(SVArenaAllocator *pvaa, uint64_t capacity, uint64_t ssize, uint64_t lsize) { /* TODO */ pvaa->ssize = ssize; @@ -171,8 +189,8 @@ static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { SListNode * pNode; SVMemAllocator *pvma; uint64_t msize; - uint64_t ssize = 0; // TODO - uint64_t lsize = 0; // TODO + uint64_t ssize = 4096; // TODO + uint64_t lsize = 1024; // TODO msize = sizeof(SListNode) + sizeof(SVMemAllocator); if (type == E_V_ARENA_ALLOCATOR) { diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/impl/src/vnodeCommit.c index 18a0c6d91d..cac7999f59 100644 --- a/source/dnode/vnode/impl/src/vnodeCommit.c +++ b/source/dnode/vnode/impl/src/vnodeCommit.c @@ -18,8 +18,6 @@ static int vnodeStartCommit(SVnode *pVnode); static int vnodeEndCommit(SVnode *pVnode); -bool vnodeShouldCommit(SVnode *pVnode) { return false; } - int vnodeAsyncCommit(SVnode *pVnode) { #if 0 if (vnodeStartCommit(pVnode) < 0) { From 556c7912a652ae3d43be76cccf2816c1f6e5a436 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 11 Dec 2021 22:41:16 +0800 Subject: [PATCH 04/24] integrate WAL --- source/dnode/vnode/impl/src/vnodeCfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/src/vnodeCfg.c b/source/dnode/vnode/impl/src/vnodeCfg.c index 01facba888..7aac9ca6a7 100644 --- a/source/dnode/vnode/impl/src/vnodeCfg.c +++ b/source/dnode/vnode/impl/src/vnodeCfg.c @@ -15,7 +15,7 @@ #include "vnodeDef.h" -const SVnodeCfg defaultVnodeOptions = {0}; /* TODO */ +const SVnodeCfg defaultVnodeOptions = {.wsize = 16*1024*1024, .walCfg = {.walLevel = TAOS_WAL_WRITE}}; /* TODO */ void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */ vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions); From 9f9e2ebd4c10955e9852645e9a4634d8393ddcc6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 10:47:52 +0800 Subject: [PATCH 05/24] add module init --- include/util/tmacro.h | 44 +++++++++++++++++++++++++ source/dnode/vnode/impl/src/vnodeMain.c | 13 +++++++- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 include/util/tmacro.h diff --git a/include/util/tmacro.h b/include/util/tmacro.h new file mode 100644 index 0000000000..74056cfe07 --- /dev/null +++ b/include/util/tmacro.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_MACRO_H_ +#define _TD_UTIL_MACRO_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Module init/clear MACRO definitions +#define TD_MOD_UNINITIALIZED 0 +#define TD_MOD_INITIALIZED 1 + +#define TD_MOD_UNCLEARD 0 +#define TD_MOD_CLEARD 1 + +#define TD_DEF_MOD_INIT_FLAG(MOD) static int8_t MOD##InitFlag = TD_MOD_UNINITIALIZED +#define TD_DEF_MOD_CLEAR_FLAG(MOD) static int8_t MOD##ClearFlag = TD_MOD_UNCLEARD + +#define TD_CHECK_AND_SET_MODE_INIT(MOD) \ + atomic_val_compare_exchange_8(&(MOD##InitFlag), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED) + +#define TD_CHECK_AND_SET_MOD_CLEAR(MOD) atomic_val_compare_exchange_8(&(MOD##ClearFlag), TD_MOD_UNCLEARD, TD_MOD_CLEARD) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_MACRO_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index ab33b58858..9b94b4a361 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tmacro.h" #include "vnodeDef.h" static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg); @@ -20,8 +21,14 @@ static void vnodeFree(SVnode *pVnode); static int vnodeOpenImpl(SVnode *pVnode); static void vnodeCloseImpl(SVnode *pVnode); +TD_DEF_MOD_INIT_FLAG(vnode); +TD_DEF_MOD_CLEAR_FLAG(vnode); + int vnodeInit() { - // TODO + if (TD_CHECK_AND_SET_MODE_INIT(vnode) == TD_MOD_INITIALIZED) { + return 0; + } + if (walInit() < 0) { return -1; } @@ -30,6 +37,10 @@ int vnodeInit() { } void vnodeClear() { + if (TD_CHECK_AND_SET_MOD_CLEAR(vnode) == TD_MOD_CLEARD) { + return; + } + walCleanUp(); } From 103a6225901a7dfa1b10aa4f5e45951b38eecb71 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 16:31:39 +0800 Subject: [PATCH 06/24] more --- include/dnode/vnode/vnode.h | 2 + include/util/tdlist.h | 132 ++++++++++++++++ source/dnode/vnode/impl/inc/vnodeDef.h | 4 +- .../dnode/vnode/impl/inc/vnodeMemAllocator.h | 28 +++- .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 70 +++++++++ source/dnode/vnode/impl/src/vnodeBufferPool.c | 141 +++++++++--------- .../dnode/vnode/impl/src/vnodeMemAllocator.c | 113 -------------- 7 files changed, 300 insertions(+), 190 deletions(-) create mode 100644 include/util/tdlist.h create mode 100644 source/dnode/vnode/impl/src/vnodeArenaMAImpl.c delete mode 100644 source/dnode/vnode/impl/src/vnodeMemAllocator.c diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 30531ad738..007ce83812 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -36,6 +36,8 @@ typedef struct SVnodeCfg { struct { /** write buffer size */ uint64_t wsize; + uint64_t ssize; + uint64_t lsize; /** use heap allocator or arena allocator */ bool isHeapAllocator; }; diff --git a/include/util/tdlist.h b/include/util/tdlist.h new file mode 100644 index 0000000000..7ebd8f5d09 --- /dev/null +++ b/include/util/tdlist.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_TDLIST_H_ +#define _TD_UTIL_TDLIST_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define TD_LIST_NODE(S) \ + struct { \ + S *prev_; \ + S *next_; \ + } + +#define TD_LIST(S) \ + struct { \ + S * head_; \ + S * tail_; \ + int neles_; \ + } + +#define tlistInit(l) \ + (l)->head_ = (l)->tail_ = NULL; \ + (l)->neles_ = 0; + +#define tlistHead(l) (l)->head_ +#define tlistTail(l) (l)->tail_ +#define tlistNEles(l) (l)->neles_ + +#define tlistAppend(l, n) \ + if ((l)->head_ == NULL) { \ + (n)->prev_ = (n)->next_ = NULL; \ + (l)->head_ = (l)->tail_ = (n); \ + } else { \ + (n)->prev_ = (l)->tail_; \ + (n)->next_ = NULL; \ + (l)->tail_->next_ = (n); \ + (l)->tail_ = (n); \ + } \ + (l)->neles_ += 1; + +#define tlistPrepend(l, n) \ + if ((l)->head_ == NULL) { \ + (n)->prev_ = (n)->next_ = NULL; \ + (l)->head_ = (l)->tail_ = (n); \ + } else { \ + (n)->prev_ = NULL; \ + (n)->next_ = (l)->head_; \ + (l)->head_->prev_ = (n); \ + (l)->head_ = (n); \ + } \ + (l)->neles_ += 1; + +#define tlistPop(l, n) \ + ({ \ + if ((n)) { \ + if ((l)->head_ == (n)) { \ + (l)->head_ = (n)->next_; \ + } \ + if ((l)->tail_ == (n)) { \ + (l)->tail_ = (n)->prev_; \ + } \ + if ((n)->prev_ != NULL) { \ + (n)->prev_->next_ = (n)->next_; \ + } \ + if ((n)->next_ != NULL) { \ + (n)->next_->prev_ = (n)->prev_; \ + } \ + (l)->neles_ -= 1; \ + (n)->prev_ = (n)->next_ = NULL; \ + } \ + (n); \ + }) + +#define tlistPopHead(l) tlistPop(l, (l)->head_) + +#define tlistPopTail(l) tlistPop(l, (l)->tail_) + +#define tlistIterInit(it, l, dir) + +// List iterator +#define TD_LIST_FITER 0 +#define TD_LIST_BITER 1 +#define TD_LIST_ITER(S) \ + struct { \ + int it_dir_; \ + S * it_next_; \ + S * it_ptr_; \ + TD_LIST(S) * it_list_; \ + } + +#define tlistIterInit(it, l, dir) \ + (it)->it_dir_ = (dir); \ + (it)->it_list_ = l; \ + if ((dir) == TD_LIST_FITER) { \ + (it)->it_next_ = (l)->head_; \ + } else { \ + (it)->it_next_ = (l)->tail_; \ + } + +#define tlistIterNext(it) \ + ({ \ + (it)->it_ptr_ = (it)->it_next_; \ + if ((it)->it_next_ != NULL) { \ + if ((it)->it_dir_ == TD_LIST_FITER) { \ + (it)->it_next_ = (it)->it_next_->next_; \ + } else { \ + (it)->it_next_ = (it)->it_next_->prev_; \ + } \ + } \ + (it)->it_ptr_; \ + }) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_TDLIST_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index c92de433c3..e3a3fac6b9 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -18,15 +18,17 @@ #include "mallocator.h" #include "sync.h" +#include "tcoding.h" +#include "tdlist.h" #include "tlockfree.h" #include "wal.h" -#include "tcoding.h" #include "vnode.h" #include "vnodeBufferPool.h" #include "vnodeCfg.h" #include "vnodeCommit.h" #include "vnodeFS.h" +#include "vnodeMemAllocator.h" #include "vnodeRequest.h" #include "vnodeStateMgr.h" #include "vnodeSync.h" diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index 9184eb416b..bd014c6ff6 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -16,15 +16,35 @@ #ifndef _TD_VNODE_MEM_ALLOCATOR_H_ #define _TD_VNODE_MEM_ALLOCATOR_H_ -#include "mallocator.h" -#include "vnode.h" +#include "os.h" #ifdef __cplusplus extern "C" { #endif -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode); -void vnodeDestroyMemAllocator(SMemAllocator *pma); +typedef struct SVArenaNode SVArenaNode; +typedef struct SVMemAllocator SVMemAllocator; + +struct SVArenaNode { + TD_LIST_NODE(SVArenaNode); + uint64_t nsize; // current node size + void * ptr; + char data[]; +}; + +struct SVMemAllocator { + TD_LIST_NODE(SVMemAllocator); + uint64_t capacity; + uint64_t ssize; + uint64_t lsize; + TD_LIST(SVArenaNode) nlist; +}; + +SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize); +void vmaDestroy(SVMemAllocator *pVMA); +void vmaReset(SVMemAllocator *pVMA); +void * vmaMalloc(SVMemAllocator *pVMA, uint64_t size); +void vmaFree(SVMemAllocator *pVMA, void *ptr); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c new file mode 100644 index 0000000000..e0c098b24b --- /dev/null +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnodeDef.h" + +static SVArenaNode *vArenaNodeNew(uint64_t capacity); +static void vArenaNodeFree(SVArenaNode *pNode); + +SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { + SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA)); + if (pVMA == NULL) { + // TODO: handle error + return NULL; + } + + pVMA->capacity = capacity; + pVMA->ssize = ssize; + pVMA->lsize = lsize; + tlistInit(&(pVMA->nlist)); + + SVArenaNode *pNode = vArenaNodeNew(capacity); + if (pNode == NULL) { + // TODO + return NULL; + } + + tlistAppend(&(pVMA->nlist), pNode); + + return pVMA; +} + +void vmaDestroy(SVMemAllocator *pVMA) { + // TODO +} + +void vmaReset(SVMemAllocator *pVMA) { + // TODO +} + +void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { + // TODO + return NULL; +} + +void vmaFree(SVMemAllocator *pVMA, void *ptr) { + // TODO +} + +/* ------------------------ STATIC METHODS ------------------------ */ +static SVArenaNode *vArenaNodeNew(uint64_t capacity) { + SVArenaNode *pNode = NULL; + // TODO + return pNode; +} + +static void vArenaNodeFree(SVArenaNode *pNode) { + // TODO +} \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 7415033f02..6a6c97a717 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -19,14 +19,76 @@ #define VNODE_BUF_POOL_SHARDS 3 struct SVBufPool { - // buffer pool impl - SList free; - SList incycle; - SListNode *inuse; + TD_LIST(SVMemAllocator) free; + TD_LIST(SVMemAllocator) incycle; + SVMemAllocator *inuse; // MAF for submodules - SMemAllocatorFactory maf; + // SMemAllocatorFactory maf; }; +int vnodeOpenBufPool(SVnode *pVnode) { + uint64_t capacity; + // EVMemAllocatorT type = E_V_ARENA_ALLOCATOR; + + if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { + /* TODO */ + return -1; + } + + tlistInit(&(pVnode->pBufPool->free)); + tlistInit(&(pVnode->pBufPool->incycle)); + + pVnode->pBufPool->inuse = NULL; + + // TODO + capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS; + + for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) { + SVMemAllocator *pVMA = vmaCreate(capacity, pVnode->config.ssize, pVnode->config.lsize); + if (pVMA == NULL) { + // TODO: handle error + return -1; + } + + tlistAppend(&(pVnode->pBufPool->free), pVMA); + } + + return 0; +} + +void vnodeCloseBufPool(SVnode *pVnode) { + if (pVnode->pBufPool) { + vmaDestroy(pVnode->pBufPool->inuse); + + while (true) { + SVMemAllocator *pVMA = tlistPopHead(&(pVnode->pBufPool->incycle)); + if (pVMA == NULL) break; + vmaDestroy(pVMA); + } + + while (true) { + SVMemAllocator *pVMA = tlistPopHead(&(pVnode->pBufPool->free)); + if (pVMA == NULL) break; + vmaDestroy(pVMA); + } + + free(pVnode->pBufPool); + pVnode->pBufPool = NULL; + } +} + +void *vnodeMalloc(SVnode *pVnode, uint64_t size) { + // TODO + return NULL; +} + +bool vnodeBufPoolIsFull(SVnode *pVnode) { + // TODO + return false; +} + +#if 0 + typedef enum { // Heap allocator E_V_HEAP_ALLOCATOR = 0, @@ -57,15 +119,6 @@ typedef struct { SListNode *pNode; } SVMAWrapper; -typedef struct { - T_REF_DECLARE() - uint64_t capacity; - EVMemAllocatorT type; - union { - SVHeapAllocator vha; - SVArenaAllocator vaa; - }; -} SVMemAllocator; static SListNode * vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); static void vBufPoolFreeNode(SListNode *pNode); @@ -73,63 +126,6 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf); static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); -int vnodeOpenBufPool(SVnode *pVnode) { - uint64_t capacity; - EVMemAllocatorT type = E_V_ARENA_ALLOCATOR; - - if ((pVnode->pBufPool = (SVBufPool *)calloc(1, sizeof(SVBufPool))) == NULL) { - /* TODO */ - return -1; - } - - tdListInit(&(pVnode->pBufPool->free), 0); - tdListInit(&(pVnode->pBufPool->incycle), 0); - - capacity = pVnode->config.wsize / VNODE_BUF_POOL_SHARDS; - if (pVnode->config.isHeapAllocator) { - type = E_V_HEAP_ALLOCATOR; - } - - for (int i = 0; i < VNODE_BUF_POOL_SHARDS; i++) { - SListNode *pNode = vBufPoolNewNode(capacity, type); - if (pNode == NULL) { - vnodeCloseBufPool(pVnode); - return -1; - } - - tdListAppendNode(&(pVnode->pBufPool->free), pNode); - } - - pVnode->pBufPool->maf.impl = pVnode; - pVnode->pBufPool->maf.create = vBufPoolCreateMA; - pVnode->pBufPool->maf.destroy = vBufPoolDestroyMA; - - return 0; -} - -void vnodeCloseBufPool(SVnode *pVnode) { - SListNode *pNode; - if (pVnode->pBufPool) { - // Clear free list - while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) != NULL) { - vBufPoolFreeNode(pNode); - } - - // Clear incycle list - while ((pNode = tdListPopHead(&(pVnode->pBufPool->incycle))) != NULL) { - vBufPoolFreeNode(pNode); - } - - // Free inuse node - if (pVnode->pBufPool->inuse) { - vBufPoolFreeNode(pVnode->pBufPool->inuse); - } - - free(pVnode->pBufPool); - pVnode->pBufPool = NULL; - } -} - void *vnodeMalloc(SVnode *pVnode, uint64_t size) { void *ptr; @@ -335,4 +331,5 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma) { tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode)); // tsem_post(); todo: sem_post } -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeMemAllocator.c b/source/dnode/vnode/impl/src/vnodeMemAllocator.c deleted file mode 100644 index 902014eb47..0000000000 --- a/source/dnode/vnode/impl/src/vnodeMemAllocator.c +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" - -SMemAllocator *vnodeCreateMemAllocator(SVnode *pVnode) { - SMemAllocator *pma = NULL; - /* TODO */ - return pma; -} - -void vnodeDestroyMemAllocator(SMemAllocator *pma) { - // TODO -} - -#if 0 -#define VNODE_HEAP_ALLOCATOR 0 -#define VNODE_ARENA_ALLOCATOR 1 - -typedef struct { - uint64_t tsize; - uint64_t used; -} SVHeapAllocator; - -typedef struct SVArenaNode { - struct SVArenaNode *prev; - void * nptr; - char data[]; -} SVArenaNode; - -typedef struct { - SVArenaNode *inuse; - SVArenaNode node; -} SVArenaAllocator; - -typedef struct { - int8_t type; - uint64_t tsize; - T_REF_DECLARE() - union { - SVHeapAllocator vha; - SVArenaAllocator vaa; - }; -} SVMemAllocator; - -SMemAllocator *vnodeCreateMemAllocator(int8_t type, uint64_t tsize, uint64_t ssize /* step size only for arena */) { - SMemAllocator * pma; - uint64_t msize; - SVMemAllocator *pva; - - msize = sizeof(*pma) + sizeof(SVMemAllocator); - if (type == VNODE_ARENA_ALLOCATOR) { - msize += tsize; - } - - pma = (SMemAllocator *)calloc(1, msize); - if (pma == NULL) { - return NULL; - } - - pma->impl = POINTER_SHIFT(pma, sizeof(*pma)); - pva = (SVMemAllocator *)(pma->impl); - pva->type = type; - pva->tsize = tsize; - - if (type == VNODE_HEAP_ALLOCATOR) { - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } else if (type == VNODE_ARENA_ALLOCATOR) { - pma->malloc = NULL; - pma->calloc = NULL; - pma->realloc = NULL; - pma->free = NULL; - pma->usage = NULL; - } else { - ASSERT(0); - } - - return pma; -} - -void vnodeDestroyMemAllocator(SMemAllocator *pma) { - // TODO -} - -void vnodeRefMemAllocator(SMemAllocator *pma) { - // TODO -} - -void vnodeUnrefMemAllocator(SMemAllocator *pma) { - // TODO -} - -/* ------------------------ Heap Allocator IMPL ------------------------ */ - -/* ------------------------ Arena Allocator IMPL ------------------------ */ - -#endif \ No newline at end of file From f748f0710a20a1584dd4ae84b848f35c302c99b6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 17:19:30 +0800 Subject: [PATCH 07/24] more --- CMakeLists.txt | 6 +++++- source/dnode/vnode/impl/test/CMakeLists.txt | 7 ++++++- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 11 +++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9762a466b1..1a8cb1d710 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,9 +20,13 @@ add_library(api INTERFACE) target_include_directories(api INTERFACE "include/client") # src +if(${BUILD_TEST}) + include(CTest) + enable_testing() +endif(${BUILD_TEST}) add_subdirectory(source) # docs add_subdirectory(docs) -# tests (TODO) +# tests (TODO) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/CMakeLists.txt b/source/dnode/vnode/impl/test/CMakeLists.txt index 83506a4fde..e1226331e9 100644 --- a/source/dnode/vnode/impl/test/CMakeLists.txt +++ b/source/dnode/vnode/impl/test/CMakeLists.txt @@ -4,4 +4,9 @@ target_sources(vnodeApiTests PRIVATE "vnodeApiTests.cpp" ) -target_link_libraries(vnodeApiTests vnode gtest gtest_main) \ No newline at end of file +target_link_libraries(vnodeApiTests vnode gtest gtest_main) + +add_test( + NAME vnode_api_tests + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests + ) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index df784181b7..ac2ccbc132 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -1,3 +1,14 @@ +/** + * @file vnodeApiTests.cpp + * @author hzcheng (hzcheng@taosdata.com) + * @brief VNODE module API tests + * @version 0.1 + * @date 2021-12-13 + * + * @copyright Copyright (c) 2021 + * + */ + #include #include From 0a9eb233b78752b0c235d8181bc5e9be5754efe6 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 13 Dec 2021 04:54:45 -0500 Subject: [PATCH 08/24] TD-11819 mock catalog function for parser ut --- include/libs/parser/parser.h | 2 +- source/libs/parser/inc/dataBlockMgt.h | 16 +-- source/libs/parser/src/dataBlockMgt.c | 27 ++--- source/libs/parser/src/insertParser.c | 20 ++-- source/libs/parser/test/insertTest.cpp | 52 +++++---- source/libs/parser/test/mockCatalog.cpp | 45 ++++---- .../libs/parser/test/mockCatalogService.cpp | 101 +++++++++++++++--- source/libs/parser/test/mockCatalogService.h | 5 +- 8 files changed, 163 insertions(+), 105 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index e2817baeae..2f152c3e2b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -162,7 +162,7 @@ typedef struct SVgDataBlocks { int64_t vgId; // virtual group id int32_t numOfTables; // number of tables in current submit block uint32_t size; - char *pData; + char *pData; // SMsgDesc + SSubmitMsg + SSubmitBlk + ... } SVgDataBlocks; typedef struct SInsertStmtInfo { diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h index 350610ec06..7d2e6e3aec 100644 --- a/source/libs/parser/inc/dataBlockMgt.h +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -75,16 +75,7 @@ typedef struct { SMemRowInfo *rowInfo; } SMemRowBuilder; -typedef struct SParamInfo { - int32_t idx; - uint8_t type; - uint8_t timePrec; - int16_t bytes; - uint32_t offset; -} SParamInfo; - typedef struct STableDataBlocks { - SName tableName; int8_t tsSource; // where does the UNIX timestamp come from, server or client bool ordered; // if current rows are ordered or not int64_t vgId; // virtual group id @@ -100,11 +91,6 @@ typedef struct STableDataBlocks { STagData tagData; SParsedDataColInfo boundColumnInfo; - - // for parameter ('?') binding - uint32_t numOfAllocedParams; - uint32_t numOfParams; - SParamInfo * params; SMemRowBuilder rowBuilder; } STableDataBlocks; @@ -187,7 +173,7 @@ void destroyBoundColumnInfo(SParsedDataColInfo* pColList); int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - SName* name, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, bool freeBlockMap); #endif // TDENGINE_DATABLOCKMGT_H diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index 4ece848888..3138f0c1d8 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -108,7 +108,7 @@ void destroyBoundColumnInfo(SParsedDataColInfo* pColList) { tfree(pColList->colIdxInfo); } -static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, SName* name, +static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { @@ -145,7 +145,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star dataBuf->tsSource = -1; dataBuf->vgId = dataBuf->pTableMeta->vgId; - tNameAssign(&dataBuf->tableName, name); + // tNameAssign(&dataBuf->tableName, name); assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); @@ -154,8 +154,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star } int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - SName* name, const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, - SArray* pBlockList) { + const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); if (t1 != NULL) { @@ -163,7 +162,7 @@ int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int3 } if (*dataBlocks == NULL) { - int32_t ret = createDataBlock((size_t)size, rowSize, startOffset, name, pTableMeta, dataBlocks); + int32_t ret = createDataBlock((size_t)size, rowSize, startOffset, pTableMeta, dataBlocks); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -253,23 +252,13 @@ static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlo } } -void destroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { +void destroyDataBlock(STableDataBlocks* pDataBlock) { if (pDataBlock == NULL) { return; } tfree(pDataBlock->pData); - - if (removeMeta) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pDataBlock->tableName, name); - - // taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); - } - if (!pDataBlock->cloned) { - tfree(pDataBlock->params); - // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { tfree(pDataBlock->pTableMeta); @@ -277,7 +266,6 @@ void destroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); } - tfree(pDataBlock); } @@ -289,7 +277,7 @@ void* destroyBlockArrayList(SArray* pDataBlockList) { size_t size = taosArrayGetSize(pDataBlockList); for (int32_t i = 0; i < size; i++) { void* d = taosArrayGetP(pDataBlockList, i); - destroyDataBlock(d, false); + destroyDataBlock(d); } taosArrayDestroy(pDataBlockList); @@ -505,7 +493,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t if (pBlocks->numOfRows > 0) { STableDataBlocks* dataBuf = NULL; int32_t ret = getDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); if (ret != TSDB_CODE_SUCCESS) { taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); @@ -580,7 +568,6 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t extractTableNameList(pHashObj, freeBlockMap); // free the table data blocks; - // pInsertParam->pDataBlocks = pVnodeDataBlockList; taosHashCleanup(pVnodeDataBlockHashList); tfree(blkKeyInfo.pKeyTuple); diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 33b02f4261..c6ae149f1f 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -167,11 +167,12 @@ static int32_t skipInsertInto(SInsertParseContext* pCxt) { static int32_t buildTableName(SInsertParseContext* pCxt, SToken* pStname, SArray* tableNameList) { if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(&pCxt->msg, "invalid table name"); + return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pStname->z); } SName name = {0}; - strndequote(name.tname, pStname->z, pStname->n); + strcpy(name.dbname, pCxt->pComCxt->pDbname); + strncpy(name.tname, pStname->z, pStname->n); taosArrayPush(tableNameList, &name); return TSDB_CODE_SUCCESS; @@ -686,7 +687,6 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { NEXT_TOKEN(pCxt->pSql, sToken); - // todo bind param SSchema *pSchema = &schema[spd->boundedColumns[i]]; param.schema = pSchema; param.compareStat = pBuilder->compareStat; @@ -770,14 +770,6 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, STableDataBlocks* da int32_t numOfRows = 0; CHECK_CODE(parseValues(pCxt, dataBuf, maxNumOfRows, &numOfRows)); - for (uint32_t i = 0; i < dataBuf->numOfParams; ++i) { - SParamInfo *param = dataBuf->params + i; - if (param->idx == -1) { - // param->idx = pInsertParam->numOfParams++; - param->offset -= sizeof(SSubmitBlk); - } - } - SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); if (TSDB_CODE_SUCCESS != setBlockInfo(pBlocks, dataBuf->pTableMeta, numOfRows)) { return buildInvalidOperationMsg(&pCxt->msg, "too many rows in sql, total number of rows should be less than 32767"); @@ -815,12 +807,12 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { CHECK_CODE(parseUsingClause(pCxt, &tbnameToken)); NEXT_TOKEN(pCxt->pSql, sToken); } else { - CHECK_CODE(getTableMeta(pCxt, &sToken)); + CHECK_CODE(getTableMeta(pCxt, &tbnameToken)); } STableDataBlocks *dataBuf = NULL; CHECK_CODE(getDataBlockFromList(pCxt->pTableBlockHashObj, pCxt->pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, NULL/* tbname */, pCxt->pTableMeta, &dataBuf, NULL)); + sizeof(SSubmitBlk), getTableInfo(pCxt->pTableMeta).rowSize, pCxt->pTableMeta, &dataBuf, NULL)); if (TK_LP == sToken.type) { // pSql -> field1_name, ...) @@ -831,6 +823,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { if (TK_VALUES == sToken.type) { // pSql -> (field1_value, ...) [(field1_value2, ...) ...] CHECK_CODE(parseValuesClause(pCxt, dataBuf)); + pCxt->pOutput->insertType = TSDB_QUERY_TYPE_INSERT; continue; } @@ -842,6 +835,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", sToken.z); } // todo + pCxt->pOutput->insertType = TSDB_QUERY_TYPE_FILE_INSERT; continue; } diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp index 08ca515b17..9cf48da4eb 100644 --- a/source/libs/parser/test/insertTest.cpp +++ b/source/libs/parser/test/insertTest.cpp @@ -36,51 +36,67 @@ namespace { // [...]; class InsertTest : public Test { protected: + void setDatabase(const string& db) { + db_ = db; + } + void bind(const char* sql) { reset(); - cxt.pSql = sql; - cxt.sqlLen = strlen(sql); + cxt_.sqlLen = strlen(sql); + strcpy(sqlBuf_, sql); + sqlBuf_[cxt_.sqlLen] = '\0'; + cxt_.pSql = sqlBuf_; + cxt_.pDbname = db_.c_str(); } int32_t run() { - code = parseInsertSql(&cxt, &res); - if (code != TSDB_CODE_SUCCESS) { - cout << "code:" << toString(code) << ", msg:" << errMagBuf << endl; + code_ = parseInsertSql(&cxt_, &res_); + if (code_ != TSDB_CODE_SUCCESS) { + cout << "code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; } - return code; + return code_; } SInsertStmtInfo* reslut() { - return res; + return res_; } private: static const int max_err_len = 1024; + static const int max_sql_len = 1024 * 1024; void reset() { - memset(&cxt, 0, sizeof(cxt)); - memset(errMagBuf, 0, max_err_len); - cxt.pMsg = errMagBuf; - cxt.msgLen = max_err_len; - code = TSDB_CODE_SUCCESS; - res = nullptr; + memset(&cxt_, 0, sizeof(cxt_)); + memset(errMagBuf_, 0, max_err_len); + cxt_.pMsg = errMagBuf_; + cxt_.msgLen = max_err_len; + code_ = TSDB_CODE_SUCCESS; + res_ = nullptr; } - char errMagBuf[max_err_len]; - SParseContext cxt; - int32_t code; - SInsertStmtInfo* res; + string db_; + char errMagBuf_[max_err_len]; + char sqlBuf_[max_sql_len]; + SParseContext cxt_; + int32_t code_; + SInsertStmtInfo* res_; }; // INSERT INTO tb_name VALUES (field1_value, ...) TEST_F(InsertTest, simpleTest) { - bind("insert into .. values (...)"); + setDatabase("test"); + + bind("insert into t1 values (now, 1, \"wxy\")"); ASSERT_EQ(run(), TSDB_CODE_SUCCESS); SInsertStmtInfo* res = reslut(); // todo check + ASSERT_EQ(res->insertType, TSDB_QUERY_TYPE_INSERT); + // ASSERT_EQ(taosArrayGetSize(res->pDataBlocks), 1); } TEST_F(InsertTest, toleranceTest) { + setDatabase("test"); + bind("insert into"); ASSERT_NE(run(), TSDB_CODE_SUCCESS); bind("insert into t"); diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index fc988512c9..1f6fd71030 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -17,29 +17,30 @@ #include +namespace { + +void generateTestT1(MockCatalogService* mcs) { + ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 3) + .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + builder.done(); +} + +void generateTestST1(MockCatalogService* mcs) { + ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2) + .setPrecision(TSDB_TIME_PRECISION_MILLI).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) + .addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 10) + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 10); + builder.done(); + mcs->createSubTable("test", "st1", "st1s1", 1); + mcs->createSubTable("test", "st1", "st1s2", 2); +} + +} + void generateMetaData(MockCatalogService* mcs) { - { - ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, MockCatalogService::numOfDataTypes) - .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP); - for (int32_t i = 0; i < MockCatalogService::numOfDataTypes; ++i) { - if (TSDB_DATA_TYPE_NULL == tDataTypes[i].type) { - continue; - } - builder = builder.addColumn("c" + std::to_string(i + 1), tDataTypes[i].type); - } - builder.done(); - } - { - ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, MockCatalogService::numOfDataTypes, 2) - .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(2).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP); - for (int32_t i = 0; i < MockCatalogService::numOfDataTypes; ++i) { - if (TSDB_DATA_TYPE_NULL == tDataTypes[i].type) { - continue; - } - builder = builder.addColumn("c" + std::to_string(i + 1), tDataTypes[i].type); - } - builder.done(); - } + generateTestT1(mcs); + generateTestST1(mcs); mcs->showTables(); } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index de4fa8ee7e..96429e1031 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -19,6 +19,7 @@ #include #include +#include "tname.h" #include "ttypes.h" std::unique_ptr mockCatalogService; @@ -82,12 +83,26 @@ public: MockCatalogServiceImpl() { } - struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) { + struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const { return (struct SCatalog*)0x01; } - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { - return 0; + int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) const { + assert(nullptr != pMetaReq && 1 == taosArrayGetSize(pMetaReq->pTableName)); + SName* fullName = (SName*)taosArrayGet(pMetaReq->pTableName, 0); + std::unique_ptr table; + int32_t code = copyTableMeta(fullName->dbname, fullName->tname, &table); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + std::unique_ptr tables((SArray*)taosArrayInit(1, sizeof(STableMeta*))); + if (!tables) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + STableMeta* elem = table.release(); + taosArrayPush(tables.get(), &elem); + pMetaData->pTableMeta = tables.release(); + return TSDB_CODE_SUCCESS; } TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { @@ -97,6 +112,15 @@ public: return *(builder_.get()); } + void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) { + std::unique_ptr table; + if (TSDB_CODE_SUCCESS != copyTableMeta(db, stbname, &table)) { + throw std::runtime_error("copyTableMeta failed"); + } + meta_[db][tbname].reset(table.release()); + meta_[db][tbname]->uid = id_++; + } + void showTables() const { // number of forward fills #define NOF(n) ((n) / 2) @@ -120,20 +144,35 @@ public: #define SL(sn, in) std::setfill('=') << std::setw((sn) * (SFL + 1) + (in) * (IFL + 1)) << "" << std::setfill(' ') for (const auto& db : meta_) { - std::cout << SH("Database") << SH("Table") << SH("Type") << SH("Precision") << IH(std::string("Vgid")) << std::endl; - std::cout << SL(4, 1) << std::endl; + std::cout << "Databse:" << db.first << std::endl; + std::cout << SH("Table") << SH("Type") << SH("Precision") << IH("Vgid") << std::endl; + std::cout << SL(3, 1) << std::endl; for (const auto& table : db.second) { - std::cout << SF(db.first) << SF(table.first) << SF(ttToString(table.second->tableType)) << SF(pToString(table.second->tableInfo.precision)) << IF(table.second->vgId) << std::endl; - // int16_t numOfFields = table.second->tableInfo.numOfTags + table.second->tableInfo.numOfColumns; - // for (int16_t i = 0; i < numOfFields; ++i) { - // const SSchema* schema = table.second->schema + i; - // std::cout << schema->name << " " << schema->type << " " << schema->bytes << std::endl; - // } + std::cout << SF(table.first) << SF(ttToString(table.second->tableType)) << SF(pToString(table.second->tableInfo.precision)) << IF(table.second->vgId) << std::endl; + } + std::cout << std::endl; + } + + for (const auto& db : meta_) { + for (const auto& table : db.second) { + std::cout << "Table:" << table.first << std::endl; + std::cout << SH("Field") << SH("Type") << SH("DataType") << IH("Bytes") << std::endl; + std::cout << SL(3, 1) << std::endl; + int16_t numOfTags = table.second->tableInfo.numOfTags; + int16_t numOfFields = numOfTags + table.second->tableInfo.numOfColumns; + for (int16_t i = 0; i < numOfFields; ++i) { + const SSchema* schema = table.second->schema + i; + std::cout << SF(std::string(schema->name)) << SH((i < numOfTags ? std::string("tag") : std::string("column"))) << SH(dtToString(schema->type)) << IF(schema->bytes) << std::endl; + } + std::cout << std::endl; } } } private: + typedef std::map > TableMetaCache; + typedef std::map DbMetaCache; + std::string ttToString(int8_t tableType) const { switch (tableType) { case TSDB_SUPER_TABLE: @@ -160,9 +199,39 @@ private: } } + std::string dtToString(int8_t type) const { + return tDataTypes[type].name; + } + + std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { + DbMetaCache::const_iterator it = meta_.find(db); + if (meta_.end() == it) { + return std::shared_ptr(); + } + TableMetaCache::const_iterator tit = it->second.find(tbname); + if (it->second.end() == tit) { + return std::shared_ptr(); + } + return tit->second; + } + + int32_t copyTableMeta(const std::string& db, const std::string& tbname, std::unique_ptr* dst) const { + std::shared_ptr src = getTableMeta(db, tbname); + if (!src) { + return TSDB_CODE_TSC_INVALID_TABLE_NAME; + } + int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); + dst->reset((STableMeta*)std::calloc(1, len)); + if (!dst) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(dst->get(), src.get(), len); + return TSDB_CODE_SUCCESS; + } + uint64_t id_; std::unique_ptr builder_; - std::map > > meta_; + DbMetaCache meta_; }; MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) { @@ -171,11 +240,11 @@ MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) { MockCatalogService::~MockCatalogService() { } -struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) { +struct SCatalog* MockCatalogService::getCatalogHandle(const SEpSet* pMgmtEps) const { return impl_->getCatalogHandle(pMgmtEps); } -int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) { +int32_t MockCatalogService::catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) const { return impl_->catalogGetMetaData(pCatalog, pMetaReq, pMetaData); } @@ -183,6 +252,10 @@ ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, con return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags); } +void MockCatalogService::createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) { + impl_->createSubTable(db, stbname, tbname, vgid); +} + void MockCatalogService::showTables() const { impl_->showTables(); } \ No newline at end of file diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index ba974ab200..79572086a1 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -49,9 +49,10 @@ public: MockCatalogService(); ~MockCatalogService(); - struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); - int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData); + struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) const; + int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) const; ITableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, int32_t numOfColumns, int32_t numOfTags = 0); + void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid); void showTables() const; private: From 8b6eba666fff5d8a519e2d839148f7e55b16f9a1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 17:57:14 +0800 Subject: [PATCH 09/24] more --- include/util/tdlist.h | 2 - .../dnode/vnode/impl/inc/vnodeMemAllocator.h | 3 +- .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 65 ++++++++++++++++--- 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 7ebd8f5d09..757b404ad6 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -90,8 +90,6 @@ extern "C" { #define tlistPopTail(l) tlistPop(l, (l)->tail_) -#define tlistIterInit(it, l, dir) - // List iterator #define TD_LIST_FITER 0 #define TD_LIST_BITER 1 diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index bd014c6ff6..df8b367d25 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -27,7 +27,7 @@ typedef struct SVMemAllocator SVMemAllocator; struct SVArenaNode { TD_LIST_NODE(SVArenaNode); - uint64_t nsize; // current node size + uint64_t size; // current node size void * ptr; char data[]; }; @@ -45,6 +45,7 @@ void vmaDestroy(SVMemAllocator *pVMA); void vmaReset(SVMemAllocator *pVMA); void * vmaMalloc(SVMemAllocator *pVMA, uint64_t size); void vmaFree(SVMemAllocator *pVMA, void *ptr); +bool vmaIsFull(SVMemAllocator *pVMA); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index e0c098b24b..532c2dad8e 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -21,7 +21,6 @@ static void vArenaNodeFree(SVArenaNode *pNode); SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { SVMemAllocator *pVMA = (SVMemAllocator *)malloc(sizeof(*pVMA)); if (pVMA == NULL) { - // TODO: handle error return NULL; } @@ -32,7 +31,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { SVArenaNode *pNode = vArenaNodeNew(capacity); if (pNode == NULL) { - // TODO + free(pVMA); return NULL; } @@ -42,29 +41,79 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { } void vmaDestroy(SVMemAllocator *pVMA) { - // TODO + if (pVMA) { + while (true) { + SVArenaNode *pNode = tlistPopTail(&(pVMA->nlist)); + + if (pNode) { + vArenaNodeFree(pNode); + } else { + break; + } + } + + free(pVMA); + } } void vmaReset(SVMemAllocator *pVMA) { - // TODO + while (tlistNEles(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = tlistPopTail(&(pVMA->nlist)); + vArenaNodeFree(pNode); + } + + SVArenaNode *pNode = tlistHead(&(pVMA->nlist)); + pNode->ptr = pNode->data; } void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { - // TODO - return NULL; + SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + void * ptr; + + if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { + uint64_t capacity = MAX(pVMA->ssize, size); + pNode = vArenaNodeNew(capacity); + if (pNode == NULL) { + // TODO: handle error + return NULL; + } + + tlistAppend(&(pVMA->nlist), pNode); + } + + ptr = pNode->ptr; + pNode->ptr = POINTER_SHIFT(ptr, size); + + return ptr; } void vmaFree(SVMemAllocator *pVMA, void *ptr) { // TODO } +bool vmaIsFull(SVMemAllocator *pVMA) { + SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + + return (tlistNEles(&(pVMA->nlist)) > 1) || (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); +} + /* ------------------------ STATIC METHODS ------------------------ */ static SVArenaNode *vArenaNodeNew(uint64_t capacity) { SVArenaNode *pNode = NULL; - // TODO + + pNode = (SVArenaNode *)malloc(sizeof(*pNode) + capacity); + if (pNode == NULL) { + return NULL; + } + + pNode->size = capacity; + pNode->ptr = pNode->data; + return pNode; } static void vArenaNodeFree(SVArenaNode *pNode) { - // TODO + if (pNode) { + free(pNode); + } } \ No newline at end of file From da065e3115bee42e3e1a95752392d20ed7e382db Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 13 Dec 2021 05:01:34 -0500 Subject: [PATCH 10/24] TD-11819 mock catalog function for parser ut --- source/libs/parser/test/mockCatalogService.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 96429e1031..46e82566d6 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -162,7 +162,7 @@ public: int16_t numOfFields = numOfTags + table.second->tableInfo.numOfColumns; for (int16_t i = 0; i < numOfFields; ++i) { const SSchema* schema = table.second->schema + i; - std::cout << SF(std::string(schema->name)) << SH((i < numOfTags ? std::string("tag") : std::string("column"))) << SH(dtToString(schema->type)) << IF(schema->bytes) << std::endl; + std::cout << SF(std::string(schema->name)) << SH(ftToString(i, numOfTags)) << SH(dtToString(schema->type)) << IF(schema->bytes) << std::endl; } std::cout << std::endl; } @@ -203,6 +203,10 @@ private: return tDataTypes[type].name; } + std::string ftToString(int16_t colid, int16_t numOfTags) const { + return (0 == colid ? "column" : (colid <= numOfTags ? "tag" : "column")); + } + std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { DbMetaCache::const_iterator it = meta_.find(db); if (meta_.end() == it) { From b5b484a79cf6f427e5da3a4a5ab35dd0bfc6dc72 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 18:06:32 +0800 Subject: [PATCH 11/24] more --- source/dnode/vnode/impl/src/vnodeBufferPool.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 6a6c97a717..7738b81bb2 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -78,13 +78,24 @@ void vnodeCloseBufPool(SVnode *pVnode) { } void *vnodeMalloc(SVnode *pVnode, uint64_t size) { - // TODO - return NULL; + SVBufPool *pBufPool = pVnode->pBufPool; + + if (pBufPool->inuse == NULL) { + while (true) { + // TODO: add sem_wait and sem_post + pBufPool->inuse = tlistPopHead(&(pBufPool->free)); + if (pBufPool->inuse) { + break; + } + } + } + + return vmaMalloc(pBufPool->inuse, size); } bool vnodeBufPoolIsFull(SVnode *pVnode) { - // TODO - return false; + if (pVnode->pBufPool->inuse == NULL) return false; + return vmaIsFull(pVnode->pBufPool->inuse); } #if 0 From 0eff683d18caa4906f7b4516140b22223aa2b522 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 18:14:45 +0800 Subject: [PATCH 12/24] more work --- source/dnode/vnode/impl/src/vnodeBufferPool.c | 54 ------------------- 1 file changed, 54 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 7738b81bb2..fe15e9ebff 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -137,61 +137,7 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf); static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); -void *vnodeMalloc(SVnode *pVnode, uint64_t size) { - void *ptr; - - if (pVnode->pBufPool->inuse == NULL) { - SListNode *pNode; - while ((pNode = tdListPopHead(&(pVnode->pBufPool->free))) == NULL) { - // todo - // tsem_wait(); - ASSERT(0); - } - - pVnode->pBufPool->inuse = pNode; - } - - SVMemAllocator *pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - return vBufPoolMalloc(pvma, size); -} - -bool vnodeBufPoolIsFull(SVnode *pVnode) { - SVBufPool * pBufPool = pVnode->pBufPool; - SVMemAllocator *pvma; - - if (pBufPool->inuse == NULL) return false; - - pvma = (SVMemAllocator *)(pBufPool->inuse->data); - if (pvma->type == E_V_HEAP_ALLOCATOR) { - ASSERT(0); - } else { - SVArenaNode *pNode = pvma->vaa.inuse; - bool ret = - (pNode != &(pvma->vaa.node)) || ((pNode->size - POINTER_DISTANCE(pNode->ptr, pNode->data)) <= pvma->vaa.lsize); - - return ret; - } -} - /* ------------------------ STATIC METHODS ------------------------ */ -static void vArenaAllocatorInit(SVArenaAllocator *pvaa, uint64_t capacity, uint64_t ssize, uint64_t lsize) { /* TODO */ - pvaa->ssize = ssize; - pvaa->lsize = lsize; - pvaa->inuse = &pvaa->node; - - pvaa->node.prev = NULL; - pvaa->node.size = capacity; - pvaa->node.ptr = pvaa->node.data; -} - -static void vArenaAllocatorClear(SVArenaAllocator *pvaa) { /* TODO */ - while (pvaa->inuse != &(pvaa->node)) { - SVArenaNode *pANode = pvaa->inuse; - pvaa->inuse = pANode->prev; - free(pANode); - } -} - static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { SListNode * pNode; SVMemAllocator *pvma; From 2589c3bcb9f6fb1206f88c5b65bef3c308ffc3cc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 13 Dec 2021 18:19:09 +0800 Subject: [PATCH 13/24] TD-10431 create user test --- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 11 +- source/dnode/mgmt/impl/test/user/user.cpp | 489 +++++++------------- source/dnode/mnode/impl/src/mndUser.c | 9 +- source/dnode/mnode/sdb/src/sdb.c | 4 +- 4 files changed, 183 insertions(+), 330 deletions(-) diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index c156852905..fba3794f6a 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -271,7 +271,6 @@ TEST_F(DndTestDnode, DropDnode_01) { ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - taosMsleep(1300); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); SendThenCheckShowRetrieveMsg(1); CheckInt16(1); @@ -363,7 +362,7 @@ TEST_F(DndTestDnode, CreateDnode_02) { } TEST_F(DndTestDnode, RestartDnode_01) { - uInfo("===> stop all server"); + uInfo("stop all server"); stopServer(pServer1); stopServer(pServer2); stopServer(pServer3); @@ -375,14 +374,16 @@ TEST_F(DndTestDnode, RestartDnode_01) { pServer4 = NULL; pServer5 = NULL; - taosMsleep(3000); // wait tcp port cleanedup - uInfo("===> start all server"); + uInfo("start all server"); const char* fqdn = "localhost"; const char* firstEp = "localhost:9521"; pServer1 = startServer("/tmp/dndTestDnode1", fqdn, 9521, firstEp); + // pServer1 = startServer("/tmp/dndTestDnode3", fqdn, 9523, firstEp); + // pServer1 = startServer("/tmp/dndTestDnode4", fqdn, 9524, firstEp); + // pServer1 = startServer("/tmp/dndTestDnode5", fqdn, 9525, firstEp); - uInfo("===> all server is running"); + uInfo("all server is running"); // taosMsleep(1300); // SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index c3c1d1d406..137e271b32 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -17,202 +17,174 @@ class DndTestUser : public ::testing::Test { protected: - void SetUp() override {} - void TearDown() override {} + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } static void SetUpTestSuite() { - const char* user = "root"; - const char* pass = "taosdata"; - const char* path = "/tmp/dndTestUser"; - const char* fqdn = "localhost"; - uint16_t port = 9524; - const char* firstEp = "localhost:9524"; + initLog("/tmp/dndTestUser"); - pServer = createServer(path, fqdn, port, firstEp); - ASSERT(pServer); - pClient = createClient(user, pass, fqdn, port); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9530"; + pServer = CreateServer("/tmp/dndTestUser", fqdn, 9530, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9530); + taosMsleep(300); } static void TearDownTestSuite() { stopServer(pServer); dropClient(pClient); + pServer = NULL; + pClient = NULL; } static SServer* pServer; static SClient* pClient; static int32_t connId; + + public: + void SetUp() override {} + void TearDown() override {} + + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = showType; + strcpy(pShow->db, ""); + + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htons(pMeta->numOfTags); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, showName); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, columns); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + } + + void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, type); + EXPECT_EQ(pSchema->bytes, bytes); + EXPECT_STREQ(pSchema->name, name); + } + + void SendThenCheckShowRetrieveMsg(int32_t rows) { + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, rows); + EXPECT_EQ(pRetrieveRsp->offset, 0); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + // EXPECT_EQ(pRetrieveRsp->completed, completed); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->reserved, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + pData = pRetrieveRsp->data; + pos = 0; + } + + void CheckInt16(int16_t val) { + int16_t data = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + EXPECT_EQ(data, val); + } + + void CheckInt64(int64_t val) { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_EQ(data, val); + } + + void CheckTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(data, 0); + } + + void CheckBinary(const char* val, int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + EXPECT_STREQ(data, val); + } + + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; }; SServer* DndTestUser::pServer; SClient* DndTestUser::pClient; int32_t DndTestUser::connId; -#if 0 TEST_F(DndTestUser, ShowUser) { - int32_t showId = 0; + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); + CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); + CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(3, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "account"); - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_USER; - strcpy(pShow->db, ""); - - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - ASSERT_NE(pShowRsp, nullptr); - pShowRsp->showId = htonl(pShowRsp->showId); - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pShowRsp->showId; - - EXPECT_NE(pShowRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tbFname, "show users"); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 4); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "name"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, 10 + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "privilege"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "create_time"); - - pSchema = &pMeta->pSchema[3]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "account"); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = htonl(showId); - pRetrieve->free = 0; - - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - ASSERT_NE(pClient->pRsp, nullptr); - ASSERT_EQ(pClient->pRsp->code, 0); - - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - ASSERT_NE(pRetrieveRsp, nullptr); - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); - pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); - pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); - - EXPECT_EQ(pRetrieveRsp->numOfRows, 2); - EXPECT_EQ(pRetrieveRsp->offset, 0); - EXPECT_EQ(pRetrieveRsp->useconds, 0); - EXPECT_EQ(pRetrieveRsp->completed, 1); - EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->reserved, 0); - EXPECT_EQ(pRetrieveRsp->compLen, 0); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - int64_t int64Val = 0; - - //--- name --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "_root"); - } - - //--- privilege --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += 10; - EXPECT_STREQ(strVal, "super"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += 10; - EXPECT_STREQ(strVal, "writable"); - } - - //--- create_time --- - { - int64Val = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(int64Val, 0); - - int64Val = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - EXPECT_GT(int64Val, 0); - } - - //--- account --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); - } + SendThenCheckShowRetrieveMsg(1); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("super", 10); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); } -#endif TEST_F(DndTestUser, CreateUser_01) { - ASSERT_NE(pClient, nullptr); - - //--- create user --- SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p1"); @@ -227,60 +199,19 @@ TEST_F(DndTestUser, CreateUser_01) { ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_USER; - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->numOfColumns = htons(pMeta->numOfColumns); - EXPECT_EQ(pMeta->numOfColumns, 4); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = pShowRsp->showId; - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - EXPECT_EQ(pRetrieveRsp->numOfRows, 3); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - - //--- name --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "u1"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "_root"); - } + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(2); + CheckBinary("u1", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("normal", 10); + CheckBinary("super", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); } TEST_F(DndTestUser, AlterUser_01) { - ASSERT_NE(pClient, nullptr); - - //--- drop user --- SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); strcpy(pReq->user, "u1"); strcpy(pReq->pass, "p2"); @@ -295,60 +226,19 @@ TEST_F(DndTestUser, AlterUser_01) { ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_USER; - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->numOfColumns = htons(pMeta->numOfColumns); - EXPECT_EQ(pMeta->numOfColumns, 4); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = pShowRsp->showId; - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - EXPECT_EQ(pRetrieveRsp->numOfRows, 3); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - - //--- name --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "u1"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "_root"); - } + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(2); + CheckBinary("u1", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("normal", 10); + CheckBinary("super", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); } TEST_F(DndTestUser, DropUser_01) { - ASSERT_NE(pClient, nullptr); - - //--- drop user --- SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); strcpy(pReq->user, "u1"); @@ -362,47 +252,10 @@ TEST_F(DndTestUser, DropUser_01) { ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - //--- meta --- - SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pShow->type = TSDB_MGMT_TABLE_USER; - SRpcMsg showRpcMsg = {0}; - showRpcMsg.pCont = pShow; - showRpcMsg.contLen = sizeof(SShowMsg); - showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &showRpcMsg); - SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; - STableMetaMsg* pMeta = &pShowRsp->tableMeta; - pMeta->numOfColumns = htons(pMeta->numOfColumns); - EXPECT_EQ(pMeta->numOfColumns, 4); - - //--- retrieve --- - SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pRetrieve->showId = pShowRsp->showId; - SRpcMsg retrieveRpcMsg = {0}; - retrieveRpcMsg.pCont = pRetrieve; - retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); - retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &retrieveRpcMsg); - SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; - pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - EXPECT_EQ(pRetrieveRsp->numOfRows, 2); - - char* pData = pRetrieveRsp->data; - int32_t pos = 0; - char* strVal = NULL; - - //--- name --- - { - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "root"); - - pos += sizeof(VarDataLenT); - strVal = (char*)(pData + pos); - pos += TSDB_USER_LEN; - EXPECT_STREQ(strVal, "_root"); - } -} \ No newline at end of file + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("super", 10); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); +} diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 2e4719cceb..2f582a810d 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -85,9 +85,11 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { return -1; } +#if 0 if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, "_" TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { return -1; } +#endif return 0; } @@ -469,7 +471,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -516,11 +518,8 @@ static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in if (pUser->superAuth) { const char *src = "super"; STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); - } else if (pUser->writeAuth) { - const char *src = "writable"; - STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); } else { - const char *src = "readable"; + const char *src = "normal"; STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); } cols++; diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 1d4888c2eb..ccff5b6c82 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -52,10 +52,10 @@ SSdb *sdbInit(SSdbOpt *pOption) { void sdbCleanup(SSdb *pSdb) { mDebug("start to cleanup sdb"); - if (pSdb->curVer != pSdb->lastCommitVer) { + // if (pSdb->curVer != pSdb->lastCommitVer) { mDebug("write sdb file for curVer:% " PRId64 " and lastVer:%" PRId64, pSdb->curVer, pSdb->lastCommitVer); sdbWriteFile(pSdb); - } + // } if (pSdb->currDir != NULL) { tfree(pSdb->currDir); From dc24592929c3dce6f8ae96585df25eb72f3d260d Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 13 Dec 2021 18:33:30 +0800 Subject: [PATCH 14/24] add wal read handle --- include/dnode/vnode/tq/tq.h | 2 +- include/libs/wal/wal.h | 46 +++++---- source/libs/wal/inc/walInt.h | 2 +- source/libs/wal/src/walMeta.c | 28 +++--- source/libs/wal/src/walMgmt.c | 52 +++++----- source/libs/wal/src/walRead.c | 143 ++++++++++++++++++++++++++- source/libs/wal/src/walSeek.c | 17 ++-- source/libs/wal/src/walUtil.c | 2 + source/libs/wal/src/walWrite.c | 108 ++++++++++---------- source/libs/wal/test/walMetaTest.cpp | 36 +++---- source/util/src/tarray.c | 2 +- 11 files changed, 292 insertions(+), 146 deletions(-) diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 85533f65bc..7359df92cc 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -256,7 +256,7 @@ typedef struct STQ { // the collection of group handle // the handle of kvstore char* path; - STqCfg* tqConfig; + STqCfg* tqConfig; TqLogReader* tqLogReader; TqMemRef tqMemRef; TqMetaStore* tqMeta; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 48c775292a..ae1e630c6f 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -44,7 +44,7 @@ typedef struct SWalReadHead { int8_t reserved[2]; int32_t len; int64_t version; - char cont[]; + char body[]; } SWalReadHead; typedef struct { @@ -52,9 +52,9 @@ typedef struct { int32_t fsyncPeriod; // millisecond int32_t retentionPeriod; // secs int32_t rollPeriod; // secs - int32_t retentionSize; // secs + int64_t retentionSize; int64_t segSize; - EWalType walLevel; // wal level + EWalType level; // wal level } SWalCfg; typedef struct { @@ -90,15 +90,17 @@ typedef struct { #define WAL_CUR_FILE_WRITABLE 2 #define WAL_CUR_FAILED 4 +typedef struct SWalVer { + int64_t firstVer; + int64_t verInSnapshotting; + int64_t snapshotVer; + int64_t commitVer; + int64_t lastVer; +} SWalVer; + typedef struct SWal { // cfg - int32_t vgId; - int32_t fsyncPeriod; // millisecond - int32_t rollPeriod; // second - int64_t segSize; - int64_t retentionSize; - int32_t retentionPeriod; - EWalType level; + SWalCfg cfg; //total size int64_t totSize; //fsync seq @@ -109,12 +111,7 @@ typedef struct SWal { int64_t writeLogTfd; int64_t writeIdxTfd; //wal lifecycle - int64_t firstVersion; - int64_t snapshotVersion; - int64_t commitVersion; - int64_t lastVersion; - //snapshotting version - int64_t snapshottingVer; + SWalVer vers; //roll status int64_t lastRollSeq; //file set @@ -126,9 +123,20 @@ typedef struct SWal { //path char path[WAL_PATH_LEN]; //reusable write head - SWalHead head; + SWalHead writeHead; } SWal; // WAL HANDLE +typedef struct SWalReadHandle { + SWal* pWal; + int64_t readLogTfd; + int64_t readIdxTfd; + int64_t curFileFirstVer; + int64_t curVersion; + int64_t capacity; + int64_t status; //if cursor valid + SWalHead head; +} SWalReadHandle; + typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); // module initialization @@ -154,6 +162,10 @@ int32_t walEndTakeSnapshot(SWal *); //int32_t walDataCorrupted(SWal*); // read +SWalReadHandle* walOpenReadHandle(SWal *); +void walCloseReadHandle(SWalReadHandle *); +int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); + int32_t walRead(SWal *, SWalHead **, int64_t ver); int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index f8f2e2eadd..ec01f7d7fc 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -90,7 +90,7 @@ static inline int walValidHeadCksum(SWalHead* pHead) { } static inline int walValidBodyCksum(SWalHead* pHead) { - return taosCheckChecksum((uint8_t*)pHead->head.cont, pHead->head.len, pHead->cksumBody); + return taosCheckChecksum((uint8_t*)pHead->head.body, pHead->head.len, pHead->cksumBody); } static inline int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 705cf051be..49f4fde3a0 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -25,15 +25,15 @@ #include int64_t walGetFirstVer(SWal *pWal) { - return pWal->firstVersion; + return pWal->vers.firstVer; } int64_t walGetSnaphostVer(SWal *pWal) { - return pWal->snapshotVersion; + return pWal->vers.snapshotVer; } int64_t walGetLastVer(SWal *pWal) { - return pWal->lastVersion; + return pWal->vers.lastVer; } int walRollFileInfo(SWal* pWal) { @@ -42,7 +42,7 @@ int walRollFileInfo(SWal* pWal) { SArray* pArray = pWal->fileInfoSet; if(taosArrayGetSize(pArray) != 0) { WalFileInfo *pInfo = taosArrayGetLast(pArray); - pInfo->lastVer = pWal->lastVersion; + pInfo->lastVer = pWal->vers.lastVer; pInfo->closeTs = ts; } @@ -51,7 +51,7 @@ int walRollFileInfo(SWal* pWal) { if(pNewInfo == NULL) { return -1; } - pNewInfo->firstVer = pWal->lastVersion + 1; + pNewInfo->firstVer = pWal->vers.lastVer + 1; pNewInfo->lastVer = -1; pNewInfo->createTs = ts; pNewInfo->closeTs = -1; @@ -74,13 +74,13 @@ char* walMetaSerialize(SWal* pWal) { return NULL; } cJSON_AddItemToObject(pRoot, "meta", pMeta); - sprintf(buf, "%" PRId64, pWal->firstVersion); + sprintf(buf, "%" PRId64, pWal->vers.firstVer); cJSON_AddStringToObject(pMeta, "firstVer", buf); - sprintf(buf, "%" PRId64, pWal->snapshotVersion); + sprintf(buf, "%" PRId64, pWal->vers.snapshotVer); cJSON_AddStringToObject(pMeta, "snapshotVer", buf); - sprintf(buf, "%" PRId64, pWal->commitVersion); + sprintf(buf, "%" PRId64, pWal->vers.commitVer); cJSON_AddStringToObject(pMeta, "commitVer", buf); - sprintf(buf, "%" PRId64, pWal->lastVersion); + sprintf(buf, "%" PRId64, pWal->vers.lastVer); cJSON_AddStringToObject(pMeta, "lastVer", buf); cJSON_AddItemToObject(pRoot, "files", pFiles); @@ -116,13 +116,13 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { pRoot = cJSON_Parse(bytes); pMeta = cJSON_GetObjectItem(pRoot, "meta"); pField = cJSON_GetObjectItem(pMeta, "firstVer"); - pWal->firstVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "snapshotVer"); - pWal->snapshotVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.snapshotVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "commitVer"); - pWal->commitVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.commitVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "lastVer"); - pWal->lastVersion = atoll(cJSON_GetStringValue(pField)); + pWal->vers.lastVer = atoll(cJSON_GetStringValue(pField)); pFiles = cJSON_GetObjectItem(pRoot, "files"); int sz = cJSON_GetArraySize(pFiles); @@ -161,7 +161,7 @@ static int walFindCurMetaVer(SWal* pWal) { DIR *dir = opendir(pWal->path); if(dir == NULL) { - wError("vgId:%d, path:%s, failed to open since %s", pWal->vgId, pWal->path, strerror(errno)); + wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return -1; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index a47f27f142..31f2ef037a 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -86,21 +86,15 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeCur = -1; //set config - pWal->vgId = pCfg->vgId; - pWal->fsyncPeriod = pCfg->fsyncPeriod; - pWal->rollPeriod = pCfg->rollPeriod; - pWal->segSize = pCfg->segSize; - pWal->retentionSize = pCfg->retentionSize; - pWal->retentionPeriod = pCfg->retentionPeriod; - pWal->level = pCfg->walLevel; + memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); //init version info - pWal->firstVersion = -1; - pWal->commitVersion = -1; - pWal->snapshotVersion = -1; - pWal->lastVersion = -1; + pWal->vers.firstVer = -1; + pWal->vers.commitVer = -1; + pWal->vers.snapshotVer = -1; + pWal->vers.lastVer = -1; - pWal->snapshottingVer = -1; + pWal->vers.verInSnapshotting = -1; pWal->totSize = 0; @@ -108,8 +102,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->lastRollSeq = -1; //init write buffer - memset(&pWal->head, 0, sizeof(SWalHead)); - pWal->head.head.sver = 0; + memset(&pWal->writeHead, 0, sizeof(SWalHead)); + pWal->writeHead.head.sver = 0; tstrncpy(pWal->path, path, sizeof(pWal->path)); pthread_mutex_init(&pWal->mutex, NULL); @@ -129,7 +123,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { } walReadMeta(pWal); - wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod); + wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); return pWal; } @@ -137,17 +131,17 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { if (pWal == NULL) return TSDB_CODE_WAL_APP_ERROR; - if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) { - wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->vgId, pWal->level, - pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); + if (pWal->cfg.level == pCfg->level && pWal->cfg.fsyncPeriod == pCfg->fsyncPeriod) { + wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->cfg.vgId, pWal->cfg.level, + pWal->cfg.fsyncPeriod, pCfg->level, pCfg->fsyncPeriod); return 0; } - wInfo("vgId:%d, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->vgId, pWal->level, - pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod); + wInfo("vgId:%d, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->cfg.vgId, pWal->cfg.level, + pWal->cfg.fsyncPeriod, pCfg->level, pCfg->fsyncPeriod); - pWal->level = pCfg->walLevel; - pWal->fsyncPeriod = pCfg->fsyncPeriod; + pWal->cfg.level = pCfg->level; + pWal->cfg.fsyncPeriod = pCfg->fsyncPeriod; pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; @@ -171,22 +165,22 @@ void walClose(SWal *pWal) { static int32_t walInitObj(SWal *pWal) { if (taosMkDir(pWal->path) != 0) { - wError("vgId:%d, path:%s, failed to create directory since %s", pWal->vgId, pWal->path, strerror(errno)); + wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo)); if(pWal->fileInfoSet == NULL) { - wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->vgId, pWal->path, strerror(errno)); + wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } - wDebug("vgId:%d, object is initialized", pWal->vgId); + wDebug("vgId:%d, object is initialized", pWal->cfg.vgId); return 0; } static void walFreeObj(void *wal) { SWal *pWal = wal; - wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal); + wDebug("vgId:%d, wal:%p is freed", pWal->cfg.vgId, pWal); tfClose(pWal->writeLogTfd); tfClose(pWal->writeIdxTfd); @@ -197,7 +191,7 @@ static void walFreeObj(void *wal) { } static bool walNeedFsync(SWal *pWal) { - if (pWal->fsyncPeriod <= 0 || pWal->level != TAOS_WAL_FSYNC) { + if (pWal->cfg.fsyncPeriod <= 0 || pWal->cfg.level != TAOS_WAL_FSYNC) { return false; } @@ -217,10 +211,10 @@ static void walFsyncAll() { SWal *pWal = taosIterateRef(tsWal.refSetId, 0); while (pWal) { if (walNeedFsync(pWal)) { - wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); + wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); int32_t code = tfFsync(pWal->writeLogTfd); if (code != 0) { - wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(code)); + wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code)); } } pWal = taosIterateRef(tsWal.refSetId, pWal->refId); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 27dffddf83..554a5c846b 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -16,6 +16,147 @@ #include "walInt.h" #include "tfile.h" +SWalReadHandle* walOpenReadHandle(SWal* pWal) { + SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle)); + if(pRead == NULL) { + return NULL; + } + memset(pRead, 0, sizeof(SWalReadHandle)); + pRead->pWal = pWal; + pRead->readIdxTfd = -1; + pRead->readLogTfd = -1; + return NULL; +} + +void walCloseReadHandle(SWalReadHandle *pRead) { + tfClose(pRead->readIdxTfd); + tfClose(pRead->readLogTfd); + free(pRead); +} + +int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { + return 0; +} + +static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { + int code = 0; + + int64_t idxTfd = pRead->readIdxTfd; + int64_t logTfd = pRead->readLogTfd; + + //seek position + int64_t offset = (ver - fileFirstVer) * WAL_IDX_ENTRY_SIZE; + code = tfLseek(idxTfd, offset, SEEK_SET); + if(code != 0) { + return -1; + } + WalIdxEntry entry; + code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry)); + if(code != 0) { + return -1; + } + //TODO:deserialize + ASSERT(entry.ver == ver); + code = tfLseek(logTfd, entry.offset, SEEK_SET); + if (code != 0) { + return -1; + } + return code; +} + +static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { + char fnameStr[WAL_FILE_LEN]; + + tfClose(pRead->readIdxTfd); + tfClose(pRead->readLogTfd); + + walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); + int logTfd = tfOpenRead(fnameStr); + if(logTfd < 0) { + return -1; + } + + walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); + int idxTfd = tfOpenRead(fnameStr); + if(idxTfd < 0) { + return -1; + } + + pRead->readLogTfd = logTfd; + pRead->readIdxTfd = idxTfd; + return 0; +} + +static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { + int code; + SWal *pWal = pRead->pWal; + if(ver == pWal->vers.lastVer) { + return 0; + } + if(ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { + return -1; + } + if(ver < pWal->vers.snapshotVer) { + + } + + WalFileInfo tmpInfo; + tmpInfo.firstVer = ver; + //bsearch in fileSet + WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + ASSERT(pRet != NULL); + if(pRead->curFileFirstVer != pRet->firstVer) { + code = walReadChangeFile(pRead, pRet->firstVer); + if(code < 0) { + //TODO: set error flag + return -1; + } + } + + code = walReadSeekFilePos(pRead, pRet->firstVer, ver); + if(code < 0) { + return -1; + } + pRead->curVersion = ver; + + return 0; +} + +int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { + int code; + //TODO: check wal life + if(pRead->curVersion != ver) { + walReadSeekVer(pRead, ver); + } + + if(!tfValid(pRead->readLogTfd)) return -1; + + if(sizeof(SWalHead) != tfRead(pRead->readLogTfd, &pRead->head, sizeof(SWalHead))) { + return -1; + } + code = walValidHeadCksum(&pRead->head); + if(code != 0) { + return -1; + } + if(pRead->capacity < pRead->head.head.len) { + void* ptr = realloc(pRead, pRead->head.head.len); + if(ptr == NULL) { + return -1; + } + pRead = ptr; + pRead->capacity = pRead->head.head.len; + } + if(pRead->head.head.len != tfRead(pRead->readLogTfd, &pRead->head.head.body, pRead->head.head.len)) { + return -1; + } + code = walValidBodyCksum(&pRead->head); + if(code != 0) { + return -1; + } + + return 0; +} + int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { int code; code = walSeekVer(pWal, ver); @@ -42,7 +183,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { *ppHead = NULL; return -1; } - if(tfRead(pWal->writeLogTfd, (*ppHead)->head.cont, (*ppHead)->head.len) != (*ppHead)->head.len) { + if(tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } //TODO: endian compatibility processing after read diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 48272f8f32..953aae703c 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -78,10 +78,12 @@ int walChangeFile(SWal *pWal, int64_t ver) { code = tfClose(pWal->writeLogTfd); if(code != 0) { //TODO + return -1; } code = tfClose(pWal->writeIdxTfd); if(code != 0) { //TODO + return -1; } WalFileInfo tmpInfo; tmpInfo.firstVer = ver; @@ -106,24 +108,19 @@ int walChangeFile(SWal *pWal, int64_t ver) { pWal->writeLogTfd = logTfd; pWal->writeIdxTfd = idxTfd; - return code; -} - -int walGetVerOffset(SWal* pWal, int64_t ver) { - int code; - return 0; + return fileFirstVer; } int walSeekVer(SWal *pWal, int64_t ver) { int code; - if(ver == pWal->lastVersion) { + if(ver == pWal->vers.lastVer) { return 0; } - if(ver > pWal->lastVersion || ver < pWal->firstVersion) { + if(ver > pWal->vers.lastVer|| ver < pWal->vers.firstVer) { return -1; } - if(ver < pWal->snapshotVersion) { - //TODO: set flag to prevent roll back + if(ver < pWal->vers.snapshotVer) { + } if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { code = walChangeFile(pWal, ver); diff --git a/source/libs/wal/src/walUtil.c b/source/libs/wal/src/walUtil.c index c88cc918fe..849d0c3e51 100644 --- a/source/libs/wal/src/walUtil.c +++ b/source/libs/wal/src/walUtil.c @@ -17,6 +17,7 @@ #include "os.h" #include "walInt.h" +#if 0 int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId) { int64_t curFileId = *nextFileId; int64_t minFileId = INT64_MAX; @@ -116,3 +117,4 @@ int32_t walGetNewFile(SWal *pWal, int64_t *newFileId) { return 0; } +#endif diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 972ac5c682..44e8cec153 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -114,22 +114,22 @@ void walRemoveAllOldFiles(void *handle) { #endif int32_t walCommit(SWal *pWal, int64_t ver) { - ASSERT(pWal->commitVersion >= pWal->snapshotVersion); - ASSERT(pWal->commitVersion <= pWal->lastVersion); - if(ver < pWal->commitVersion || ver > pWal->lastVersion) { + ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer); + ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer); + if(ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) { return -1; } - pWal->commitVersion = ver; + pWal->vers.commitVer = ver; return 0; } int32_t walRollback(SWal *pWal, int64_t ver) { int code; char fnameStr[WAL_FILE_LEN]; - if(ver == pWal->lastVersion) { + if(ver == pWal->vers.lastVer) { return 0; } - if(ver > pWal->lastVersion || ver < pWal->commitVersion) { + if(ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { return -1; } pthread_mutex_lock(&pWal->mutex); @@ -220,7 +220,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { if(code < 0) { return -1; } - pWal->lastVersion = ver - 1; + pWal->vers.lastVer = ver - 1; ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; @@ -230,9 +230,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } int32_t walBeginTakeSnapshot(SWal* pWal, int64_t ver) { - pWal->snapshottingVer = ver; + pWal->vers.verInSnapshotting = ver; //check file rolling - if(pWal->retentionPeriod == 0) { + if(pWal->cfg.retentionPeriod == 0) { walRoll(pWal); } @@ -240,10 +240,10 @@ int32_t walBeginTakeSnapshot(SWal* pWal, int64_t ver) { } int32_t walEndTakeSnapshot(SWal *pWal) { - int64_t ver = pWal->snapshottingVer; + int64_t ver = pWal->vers.verInSnapshotting; if(ver == -1) return -1; - pWal->snapshotVersion = ver; + pWal->vers.snapshotVer = ver; int ts = taosGetTimestampSec(); int deleteCnt = 0; @@ -257,8 +257,8 @@ int32_t walEndTakeSnapshot(SWal *pWal) { } //iterate files, until the searched result for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { - if(pWal->totSize > pWal->retentionSize || - iter->closeTs + pWal->retentionPeriod > ts) { + if(pWal->totSize > pWal->cfg.retentionSize || + iter->closeTs + pWal->cfg.retentionPeriod > ts) { //delete according to file size or close time deleteCnt++; newTotSize -= iter->fileSize; @@ -278,13 +278,13 @@ int32_t walEndTakeSnapshot(SWal *pWal) { taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); if(taosArrayGetSize(pWal->fileInfoSet) == 0) { pWal->writeCur = -1; - pWal->firstVersion = -1; + pWal->vers.firstVer = -1; } else { - pWal->firstVersion = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + pWal->vers.firstVer = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; } pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;; pWal->totSize = newTotSize; - pWal->snapshottingVer = -1; + pWal->vers.verInSnapshotting = -1; //save snapshot ver, commit ver int code = walWriteMeta(pWal); @@ -311,7 +311,7 @@ int walRoll(SWal *pWal) { } int64_t idxTfd, logTfd; //create new file - int64_t newFileFirstVersion = pWal->lastVersion + 1; + int64_t newFileFirstVersion = pWal->vers.lastVer + 1; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVersion, fnameStr); idxTfd = tfOpenCreateWrite(fnameStr); @@ -357,18 +357,18 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i int code = 0; // no wal - if (pWal->level == TAOS_WAL_NOLOG) return 0; + if (pWal->cfg.level == TAOS_WAL_NOLOG) return 0; - if (index == pWal->lastVersion + 1) { + if (index == pWal->vers.lastVer + 1) { if(taosArrayGetSize(pWal->fileInfoSet) == 0) { - pWal->firstVersion = index; + pWal->vers.firstVer = index; code = walRoll(pWal); ASSERT(code == 0); } else { int64_t passed = walGetSeq() - pWal->lastRollSeq; - if(pWal->rollPeriod != -1 && pWal->rollPeriod != 0 && passed > pWal->rollPeriod) { + if(pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { walRoll(pWal); - } else if(pWal->segSize != -1 && pWal->segSize != 0 && walGetLastFileSize(pWal) > pWal->segSize) { + } else if(pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { walRoll(pWal); } } @@ -380,23 +380,23 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i /*if (!tfValid(pWal->curLogTfd)) return 0;*/ pthread_mutex_lock(&pWal->mutex); - pWal->head.head.version = index; + pWal->writeHead.head.version = index; - pWal->head.head.len = bodyLen; - pWal->head.head.msgType = msgType; - pWal->head.cksumHead = walCalcHeadCksum(&pWal->head); - pWal->head.cksumBody = walCalcBodyCksum(body, bodyLen); + pWal->writeHead.head.len = bodyLen; + pWal->writeHead.head.msgType = msgType; + pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); + pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); - if (tfWrite(pWal->writeLogTfd, &pWal->head, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { //ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } if (tfWrite(pWal->writeLogTfd, &body, bodyLen) != bodyLen) { //ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } code = walWriteIndex(pWal, index, walGetCurFileOffset(pWal)); if(code != 0) { @@ -405,7 +405,7 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i } //set status - pWal->lastVersion = index; + pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalHead) + bodyLen; walGetCurFileInfo(pWal)->lastVer = index; walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen; @@ -416,10 +416,10 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i } void walFsync(SWal *pWal, bool forceFsync) { - if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { - wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->vgId, walGetCurFileFirstVer(pWal)); + if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { + wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (tfFsync(pWal->writeLogTfd) < 0) { - wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->vgId, walGetCurFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); } } } @@ -492,29 +492,29 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { } #endif -static int walValidateOffset(SWal* pWal, int64_t ver) { - int code = 0; - SWalHead *pHead = NULL; - code = (int)walRead(pWal, &pHead, ver); - if(pHead->head.version != ver) { - return -1; - } - return 0; -} +/*static int walValidateOffset(SWal* pWal, int64_t ver) {*/ + /*int code = 0;*/ + /*SWalHead *pHead = NULL;*/ + /*code = (int)walRead(pWal, &pHead, ver);*/ + /*if(pHead->head.version != ver) {*/ + /*return -1;*/ + /*}*/ + /*return 0;*/ +/*}*/ -static int64_t walGetOffset(SWal* pWal, int64_t ver) { - int code = walSeekVer(pWal, ver); - if(code != 0) { - return -1; - } +/*static int64_t walGetOffset(SWal* pWal, int64_t ver) {*/ + /*int code = walSeekVer(pWal, ver);*/ + /*if(code != 0) {*/ + /*return -1;*/ + /*}*/ - code = walValidateOffset(pWal, ver); - if(code != 0) { - return -1; - } + /*code = walValidateOffset(pWal, ver);*/ + /*if(code != 0) {*/ + /*return -1;*/ + /*}*/ - return 0; -} + /*return 0;*/ +/*}*/ #if 0 static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, int64_t *offset) { diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index c81343d03b..504f1ada3f 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -24,7 +24,7 @@ class WalCleanEnv : public ::testing::Test { pCfg->segSize = -1; pCfg->retentionPeriod = 0; pCfg->retentionSize = 0; - pCfg->walLevel = TAOS_WAL_FSYNC; + pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); free(pCfg); ASSERT(pWal != NULL); @@ -56,7 +56,7 @@ class WalCleanDeleteEnv : public ::testing::Test { memset(pCfg, 0, sizeof(SWalCfg)); pCfg->retentionPeriod = 0; pCfg->retentionSize = 0; - pCfg->walLevel = TAOS_WAL_FSYNC; + pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); free(pCfg); ASSERT(pWal != NULL); @@ -95,7 +95,7 @@ class WalKeepEnv : public ::testing::Test { pCfg->segSize = -1; pCfg->retentionPeriod = 0; pCfg->retentionSize = 0; - pCfg->walLevel = TAOS_WAL_FSYNC; + pCfg->level = TAOS_WAL_FSYNC; pWal = walOpen(pathName, pCfg); free(pCfg); ASSERT(pWal != NULL); @@ -164,18 +164,18 @@ TEST_F(WalKeepEnv, readOldMeta) { for(int i = 0; i < 10; i++) { code = walWrite(pWal, i, i+1, (void*)ranStr, len); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->lastVersion, i); + ASSERT_EQ(pWal->vers.lastVer, i); code = walWrite(pWal, i+2, i, (void*)ranStr, len); ASSERT_EQ(code, -1); - ASSERT_EQ(pWal->lastVersion, i); + ASSERT_EQ(pWal->vers.lastVer, i); } char* oldss = walMetaSerialize(pWal); TearDown(); SetUp(); - ASSERT_EQ(pWal->firstVersion, 0); - ASSERT_EQ(pWal->lastVersion, 9); + ASSERT_EQ(pWal->vers.firstVer, 0); + ASSERT_EQ(pWal->vers.lastVer, 9); char* newss = walMetaSerialize(pWal); @@ -195,10 +195,10 @@ TEST_F(WalCleanEnv, write) { for(int i = 0; i < 10; i++) { code = walWrite(pWal, i, i+1, (void*)ranStr, len); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->lastVersion, i); + ASSERT_EQ(pWal->vers.lastVer, i); code = walWrite(pWal, i+2, i, (void*)ranStr, len); ASSERT_EQ(code, -1); - ASSERT_EQ(pWal->lastVersion, i); + ASSERT_EQ(pWal->vers.lastVer, i); } code = walWriteMeta(pWal); ASSERT_EQ(code, 0); @@ -211,14 +211,14 @@ TEST_F(WalCleanEnv, rollback) { for(int i = 0; i < 10; i++) { code = walWrite(pWal, i, i+1, (void*)ranStr, len); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->lastVersion, i); + ASSERT_EQ(pWal->vers.lastVer, i); } code = walRollback(pWal, 5); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->lastVersion, 4); + ASSERT_EQ(pWal->vers.lastVer, 4); code = walRollback(pWal, 3); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->lastVersion, 2); + ASSERT_EQ(pWal->vers.lastVer, 2); code = walWriteMeta(pWal); ASSERT_EQ(code, 0); } @@ -231,16 +231,16 @@ TEST_F(WalCleanDeleteEnv, roll) { for(i = 0; i < 100; i++) { code = walWrite(pWal, i, 0, (void*)ranStr, len); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->lastVersion, i); + ASSERT_EQ(pWal->vers.lastVer, i); code = walCommit(pWal, i); - ASSERT_EQ(pWal->commitVersion, i); + ASSERT_EQ(pWal->vers.commitVer, i); } walBeginTakeSnapshot(pWal, i-1); - ASSERT_EQ(pWal->snapshottingVer, i-1); + ASSERT_EQ(pWal->vers.verInSnapshotting, i-1); walEndTakeSnapshot(pWal); - ASSERT_EQ(pWal->snapshotVersion, i-1); - ASSERT_EQ(pWal->snapshottingVer, -1); + ASSERT_EQ(pWal->vers.snapshotVer, i-1); + ASSERT_EQ(pWal->vers.verInSnapshotting, -1); code = walWrite(pWal, 5, 0, (void*)ranStr, len); ASSERT_NE(code, 0); @@ -249,7 +249,7 @@ TEST_F(WalCleanDeleteEnv, roll) { code = walWrite(pWal, i, 0, (void*)ranStr, len); ASSERT_EQ(code, 0); code = walCommit(pWal, i); - ASSERT_EQ(pWal->commitVersion, i); + ASSERT_EQ(pWal->vers.commitVer, i); } //code = walWriteMeta(pWal); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 6216188496..581a797343 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -250,7 +250,7 @@ void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { if(pArray->size == 0) { return; } - memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size); + memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size * pArray->elemSize); } void taosArrayPopTailBatch(SArray* pArray, size_t cnt) { From 43f600e01b3eff3f3cf955e3a8817daca6255c74 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 13 Dec 2021 19:02:26 +0800 Subject: [PATCH 15/24] minor changes --- source/dnode/mgmt/impl/test/user/user.cpp | 82 +++++++++++++++++++---- 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 137e271b32..48be2635cd 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -185,28 +185,50 @@ TEST_F(DndTestUser, ShowUser) { } TEST_F(DndTestUser, CreateUser_01) { - SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); - strcpy(pReq->user, "u1"); - strcpy(pReq->pass, "p1"); + { + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p1"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + { + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + strcpy(pReq->user, "u2"); + strcpy(pReq->pass, "p2"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(2); + SendThenCheckShowRetrieveMsg(3); CheckBinary("u1", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); CheckBinary("normal", 10); CheckBinary("super", 10); + CheckBinary("normal", 10); CheckTimestamp(); CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); } @@ -227,13 +249,17 @@ TEST_F(DndTestUser, AlterUser_01) { ASSERT_EQ(pMsg->code, 0); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(2); + SendThenCheckShowRetrieveMsg(3); CheckBinary("u1", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); CheckBinary("normal", 10); CheckBinary("super", 10); + CheckBinary("normal", 10); CheckTimestamp(); CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); } @@ -253,9 +279,37 @@ TEST_F(DndTestUser, DropUser_01) { ASSERT_EQ(pMsg->code, 0); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); - SendThenCheckShowRetrieveMsg(1); + SendThenCheckShowRetrieveMsg(2); CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); CheckTimestamp(); CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); +} + +TEST_F(DndTestUser, RestartDnode) { + stopServer(pServer); + pServer = NULL; + + uInfo("start all server"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9530"; + pServer = startServer("/tmp/dndTestUser", fqdn, 9530, firstEp); + + uInfo("all server is running"); + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); + SendThenCheckShowRetrieveMsg(2); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("u2", TSDB_USER_LEN); + CheckBinary("super", 10); + CheckBinary("normal", 10); + CheckTimestamp(); + CheckTimestamp(); + CheckBinary("root", TSDB_USER_LEN); + CheckBinary("root", TSDB_USER_LEN); } From 04c83f497d52dfb4386ac517c6f45701668dad54 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 20:41:03 +0800 Subject: [PATCH 16/24] more --- include/util/tdlist.h | 39 +++++++------------ .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 6 ++- source/dnode/vnode/impl/src/vnodeBufferPool.c | 9 +++-- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 2 +- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 757b404ad6..0f341801e0 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -65,30 +65,21 @@ extern "C" { } \ (l)->neles_ += 1; -#define tlistPop(l, n) \ - ({ \ - if ((n)) { \ - if ((l)->head_ == (n)) { \ - (l)->head_ = (n)->next_; \ - } \ - if ((l)->tail_ == (n)) { \ - (l)->tail_ = (n)->prev_; \ - } \ - if ((n)->prev_ != NULL) { \ - (n)->prev_->next_ = (n)->next_; \ - } \ - if ((n)->next_ != NULL) { \ - (n)->next_->prev_ = (n)->prev_; \ - } \ - (l)->neles_ -= 1; \ - (n)->prev_ = (n)->next_ = NULL; \ - } \ - (n); \ - }) - -#define tlistPopHead(l) tlistPop(l, (l)->head_) - -#define tlistPopTail(l) tlistPop(l, (l)->tail_) +#define tlistPop(l, n) \ + if ((l)->head_ == (n)) { \ + (l)->head_ = (n)->next_; \ + } \ + if ((l)->tail_ == (n)) { \ + (l)->tail_ = (n)->prev_; \ + } \ + if ((n)->prev_ != NULL) { \ + (n)->prev_->next_ = (n)->next_; \ + } \ + if ((n)->next_ != NULL) { \ + (n)->next_->prev_ = (n)->prev_; \ + } \ + (l)->neles_ -= 1; \ + (n)->prev_ = (n)->next_ = NULL; // List iterator #define TD_LIST_FITER 0 diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index 532c2dad8e..748808a9fd 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -43,9 +43,10 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { void vmaDestroy(SVMemAllocator *pVMA) { if (pVMA) { while (true) { - SVArenaNode *pNode = tlistPopTail(&(pVMA->nlist)); + SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); if (pNode) { + tlistPop(&(pVMA->nlist), pNode); vArenaNodeFree(pNode); } else { break; @@ -58,7 +59,8 @@ void vmaDestroy(SVMemAllocator *pVMA) { void vmaReset(SVMemAllocator *pVMA) { while (tlistNEles(&(pVMA->nlist)) > 1) { - SVArenaNode *pNode = tlistPopTail(&(pVMA->nlist)); + SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + tlistPop(&(pVMA->nlist), pNode); vArenaNodeFree(pNode); } diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index fe15e9ebff..d5fcdf91e3 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -61,14 +61,16 @@ void vnodeCloseBufPool(SVnode *pVnode) { vmaDestroy(pVnode->pBufPool->inuse); while (true) { - SVMemAllocator *pVMA = tlistPopHead(&(pVnode->pBufPool->incycle)); + SVMemAllocator *pVMA = tlistHead(&(pVnode->pBufPool->incycle)); if (pVMA == NULL) break; + tlistPop(&(pVnode->pBufPool->incycle), pVMA); vmaDestroy(pVMA); } while (true) { - SVMemAllocator *pVMA = tlistPopHead(&(pVnode->pBufPool->free)); + SVMemAllocator *pVMA = tlistHead(&(pVnode->pBufPool->free)); if (pVMA == NULL) break; + tlistPop(&(pVnode->pBufPool->free), pVMA); vmaDestroy(pVMA); } @@ -83,8 +85,9 @@ void *vnodeMalloc(SVnode *pVnode, uint64_t size) { if (pBufPool->inuse == NULL) { while (true) { // TODO: add sem_wait and sem_post - pBufPool->inuse = tlistPopHead(&(pBufPool->free)); + pBufPool->inuse = tlistHead(&(pBufPool->free)); if (pBufPool->inuse) { + tlistPop(&(pBufPool->free), pBufPool->inuse); break; } } diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index ac2ccbc132..ec7e40dc0d 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -131,7 +131,7 @@ TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { { // Create some child tables - int ntables = 1000000; + int ntables = 1000; int batch = 10; for (int i = 0; i < ntables / batch; i++) { SArray *pMsgs = (SArray *)taosArrayInit(batch, sizeof(SRpcMsg *)); From 2c053411122fa037578f2c4bf78015d3f11a2990 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 20:44:55 +0800 Subject: [PATCH 17/24] more --- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index ec7e40dc0d..ac2ccbc132 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -131,7 +131,7 @@ TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { { // Create some child tables - int ntables = 1000; + int ntables = 1000000; int batch = 10; for (int i = 0; i < ntables / batch; i++) { SArray *pMsgs = (SArray *)taosArrayInit(batch, sizeof(SRpcMsg *)); From d66cba75e1e4aba185cb69629b6d564fe5bf696e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 21:06:33 +0800 Subject: [PATCH 18/24] more --- include/util/tdlist.h | 49 +++++++++++++------ .../dnode/vnode/impl/inc/vnodeMemAllocator.h | 6 +-- source/dnode/vnode/impl/src/vnodeBufferPool.c | 4 +- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 0f341801e0..4be1779c74 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -20,17 +20,36 @@ extern "C" { #endif -#define TD_LIST_NODE(S) \ - struct { \ - S *prev_; \ - S *next_; \ +// Single linked list +#define TD_SLIST_NODE(TYPE) \ + struct { \ + struct type *sl_next_; \ } -#define TD_LIST(S) \ - struct { \ - S * head_; \ - S * tail_; \ - int neles_; \ +#define TD_SLIST(TYPE) \ + struct { \ + struct TYPE *sl_head_; \ + } + +#define TD_SLIST_NODE_NEXT(sln) (sln)->sl_next_ + +#define tSListInit(sl) \ + do { \ + (sl)->sl_head_ = NULL; \ + } while (0) + +// Double linked list +#define TD_DLIST_NODE(S) \ + struct { \ + S *prev_; \ + S *next_; \ + } + +#define TD_DLIST(S) \ + struct { \ + S * head_; \ + S * tail_; \ + int neles_; \ } #define tlistInit(l) \ @@ -84,12 +103,12 @@ extern "C" { // List iterator #define TD_LIST_FITER 0 #define TD_LIST_BITER 1 -#define TD_LIST_ITER(S) \ - struct { \ - int it_dir_; \ - S * it_next_; \ - S * it_ptr_; \ - TD_LIST(S) * it_list_; \ +#define TD_LIST_ITER(S) \ + struct { \ + int it_dir_; \ + S * it_next_; \ + S * it_ptr_; \ + TD_DLIST(S) * it_list_; \ } #define tlistIterInit(it, l, dir) \ diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index df8b367d25..6b9fddaa02 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -26,18 +26,18 @@ typedef struct SVArenaNode SVArenaNode; typedef struct SVMemAllocator SVMemAllocator; struct SVArenaNode { - TD_LIST_NODE(SVArenaNode); + TD_DLIST_NODE(SVArenaNode); uint64_t size; // current node size void * ptr; char data[]; }; struct SVMemAllocator { - TD_LIST_NODE(SVMemAllocator); + TD_DLIST_NODE(SVMemAllocator); uint64_t capacity; uint64_t ssize; uint64_t lsize; - TD_LIST(SVArenaNode) nlist; + TD_DLIST(SVArenaNode) nlist; }; SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize); diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index d5fcdf91e3..347fe44e26 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -19,8 +19,8 @@ #define VNODE_BUF_POOL_SHARDS 3 struct SVBufPool { - TD_LIST(SVMemAllocator) free; - TD_LIST(SVMemAllocator) incycle; + TD_DLIST(SVMemAllocator) free; + TD_DLIST(SVMemAllocator) incycle; SVMemAllocator *inuse; // MAF for submodules // SMemAllocatorFactory maf; From aaf9ca18c51ade1843dee8ab46fd38dfb8cf4ec9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 21:18:43 +0800 Subject: [PATCH 19/24] more --- include/util/tdlist.h | 29 ++++++++++++------- .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 2 +- source/dnode/vnode/impl/src/vnodeBufferPool.c | 4 +-- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 4be1779c74..2be9ba3895 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -31,28 +31,35 @@ extern "C" { struct TYPE *sl_head_; \ } -#define TD_SLIST_NODE_NEXT(sln) (sln)->sl_next_ +#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) #define tSListInit(sl) \ do { \ (sl)->sl_head_ = NULL; \ } while (0) +#define tSListPrepend(sl, sln) \ + do { \ + TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ + TD_SLIST_HEAD(sl) = (sln); \ + } while (0); + // Double linked list -#define TD_DLIST_NODE(S) \ - struct { \ - S *prev_; \ - S *next_; \ +#define TD_DLIST_NODE(TYPE) \ + struct { \ + TYPE *prev_; \ + TYPE *next_; \ } -#define TD_DLIST(S) \ - struct { \ - S * head_; \ - S * tail_; \ - int neles_; \ +#define TD_DLIST(TYPE) \ + struct { \ + TYPE *head_; \ + TYPE *tail_; \ + int neles_; \ } -#define tlistInit(l) \ +#define tDListInit(l) \ (l)->head_ = (l)->tail_ = NULL; \ (l)->neles_ = 0; diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index 748808a9fd..b5424894ca 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -27,7 +27,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { pVMA->capacity = capacity; pVMA->ssize = ssize; pVMA->lsize = lsize; - tlistInit(&(pVMA->nlist)); + tDListInit(&(pVMA->nlist)); SVArenaNode *pNode = vArenaNodeNew(capacity); if (pNode == NULL) { diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 347fe44e26..084a7d18a7 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -35,8 +35,8 @@ int vnodeOpenBufPool(SVnode *pVnode) { return -1; } - tlistInit(&(pVnode->pBufPool->free)); - tlistInit(&(pVnode->pBufPool->incycle)); + tDListInit(&(pVnode->pBufPool->free)); + tDListInit(&(pVnode->pBufPool->incycle)); pVnode->pBufPool->inuse = NULL; From c0e308f275ddcd22ae8a1346823a95b8baec3c66 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 21:50:34 +0800 Subject: [PATCH 20/24] more --- include/util/tdlist.h | 154 ++++++++++-------- .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 23 +-- source/dnode/vnode/impl/src/vnodeBufferPool.c | 14 +- 3 files changed, 107 insertions(+), 84 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 2be9ba3895..8360d2713f 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -26,87 +26,108 @@ extern "C" { struct type *sl_next_; \ } -#define TD_SLIST(TYPE) \ - struct { \ - struct TYPE *sl_head_; \ +#define TD_SLIST(TYPE) \ + struct { \ + struct TYPE *sl_dl_head_; \ + int sl_dl_neles_; \ } -#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_HEAD(sl) ((sl)->sl_dl_head_) +#define TD_SLIST_NELES(sl) ((sl)->sl_dl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) -#define tSListInit(sl) \ - do { \ - (sl)->sl_head_ = NULL; \ +#define tSListInit(sl) \ + do { \ + (sl)->sl_dl_head_ = NULL; \ + (sl)->sl_dl_neles_ = 0; \ } while (0) -#define tSListPrepend(sl, sln) \ +#define tSListPush(sl, sln) \ do { \ TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ TD_SLIST_HEAD(sl) = (sln); \ - } while (0); + TD_SLIST_NELES(sl) += 1; \ + } while (0) + +#define tSListPop(sl) \ + do { \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ + TD_SLIST_NELES(sl) -= 1; \ + } while (0) // Double linked list #define TD_DLIST_NODE(TYPE) \ struct { \ - TYPE *prev_; \ - TYPE *next_; \ + TYPE *dl_prev_; \ + TYPE *dl_next_; \ } -#define TD_DLIST(TYPE) \ - struct { \ - TYPE *head_; \ - TYPE *tail_; \ - int neles_; \ +#define TD_DLIST(TYPE) \ + struct { \ + struct TYPE *dl_head_; \ + struct TYPE *dl_tail_; \ + int dl_neles_; \ } -#define tDListInit(l) \ - (l)->head_ = (l)->tail_ = NULL; \ - (l)->neles_ = 0; +#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) +#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) +#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) +#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) -#define tlistHead(l) (l)->head_ -#define tlistTail(l) (l)->tail_ -#define tlistNEles(l) (l)->neles_ +#define tDListInit(dl) \ + do { \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \ + TD_DLIST_NELES(dl) = 0; \ + } while (0) -#define tlistAppend(l, n) \ - if ((l)->head_ == NULL) { \ - (n)->prev_ = (n)->next_ = NULL; \ - (l)->head_ = (l)->tail_ = (n); \ - } else { \ - (n)->prev_ = (l)->tail_; \ - (n)->next_ = NULL; \ - (l)->tail_->next_ = (n); \ - (l)->tail_ = (n); \ - } \ - (l)->neles_ += 1; +#define tDListAppend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln); \ + TD_DLIST_TAIL(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) -#define tlistPrepend(l, n) \ - if ((l)->head_ == NULL) { \ - (n)->prev_ = (n)->next_ = NULL; \ - (l)->head_ = (l)->tail_ = (n); \ - } else { \ - (n)->prev_ = NULL; \ - (n)->next_ = (l)->head_; \ - (l)->head_->prev_ = (n); \ - (l)->head_ = (n); \ - } \ - (l)->neles_ += 1; +#define tDListPrepend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = NULL; \ + TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln); \ + TD_DLIST_HEAD(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) -#define tlistPop(l, n) \ - if ((l)->head_ == (n)) { \ - (l)->head_ = (n)->next_; \ - } \ - if ((l)->tail_ == (n)) { \ - (l)->tail_ = (n)->prev_; \ - } \ - if ((n)->prev_ != NULL) { \ - (n)->prev_->next_ = (n)->next_; \ - } \ - if ((n)->next_ != NULL) { \ - (n)->next_->prev_ = (n)->prev_; \ - } \ - (l)->neles_ -= 1; \ - (n)->prev_ = (n)->next_ = NULL; +#define tDListPop(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == (dln)) { \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_TAIL(dl) == (dln)) { \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln); \ + } \ + if (TD_DLIST_NODE_PREV(dln) != NULL) { \ + TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_NODE_NEXT(dln) != NULL) { \ + TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \ + } \ + TD_DLIST_NELES(dl) -= 1; \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + } while (0) +#if 0 // List iterator #define TD_LIST_FITER 0 #define TD_LIST_BITER 1 @@ -118,13 +139,13 @@ extern "C" { TD_DLIST(S) * it_list_; \ } -#define tlistIterInit(it, l, dir) \ - (it)->it_dir_ = (dir); \ - (it)->it_list_ = l; \ - if ((dir) == TD_LIST_FITER) { \ - (it)->it_next_ = (l)->head_; \ - } else { \ - (it)->it_next_ = (l)->tail_; \ +#define tlistIterInit(it, l, dir) \ + (it)->it_dir_ = (dir); \ + (it)->it_list_ = l; \ + if ((dir) == TD_LIST_FITER) { \ + (it)->it_next_ = (l)->dl_head_; \ + } else { \ + (it)->it_next_ = (l)->dl_tail_; \ } #define tlistIterNext(it) \ @@ -139,6 +160,7 @@ extern "C" { } \ (it)->it_ptr_; \ }) +#endif #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index b5424894ca..5d2b404314 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -35,7 +35,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { return NULL; } - tlistAppend(&(pVMA->nlist), pNode); + tDListAppend(&(pVMA->nlist), pNode); return pVMA; } @@ -43,10 +43,10 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { void vmaDestroy(SVMemAllocator *pVMA) { if (pVMA) { while (true) { - SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); if (pNode) { - tlistPop(&(pVMA->nlist), pNode); + tDListPop(&(pVMA->nlist), pNode); vArenaNodeFree(pNode); } else { break; @@ -58,18 +58,18 @@ void vmaDestroy(SVMemAllocator *pVMA) { } void vmaReset(SVMemAllocator *pVMA) { - while (tlistNEles(&(pVMA->nlist)) > 1) { - SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); - tlistPop(&(pVMA->nlist), pNode); + while (TD_DLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); + tDListPop(&(pVMA->nlist), pNode); vArenaNodeFree(pNode); } - SVArenaNode *pNode = tlistHead(&(pVMA->nlist)); + SVArenaNode *pNode = TD_DLIST_HEAD(&(pVMA->nlist)); pNode->ptr = pNode->data; } void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { - SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); void * ptr; if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { @@ -80,7 +80,7 @@ void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { return NULL; } - tlistAppend(&(pVMA->nlist), pNode); + tDListAppend(&(pVMA->nlist), pNode); } ptr = pNode->ptr; @@ -94,9 +94,10 @@ void vmaFree(SVMemAllocator *pVMA, void *ptr) { } bool vmaIsFull(SVMemAllocator *pVMA) { - SVArenaNode *pNode = tlistTail(&(pVMA->nlist)); + SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); - return (tlistNEles(&(pVMA->nlist)) > 1) || (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); + return (TD_DLIST_NELES(&(pVMA->nlist)) > 1) || + (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); } /* ------------------------ STATIC METHODS ------------------------ */ diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 084a7d18a7..1db15c3990 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -50,7 +50,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { return -1; } - tlistAppend(&(pVnode->pBufPool->free), pVMA); + tDListAppend(&(pVnode->pBufPool->free), pVMA); } return 0; @@ -61,16 +61,16 @@ void vnodeCloseBufPool(SVnode *pVnode) { vmaDestroy(pVnode->pBufPool->inuse); while (true) { - SVMemAllocator *pVMA = tlistHead(&(pVnode->pBufPool->incycle)); + SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle)); if (pVMA == NULL) break; - tlistPop(&(pVnode->pBufPool->incycle), pVMA); + tDListPop(&(pVnode->pBufPool->incycle), pVMA); vmaDestroy(pVMA); } while (true) { - SVMemAllocator *pVMA = tlistHead(&(pVnode->pBufPool->free)); + SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free)); if (pVMA == NULL) break; - tlistPop(&(pVnode->pBufPool->free), pVMA); + tDListPop(&(pVnode->pBufPool->free), pVMA); vmaDestroy(pVMA); } @@ -85,9 +85,9 @@ void *vnodeMalloc(SVnode *pVnode, uint64_t size) { if (pBufPool->inuse == NULL) { while (true) { // TODO: add sem_wait and sem_post - pBufPool->inuse = tlistHead(&(pBufPool->free)); + pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free)); if (pBufPool->inuse) { - tlistPop(&(pBufPool->free), pBufPool->inuse); + tDListPop(&(pBufPool->free), pBufPool->inuse); break; } } From 4503901f3e42eaab22b257ed2741de5cbb7a055d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 22:21:28 +0800 Subject: [PATCH 21/24] more --- include/util/tdlist.h | 22 +++++------ .../dnode/vnode/impl/inc/vnodeMemAllocator.h | 11 +++--- .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 37 ++++++++----------- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/include/util/tdlist.h b/include/util/tdlist.h index 8360d2713f..a19f3bebec 100644 --- a/include/util/tdlist.h +++ b/include/util/tdlist.h @@ -23,23 +23,23 @@ extern "C" { // Single linked list #define TD_SLIST_NODE(TYPE) \ struct { \ - struct type *sl_next_; \ + struct TYPE *sl_next_; \ } -#define TD_SLIST(TYPE) \ - struct { \ - struct TYPE *sl_dl_head_; \ - int sl_dl_neles_; \ +#define TD_SLIST(TYPE) \ + struct { \ + struct TYPE *sl_head_; \ + int sl_neles_; \ } -#define TD_SLIST_HEAD(sl) ((sl)->sl_dl_head_) -#define TD_SLIST_NELES(sl) ((sl)->sl_dl_neles_) +#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) -#define tSListInit(sl) \ - do { \ - (sl)->sl_dl_head_ = NULL; \ - (sl)->sl_dl_neles_ = 0; \ +#define tSListInit(sl) \ + do { \ + (sl)->sl_head_ = NULL; \ + (sl)->sl_neles_ = 0; \ } while (0) #define tSListPush(sl, sln) \ diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h index 6b9fddaa02..c8c58e9f69 100644 --- a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h +++ b/source/dnode/vnode/impl/inc/vnodeMemAllocator.h @@ -26,7 +26,7 @@ typedef struct SVArenaNode SVArenaNode; typedef struct SVMemAllocator SVMemAllocator; struct SVArenaNode { - TD_DLIST_NODE(SVArenaNode); + TD_SLIST_NODE(SVArenaNode); uint64_t size; // current node size void * ptr; char data[]; @@ -34,10 +34,11 @@ struct SVArenaNode { struct SVMemAllocator { TD_DLIST_NODE(SVMemAllocator); - uint64_t capacity; - uint64_t ssize; - uint64_t lsize; - TD_DLIST(SVArenaNode) nlist; + uint64_t capacity; + uint64_t ssize; + uint64_t lsize; + SVArenaNode *pNode; + TD_SLIST(SVArenaNode) nlist; }; SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize); diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index 5d2b404314..99d4781df9 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -27,30 +27,25 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { pVMA->capacity = capacity; pVMA->ssize = ssize; pVMA->lsize = lsize; - tDListInit(&(pVMA->nlist)); + tSListInit(&(pVMA->nlist)); - SVArenaNode *pNode = vArenaNodeNew(capacity); - if (pNode == NULL) { + pVMA->pNode = vArenaNodeNew(capacity); + if (pVMA->pNode == NULL) { free(pVMA); return NULL; } - tDListAppend(&(pVMA->nlist), pNode); + tSListPush(&(pVMA->nlist), pVMA->pNode); return pVMA; } void vmaDestroy(SVMemAllocator *pVMA) { if (pVMA) { - while (true) { - SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); - - if (pNode) { - tDListPop(&(pVMA->nlist), pNode); - vArenaNodeFree(pNode); - } else { - break; - } + while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + tSListPop(&(pVMA->nlist)); + vArenaNodeFree(pNode); } free(pVMA); @@ -58,18 +53,18 @@ void vmaDestroy(SVMemAllocator *pVMA) { } void vmaReset(SVMemAllocator *pVMA) { - while (TD_DLIST_NELES(&(pVMA->nlist)) > 1) { - SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); - tDListPop(&(pVMA->nlist), pNode); + while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); + tSListPop(&(pVMA->nlist)); vArenaNodeFree(pNode); } - SVArenaNode *pNode = TD_DLIST_HEAD(&(pVMA->nlist)); + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); pNode->ptr = pNode->data; } void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { - SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); void * ptr; if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { @@ -80,7 +75,7 @@ void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { return NULL; } - tDListAppend(&(pVMA->nlist), pNode); + tSListPush(&(pVMA->nlist), pNode); } ptr = pNode->ptr; @@ -94,9 +89,9 @@ void vmaFree(SVMemAllocator *pVMA, void *ptr) { } bool vmaIsFull(SVMemAllocator *pVMA) { - SVArenaNode *pNode = TD_DLIST_TAIL(&(pVMA->nlist)); + SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); - return (TD_DLIST_NELES(&(pVMA->nlist)) > 1) || + return (TD_SLIST_NELES(&(pVMA->nlist)) > 1) || (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + pVMA->lsize); } From e5cd09151bb3fd7127272b9b7c4f9019c02c77c6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 13 Dec 2021 22:53:28 +0800 Subject: [PATCH 22/24] more --- source/dnode/vnode/impl/src/vnodeCfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/src/vnodeCfg.c b/source/dnode/vnode/impl/src/vnodeCfg.c index 7aac9ca6a7..f5bb7e35d2 100644 --- a/source/dnode/vnode/impl/src/vnodeCfg.c +++ b/source/dnode/vnode/impl/src/vnodeCfg.c @@ -15,7 +15,7 @@ #include "vnodeDef.h" -const SVnodeCfg defaultVnodeOptions = {.wsize = 16*1024*1024, .walCfg = {.walLevel = TAOS_WAL_WRITE}}; /* TODO */ +const SVnodeCfg defaultVnodeOptions = {.wsize = 16 * 1024 * 1024, .walCfg = {.level = TAOS_WAL_WRITE}}; /* TODO */ void vnodeOptionsInit(SVnodeCfg *pVnodeOptions) { /* TODO */ vnodeOptionsCopy(pVnodeOptions, &defaultVnodeOptions); From 242e8246e5d2373af6bfffcc3b4631d08286e952 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Dec 2021 10:02:36 +0800 Subject: [PATCH 23/24] make wal cleanup called once for each init --- source/libs/wal/src/walMgmt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 31f2ef037a..629451a722 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -68,9 +68,12 @@ int32_t walInit() { } void walCleanUp() { + int old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0); + if(old == 0) { + return; + } walStopThread(); taosCloseRef(tsWal.refSetId); - atomic_store_8(&tsWal.inited, 0); wInfo("wal module is cleaned up"); } From 43a2bcdce327a59db9895d707911d77573e8e5d9 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Dec 2021 14:38:52 +0800 Subject: [PATCH 24/24] fix wal read handle --- include/libs/wal/wal.h | 58 +++++++++++------------- source/libs/wal/inc/walInt.h | 6 ++- source/libs/wal/src/walMgmt.c | 3 +- source/libs/wal/src/walRead.c | 52 +++++++++++++-------- source/libs/wal/src/walWrite.c | 7 +-- source/libs/wal/test/walMetaTest.cpp | 67 ++++++++++++++++++++-------- 6 files changed, 118 insertions(+), 75 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index ae1e630c6f..744275e6ff 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -32,6 +32,23 @@ extern int32_t wDebugFlag; #define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} #define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} +#define WAL_PREFIX "wal" +#define WAL_PREFIX_LEN 3 +#define WAL_NOSUFFIX_LEN 20 +#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) +#define WAL_LOG_SUFFIX "log" +#define WAL_INDEX_SUFFIX "idx" +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (WAL_PATH_LEN + 32) + +#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) +#define WAL_CUR_POS_WRITABLE 1 +#define WAL_CUR_FILE_WRITABLE 2 +#define WAL_CUR_FAILED 4 + +#pragma pack(push,1) typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, @@ -43,6 +60,7 @@ typedef struct SWalReadHead { uint8_t msgType; int8_t reserved[2]; int32_t len; + //int64_t ingestTs; //not implemented int64_t version; char body[]; } SWalReadHead; @@ -71,25 +89,6 @@ typedef struct { SWalReadHead head; } SWalHead; -#define WAL_PREFIX "wal" -#define WAL_PREFIX_LEN 3 -#define WAL_NOSUFFIX_LEN 20 -#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) -#define WAL_LOG_SUFFIX "log" -#define WAL_INDEX_SUFFIX "idx" -#define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) -#define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFEUL)) -#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) -#define WAL_FILE_LEN (WAL_PATH_LEN + 32) -//#define WAL_FILE_NUM 1 // 3 -#define WAL_FILESET_MAX 128 - -#define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) -#define WAL_CUR_POS_WRITABLE 1 -#define WAL_CUR_FILE_WRITABLE 2 -#define WAL_CUR_FAILED 4 - typedef struct SWalVer { int64_t firstVer; int64_t verInSnapshotting; @@ -101,24 +100,18 @@ typedef struct SWalVer { typedef struct SWal { // cfg SWalCfg cfg; - //total size - int64_t totSize; - //fsync seq - int32_t fsyncSeq; - //reference - int64_t refId; - //write tfd - int64_t writeLogTfd; - int64_t writeIdxTfd; - //wal lifecycle SWalVer vers; - //roll status - int64_t lastRollSeq; //file set int32_t writeCur; + int64_t writeLogTfd; + int64_t writeIdxTfd; SArray* fileInfoSet; //ctl int32_t curStatus; + int32_t fsyncSeq; + int64_t totSize; + int64_t refId; + int64_t lastRollSeq; pthread_mutex_t mutex; //path char path[WAL_PATH_LEN]; @@ -134,8 +127,9 @@ typedef struct SWalReadHandle { int64_t curVersion; int64_t capacity; int64_t status; //if cursor valid - SWalHead head; + SWalHead* pHead; } SWalReadHandle; +#pragma pack(pop) typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index ec01f7d7fc..e546a87326 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -33,10 +33,12 @@ typedef struct WalFileInfo { int64_t fileSize; } WalFileInfo; +#pragma pack(push,1) typedef struct WalIdxEntry { int64_t ver; int64_t offset; } WalIdxEntry; +#pragma pack(pop) static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) { WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft; @@ -78,11 +80,11 @@ static inline WalFileInfo* walGetCurFileInfo(SWal* pWal) { } static inline int walBuildLogName(SWal*pWal, int64_t fileFirstVer, char* buf) { - return sprintf(buf, "%s/%" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer); + return sprintf(buf, "%s/%020" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer); } static inline int walBuildIdxName(SWal*pWal, int64_t fileFirstVer, char* buf) { - return sprintf(buf, "%s/%" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); + return sprintf(buf, "%s/%020" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); } static inline int walValidHeadCksum(SWalHead* pHead) { diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 7c100b4883..629451a722 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -255,9 +255,8 @@ static int32_t walCreateThread() { static void walStopThread() { atomic_store_8(&tsWal.stop, 1); - if (tsWal.thread != NULL && taosCheckPthreadValid(tsWal.thread)) { + if (taosCheckPthreadValid(tsWal.thread)) { pthread_join(tsWal.thread, NULL); - tsWal.thread = NULL; } wDebug("wal thread is stopped"); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 554a5c846b..b6aafedea3 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -21,16 +21,25 @@ SWalReadHandle* walOpenReadHandle(SWal* pWal) { if(pRead == NULL) { return NULL; } - memset(pRead, 0, sizeof(SWalReadHandle)); pRead->pWal = pWal; pRead->readIdxTfd = -1; pRead->readLogTfd = -1; - return NULL; + pRead->curVersion = -1; + pRead->curFileFirstVer = -1; + pRead->capacity = 0; + pRead->status = 0; + pRead->pHead = malloc(sizeof(SWalHead)); + if(pRead->pHead == NULL) { + free(pRead); + return NULL; + } + return pRead; } void walCloseReadHandle(SWalReadHandle *pRead) { tfClose(pRead->readIdxTfd); tfClose(pRead->readLogTfd); + tfree(pRead->pHead); free(pRead); } @@ -47,18 +56,17 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i //seek position int64_t offset = (ver - fileFirstVer) * WAL_IDX_ENTRY_SIZE; code = tfLseek(idxTfd, offset, SEEK_SET); - if(code != 0) { + if(code < 0) { return -1; } WalIdxEntry entry; - code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry)); - if(code != 0) { + if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { return -1; } //TODO:deserialize ASSERT(entry.ver == ver); code = tfLseek(logTfd, entry.offset, SEEK_SET); - if (code != 0) { + if (code < 0) { return -1; } return code; @@ -71,13 +79,13 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { tfClose(pRead->readLogTfd); walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); - int logTfd = tfOpenRead(fnameStr); + int64_t logTfd = tfOpenRead(fnameStr); if(logTfd < 0) { return -1; } walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); - int idxTfd = tfOpenRead(fnameStr); + int64_t idxTfd = tfOpenRead(fnameStr); if(idxTfd < 0) { return -1; } @@ -90,7 +98,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { int code; SWal *pWal = pRead->pWal; - if(ver == pWal->vers.lastVer) { + if(ver == pRead->curVersion) { return 0; } if(ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { @@ -126,33 +134,41 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int code; //TODO: check wal life if(pRead->curVersion != ver) { - walReadSeekVer(pRead, ver); + code = walReadSeekVer(pRead, ver); + if(code != 0) { + return -1; + } } if(!tfValid(pRead->readLogTfd)) return -1; - if(sizeof(SWalHead) != tfRead(pRead->readLogTfd, &pRead->head, sizeof(SWalHead))) { + code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); + if(code != sizeof(SWalHead)) { return -1; } - code = walValidHeadCksum(&pRead->head); + code = walValidHeadCksum(pRead->pHead); if(code != 0) { return -1; } - if(pRead->capacity < pRead->head.head.len) { - void* ptr = realloc(pRead, pRead->head.head.len); + if(pRead->capacity < pRead->pHead->head.len) { + void* ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); if(ptr == NULL) { return -1; } - pRead = ptr; - pRead->capacity = pRead->head.head.len; + pRead->pHead = ptr; + pRead->capacity = pRead->pHead->head.len; } - if(pRead->head.head.len != tfRead(pRead->readLogTfd, &pRead->head.head.body, pRead->head.head.len)) { + if(pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { return -1; } - code = walValidBodyCksum(&pRead->head); + + /*code = walValidBodyCksum(pRead->pHead);*/ + ASSERT(pRead->pHead->head.version == ver); + if(code != 0) { return -1; } + pRead->curVersion++; return 0; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 44e8cec153..994b8fc333 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -377,11 +377,12 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i //must truncate explicitly first return -1; } - /*if (!tfValid(pWal->curLogTfd)) return 0;*/ + /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ pthread_mutex_lock(&pWal->mutex); pWal->writeHead.head.version = index; + int64_t offset = walGetCurFileOffset(pWal); pWal->writeHead.head.len = bodyLen; pWal->writeHead.head.msgType = msgType; pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); @@ -393,12 +394,12 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - if (tfWrite(pWal->writeLogTfd, &body, bodyLen) != bodyLen) { + if (tfWrite(pWal->writeLogTfd, (char*)body, bodyLen) != bodyLen) { //ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - code = walWriteIndex(pWal, index, walGetCurFileOffset(pWal)); + code = walWriteIndex(pWal, index, offset); if(code != 0) { //TODO return -1; diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index 504f1ada3f..200bf39c5a 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -5,6 +5,9 @@ #include "walInt.h" +const char* ranStr = "tvapq02tcp"; +const int ranStrLen = strlen(ranStr); + class WalCleanEnv : public ::testing::Test { protected: static void SetUpTestCase() { @@ -157,15 +160,13 @@ TEST_F(WalCleanEnv, removeOldMeta) { TEST_F(WalKeepEnv, readOldMeta) { walResetEnv(); - const char* ranStr = "tvapq02tcp"; - int len = strlen(ranStr); int code; for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, len); + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); - code = walWrite(pWal, i+2, i, (void*)ranStr, len); + code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); ASSERT_EQ(code, -1); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -179,7 +180,7 @@ TEST_F(WalKeepEnv, readOldMeta) { char* newss = walMetaSerialize(pWal); - len = strlen(oldss); + int len = strlen(oldss); ASSERT_EQ(len, strlen(newss)); for(int i = 0; i < len; i++) { EXPECT_EQ(oldss[i], newss[i]); @@ -189,14 +190,12 @@ TEST_F(WalKeepEnv, readOldMeta) { } TEST_F(WalCleanEnv, write) { - const char* ranStr = "tvapq02tcp"; - const int len = strlen(ranStr); int code; for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, len); + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); - code = walWrite(pWal, i+2, i, (void*)ranStr, len); + code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); ASSERT_EQ(code, -1); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -205,11 +204,9 @@ TEST_F(WalCleanEnv, write) { } TEST_F(WalCleanEnv, rollback) { - const char* ranStr = "tvapq02tcp"; - const int len = strlen(ranStr); int code; for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, len); + code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -224,12 +221,10 @@ TEST_F(WalCleanEnv, rollback) { } TEST_F(WalCleanDeleteEnv, roll) { - const char* ranStr = "tvapq02tcp"; - const int len = strlen(ranStr); int code; int i; for(i = 0; i < 100; i++) { - code = walWrite(pWal, i, 0, (void*)ranStr, len); + code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); code = walCommit(pWal, i); @@ -242,19 +237,55 @@ TEST_F(WalCleanDeleteEnv, roll) { ASSERT_EQ(pWal->vers.snapshotVer, i-1); ASSERT_EQ(pWal->vers.verInSnapshotting, -1); - code = walWrite(pWal, 5, 0, (void*)ranStr, len); + code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen); ASSERT_NE(code, 0); for(; i < 200; i++) { - code = walWrite(pWal, i, 0, (void*)ranStr, len); + code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); code = walCommit(pWal, i); ASSERT_EQ(pWal->vers.commitVer, i); } - //code = walWriteMeta(pWal); code = walBeginTakeSnapshot(pWal, i - 1); ASSERT_EQ(code, 0); code = walEndTakeSnapshot(pWal); ASSERT_EQ(code, 0); } + +TEST_F(WalKeepEnv, readHandleRead) { + walResetEnv(); + int code; + SWalReadHandle* pRead = walOpenReadHandle(pWal); + ASSERT(pRead != NULL); + + int i ; + for(i = 0; i < 100; i++) { + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, i); + int len = strlen(newStr); + code = walWrite(pWal, i, 0, newStr, len); + ASSERT_EQ(code, 0); + } + for(int i = 0; i < 1000; i++) { + int ver = rand() % 100; + code = walReadWithHandle(pRead, ver); + ASSERT_EQ(code, 0); + + //printf("rrbody: \n"); + //for(int i = 0; i < pRead->pHead->head.len; i++) { + //printf("%d ", pRead->pHead->head.body[i]); + //} + //printf("\n"); + + ASSERT_EQ(pRead->pHead->head.version, ver); + ASSERT_EQ(pRead->curVersion, ver+1); + char newStr[100]; + sprintf(newStr, "%s-%d", ranStr, ver); + int len = strlen(newStr); + ASSERT_EQ(pRead->pHead->head.len, len); + for(int j = 0; j < len; j++) { + EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); + } + } +}