From e9bf4fceb057819994e36541d7fd354bb332a150 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 2 Nov 2021 15:06:56 +0800 Subject: [PATCH] refact --- include/os/osDir.h | 2 +- source/dnode/vnode/meta/inc/metaDB.h | 39 +++++++ source/dnode/vnode/meta/inc/metaDef.h | 14 +-- source/dnode/vnode/meta/inc/metaOptions.h | 33 ++++++ source/dnode/vnode/meta/inc/metaTbOptions.h | 27 +++++ source/dnode/vnode/meta/inc/metaUid.h | 13 +-- source/dnode/vnode/meta/src/metaDB.c | 14 +++ source/dnode/vnode/meta/src/metaMain.c | 113 ++++++++++++-------- source/dnode/vnode/meta/src/metaOptions.c | 10 +- source/dnode/vnode/meta/src/metaTbOptions.c | 14 +++ source/dnode/vnode/meta/src/metaUid.c | 4 +- source/dnode/vnode/meta/test/CMakeLists.txt | 48 ++++----- source/os/src/osDir.c | 2 +- 13 files changed, 239 insertions(+), 94 deletions(-) create mode 100644 source/dnode/vnode/meta/inc/metaDB.h create mode 100644 source/dnode/vnode/meta/inc/metaOptions.h create mode 100644 source/dnode/vnode/meta/inc/metaTbOptions.h create mode 100644 source/dnode/vnode/meta/src/metaDB.c create mode 100644 source/dnode/vnode/meta/src/metaTbOptions.c diff --git a/include/os/osDir.h b/include/os/osDir.h index 3ee3be2c10..8aefaa171a 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -22,7 +22,7 @@ extern "C" { void taosRemoveDir(const char *dirname); bool taosDirExist(char *dirname); -bool taosMkDir(char *dirname); +bool taosMkDir(const char *dirname); void taosRemoveOldFiles(char *dirname, int32_t keepDays); bool taosExpandDir(char *dirname, char *outname, int32_t maxlen); bool taosRealPath(char *dirname, int32_t maxlen); diff --git a/source/dnode/vnode/meta/inc/metaDB.h b/source/dnode/vnode/meta/inc/metaDB.h new file mode 100644 index 0000000000..299b8e0350 --- /dev/null +++ b/source/dnode/vnode/meta/inc/metaDB.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_META_DB_H_ +#define _TD_META_DB_H_ + +#include "meta.h" +#include "tkv.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SMetaDB { + STkvDb * pDB; + STkvDb * pIdx; + STkvCache *pCache; +} SMetaDB; + +int metaOpenDB(SMeta *pMeta); +void metaCloseDB(SMeta *pMeta); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_META_DB_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/meta/inc/metaDef.h b/source/dnode/vnode/meta/inc/metaDef.h index f5d0b7f74c..ef0cc9e4c2 100644 --- a/source/dnode/vnode/meta/inc/metaDef.h +++ b/source/dnode/vnode/meta/inc/metaDef.h @@ -16,22 +16,18 @@ #ifndef _TD_META_DEF_H_ #define _TD_META_DEF_H_ +#include "metaDB.h" #include "metaUid.h" -#include "tkv.h" #ifdef __cplusplus extern "C" { #endif struct SMeta { - STableUidGenerator uidGenerator; - - STkvDb* tableDb; // uid->table obj - STkvDb* tbnameDb; // tbname --> uid - STkvDb* schemaDb; // uid+version --> schema - STkvDb* tagDb; // uid --> tag - STkvDb* tagIdx; // TODO: need to integrate lucene or our own - // STkvCache* metaCache; // TODO: add a global cache here + char* path; // path of current meta + STbUidGenerator uidGenerator; // meta table UID generator + SMetaDB* pMetaDB; // meta DB for real storage engine + SMetaOptions options; // meta option }; #ifdef __cplusplus diff --git a/source/dnode/vnode/meta/inc/metaOptions.h b/source/dnode/vnode/meta/inc/metaOptions.h new file mode 100644 index 0000000000..7033a873df --- /dev/null +++ b/source/dnode/vnode/meta/inc/metaOptions.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_META_OPTIONS_H_ +#define _TD_META_OPTIONS_H_ + +#include "meta.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern const SMetaOptions defaultMetaOptions; + +int metaValidateOptions(const SMetaOptions *); +void metaOptionsCopy(SMetaOptions *pDest, const SMetaOptions *pSrc); +#ifdef __cplusplus +} +#endif + +#endif /*_TD_META_OPTIONS_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/meta/inc/metaTbOptions.h b/source/dnode/vnode/meta/inc/metaTbOptions.h new file mode 100644 index 0000000000..cd2b2ee0e8 --- /dev/null +++ b/source/dnode/vnode/meta/inc/metaTbOptions.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_META_TABLE_OPTIONS_H_ +#define _TD_META_TABLE_OPTIONS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_META_TABLE_OPTIONS_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/meta/inc/metaUid.h b/source/dnode/vnode/meta/inc/metaUid.h index 37c3fac6ba..86f4e26cec 100644 --- a/source/dnode/vnode/meta/inc/metaUid.h +++ b/source/dnode/vnode/meta/inc/metaUid.h @@ -23,21 +23,18 @@ extern "C" { #endif /* ------------------------ APIS EXPOSED ------------------------ */ -typedef struct STableUidGenerator STableUidGenerator; +typedef struct STbUidGenerator { + tb_uid_t nextUid; +} STbUidGenerator; // tb_uid_t #define IVLD_TB_UID 0 -tb_uid_t generateUid(STableUidGenerator *); +tb_uid_t generateUid(STbUidGenerator *); // STableUidGenerator -void tableUidGeneratorInit(STableUidGenerator *, tb_uid_t suid); +void tableUidGeneratorInit(STbUidGenerator *, tb_uid_t suid); #define tableUidGeneratorClear(ug) -/* ------------------------ FOR TEST AND COMPILE ONLY ------------------------ */ -struct STableUidGenerator { - tb_uid_t nextUid; -}; - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/meta/src/metaDB.c b/source/dnode/vnode/meta/src/metaDB.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/dnode/vnode/meta/src/metaDB.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaMain.c b/source/dnode/vnode/meta/src/metaMain.c index 8844055a98..0615bf1e50 100644 --- a/source/dnode/vnode/meta/src/metaMain.c +++ b/source/dnode/vnode/meta/src/metaMain.c @@ -13,74 +13,62 @@ * along with this program. If not, see . */ -#include "meta.h" -#include "metaDef.h" #include "tcoding.h" -static int metaCreateSuperTable(SMeta *pMeta, const char *tbname, const SSuperTableOpts *pSuperTableOpts); -static int metaCreateChildTable(SMeta *pMeta, const char *tbname, const SChildTableOpts *pChildTableOpts); -static int metaCreateNormalTable(SMeta *pMeta, const char *tbname, const SNormalTableOpts *pNormalTableOpts); +#include "meta.h" +#include "metaDef.h" +#include "metaOptions.h" +#include "metaDB.h" -SMeta *metaOpen(const char *path, const SMetaOptions *pMetaOpts) { +static SMeta *metaNew(const char *path, const SMetaOptions *pMetaOptions); +static void metaFree(SMeta *pMeta); +static int metaCreateSuperTable(SMeta *pMeta, const char *tbname, const SSuperTableOpts *pSuperTableOpts); +static int metaCreateChildTable(SMeta *pMeta, const char *tbname, const SChildTableOpts *pChildTableOpts); +static int metaCreateNormalTable(SMeta *pMeta, const char *tbname, const SNormalTableOpts *pNormalTableOpts); + +SMeta *metaOpen(const char *path, const SMetaOptions *pMetaOptions) { SMeta *pMeta = NULL; - pMeta = (SMeta *)calloc(1, sizeof(*pMeta)); - if (pMeta == NULL) { + // Set default options + if (pMetaOptions == NULL) { + pMetaOptions = &defaultMetaOptions; + } + + // Validate the options + if (metaValidateOptions(pMetaOptions) < 0) { + // TODO: deal with error return NULL; } - // TODO: check if file exists and handle the error - taosMkDir("meta"); + // Allocate handle + pMeta = metaNew(path, pMetaOptions); + if (pMeta == NULL) { + // TODO: handle error + return NULL; + } - // Open tableDb - STkvOpts *tableDbOpts = tkvOptsCreate(); - tkvOptsSetCreateIfMissing(tableDbOpts, 1); - pMeta->tableDb = tkvOpen(tableDbOpts, "meta/table_db"); - tkvOptsDestroy(tableDbOpts); + // Create META path + taosMkDir(path); - // Open tbnameDb - STkvOpts *tbnameDbOpts = tkvOptsCreate(); - tkvOptsSetCreateIfMissing(tbnameDbOpts, 1); - pMeta->tbnameDb = tkvOpen(tbnameDbOpts, "meta/tbname_db"); - tkvOptsDestroy(tbnameDbOpts); + // Open the DBs needed + if (metaOpenDB(pMeta) < 0) { + // TODO: handle error + metaFree(pMeta); + return NULL; + } - // Open schemaDb - STkvOpts *schemaDbOpts = tkvOptsCreate(); - tkvOptsSetCreateIfMissing(schemaDbOpts, 1); - pMeta->schemaDb = tkvOpen(schemaDbOpts, "meta/schema_db"); - tkvOptsDestroy(schemaDbOpts); - - // Open tagDb - STkvOpts *tagDbOpts = tkvOptsCreate(); - tkvOptsSetCreateIfMissing(tagDbOpts, 1); - pMeta->tagDb = tkvOpen(tagDbOpts, "meta/tag_db"); - tkvOptsDestroy(tagDbOpts); - - // Open tagIdx - STkvOpts *tagIdxDbOpts = tkvOptsCreate(); - tkvOptsSetCreateIfMissing(tagIdxDbOpts, 1); - pMeta->tagIdx = tkvOpen(tagIdxDbOpts, "meta/tag_idx_db"); - tkvOptsDestroy(tagIdxDbOpts); - - // TODO: need to figure out how to persist the START UID - tableUidGeneratorInit(&(pMeta->uidGenerator), IVLD_TB_UID); return pMeta; } void metaClose(SMeta *pMeta) { if (pMeta) { tableUidGeneratorClear(&pMeta->uidGenerator); - - tkvClose(pMeta->tagIdx); - tkvClose(pMeta->tagDb); - tkvClose(pMeta->schemaDb); - tkvClose(pMeta->tbnameDb); - tkvClose(pMeta->tableDb); - + metaCloseDB(pMeta); free(pMeta); } } +#if 0 int metaCreateTable(SMeta *pMeta, const STableOptions *pTableOpts) { size_t vallen; char * pUid; @@ -106,8 +94,37 @@ int metaCreateTable(SMeta *pMeta, const STableOptions *pTableOpts) { return 0; } +#endif /* ------------------------ STATIC METHODS ------------------------ */ +static SMeta *metaNew(const char *path, const SMetaOptions *pMetaOptions) { + SMeta *pMeta; + size_t psize = strlen(path); + + pMeta = (SMeta *)calloc(1, sizeof(*pMeta)); + if (pMeta == NULL) { + return NULL; + } + + pMeta->path = strdup(path); + if (pMeta->path == NULL) { + return NULL; + } + + metaOptionsCopy(&(pMeta->options), pMetaOptions); + + return pMeta; +}; + +static void metaFree(SMeta *pMeta) { + if (pMeta) { + tfree(pMeta->path); + free(pMeta); + } +} + +// OLD ------------------------------------------------------------------- +#if 0 static int metaCreateSuperTable(SMeta *pMeta, const char *tbname, const SSuperTableOpts *pSuperTableOpts) { size_t vallen; size_t keylen; @@ -258,3 +275,5 @@ void metaTableOptsClear(STableOptions *pTableOpts) { } void metaDestroy(const char *path) { taosRemoveDir(path); } + +#endif \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaOptions.c b/source/dnode/vnode/meta/src/metaOptions.c index 646ac873b1..0f6ba9a9fb 100644 --- a/source/dnode/vnode/meta/src/metaOptions.c +++ b/source/dnode/vnode/meta/src/metaOptions.c @@ -26,5 +26,11 @@ void metaOptionsClear(SMetaOptions *pMetaOptions) { // TODO } -/* ------------------------ STATIC METHODS ------------------------ */ -static void metaOptionsCopy(SMetaOptions *pDest, const SMetaOptions *pSrc) { memcpy(pDest, pSrc, sizeof(*pSrc)); } \ No newline at end of file +int metaValidateOptions(const SMetaOptions *pMetaOptions) { + // TODO + return 0; +} + +void metaOptionsCopy(SMetaOptions *pDest, const SMetaOptions *pSrc) { memcpy(pDest, pSrc, sizeof(*pSrc)); } + +/* ------------------------ STATIC METHODS ------------------------ */ \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaTbOptions.c b/source/dnode/vnode/meta/src/metaTbOptions.c new file mode 100644 index 0000000000..6dea4a4e57 --- /dev/null +++ b/source/dnode/vnode/meta/src/metaTbOptions.c @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaUid.c b/source/dnode/vnode/meta/src/metaUid.c index 80afa490f3..eadacf67d3 100644 --- a/source/dnode/vnode/meta/src/metaUid.c +++ b/source/dnode/vnode/meta/src/metaUid.c @@ -15,12 +15,12 @@ #include "metaUid.h" -tb_uid_t generateUid(STableUidGenerator *pGen) { +tb_uid_t generateUid(STbUidGenerator *pGen) { // Generate a new table UID return ++(pGen->nextUid); } -void tableUidGeneratorInit(STableUidGenerator *pGen, tb_uid_t suid) { +void tableUidGeneratorInit(STbUidGenerator *pGen, tb_uid_t suid) { // Init a generator pGen->nextUid = suid; } \ No newline at end of file diff --git a/source/dnode/vnode/meta/test/CMakeLists.txt b/source/dnode/vnode/meta/test/CMakeLists.txt index b37ba6abd4..e4ea7839c8 100644 --- a/source/dnode/vnode/meta/test/CMakeLists.txt +++ b/source/dnode/vnode/meta/test/CMakeLists.txt @@ -1,24 +1,24 @@ -add_executable(metaTest "") -target_sources(metaTest - PRIVATE - "../src/metaMain.c" - "../src/metaUid.c" - "metaTests.cpp" -) -target_include_directories(metaTest - PUBLIC - "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -) -target_link_libraries(metaTest - os - util - common - gtest_main - tkv -) -enable_testing() -add_test( - NAME meta_test - COMMAND metaTest -) \ No newline at end of file +# add_executable(metaTest "") +# target_sources(metaTest +# PRIVATE +# "../src/metaMain.c" +# "../src/metaUid.c" +# "metaTests.cpp" +# ) +# target_include_directories(metaTest +# PUBLIC +# "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" +# "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +# ) +# target_link_libraries(metaTest +# os +# util +# common +# gtest_main +# tkv +# ) +# enable_testing() +# add_test( +# NAME meta_test +# COMMAND metaTest +# ) \ No newline at end of file diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 17ab88edf6..cd5f561918 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -60,7 +60,7 @@ void taosRemoveDir(const char *dirname) { bool taosDirExist(char *dirname) { return access(dirname, F_OK) == 0; } -bool taosMkDir(char *dirname) { +bool taosMkDir(const char *dirname) { int32_t code = mkdir(dirname, 0755); if (code < 0 && errno == EEXIST) { return true;