From 557905d0068c97ec557558a2699034b543a260df Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 11:15:11 +0800 Subject: [PATCH 01/14] more progress --- source/server/vnode/meta/src/meta.c | 33 +++++++++++++++++---- source/server/vnode/meta/test/metaTests.cpp | 7 +++-- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index f35de5eb9e..ff8186d1d7 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -22,6 +22,7 @@ #include "meta.h" +/* -------------------- Structures -------------------- */ typedef struct STable { uint64_t uid; tstr * name; @@ -46,9 +47,12 @@ struct SMeta { rocksdb_t *tbnameDb; // tbname --> uid rocksdb_t *tagDb; // uid --> tag rocksdb_t *schemaDb; + rocksdb_t *tagIdx; size_t totalUsed; }; +/* -------------------- Methods -------------------- */ + SMeta *metaOpen(SMetaOptions *options) { SMeta *pMeta = NULL; char * err = NULL; @@ -64,21 +68,40 @@ SMeta *metaOpen(SMetaOptions *options) { pMeta->stbList = tdListNew(sizeof(STableObj *)); + // Options + rocksdb_options_t *dbOptions = rocksdb_options_create(); + rocksdb_options_set_create_if_missing(dbOptions, 1); + + taosMkDir("meta"); + // Open tbname DB - rocksdb_options_t *tbnameDbOptions = rocksdb_options_create(); - pMeta->tbnameDb = rocksdb_open(tbnameDbOptions, "tbname_uid_db", &err); + pMeta->tbnameDb = rocksdb_open(dbOptions, "meta/tbname_uid_db", &err); // Open tag DB - pMeta->tagDb = rocksdb_open(tbnameDbOptions, "uid_tag_db", &err); + pMeta->tagDb = rocksdb_open(dbOptions, "meta/uid_tag_db", &err); // Open schema DB - pMeta->schemaDb = rocksdb_open(tbnameDbOptions, "schema_db", &err); + pMeta->schemaDb = rocksdb_open(dbOptions, "meta/schema_db", &err); + + // Open tag index + pMeta->tagIdx = rocksdb_open(dbOptions, "meta/tag_idx_db", &err); + + rocksdb_options_destroy(dbOptions); return pMeta; } void metaClose(SMeta *pMeta) { - // TODO + if (pMeta) { + rocksdb_close(pMeta->tagIdx); + rocksdb_close(pMeta->schemaDb); + rocksdb_close(pMeta->tagDb); + rocksdb_close(pMeta->tbnameDb); + + tdListFree(pMeta->stbList); + taosHashCleanup(pMeta->pTableObjHash); + pthread_rwlock_destroy(&(pMeta->rwLock)); + } } int metaCommit(SMeta *meta) { return 0; } \ No newline at end of file diff --git a/source/server/vnode/meta/test/metaTests.cpp b/source/server/vnode/meta/test/metaTests.cpp index ab204455f2..caf77f720a 100644 --- a/source/server/vnode/meta/test/metaTests.cpp +++ b/source/server/vnode/meta/test/metaTests.cpp @@ -4,6 +4,9 @@ #include "meta.h" TEST(MetaTest, meta_open_test) { - metaOpen(NULL); - std::cout << "Hello META!" << std::endl; + SMeta *meta = metaOpen(NULL); + std::cout << "Meta is opened!" << std::endl; + + metaClose(meta); + std::cout << "Meta is closed!" << std::endl; } \ No newline at end of file From be63e61e1664bcb80050096f4432cdb88f908d1c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 11:29:22 +0800 Subject: [PATCH 02/14] more --- source/server/vnode/meta/test/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/server/vnode/meta/test/CMakeLists.txt b/source/server/vnode/meta/test/CMakeLists.txt index ca82d3fb83..ce00dc0a66 100644 --- a/source/server/vnode/meta/test/CMakeLists.txt +++ b/source/server/vnode/meta/test/CMakeLists.txt @@ -15,4 +15,9 @@ target_link_libraries(metaTest common rocksdb gtest_main +) +enable_testing() +add_test( + NAME meta_test + COMMAND metaTest ) \ No newline at end of file From 98ae04e7e62872b258a72f46167e90c555834f03 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 11:41:27 +0800 Subject: [PATCH 03/14] more --- include/server/vnode/meta/meta.h | 2 +- source/os/src/osDir.c | 2 +- source/server/vnode/meta/src/meta.c | 2 ++ source/server/vnode/meta/test/metaTests.cpp | 3 +++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index 4167dedde6..bfa11b9c39 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -34,7 +34,7 @@ typedef struct SMetaQueryOptions SMetaQueryOptions; // SMeta operations int metaCreate(const char *path); -int metaDestroy(const char *path); +void metaDestroy(const char *path); SMeta *metaOpen(SMetaOptions *); void metaClose(SMeta *); int metaCreateTable(SMeta *, void *); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 0526dffe9e..839db811fd 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -48,7 +48,7 @@ void taosRemoveDir(char *dirname) { taosRemoveDir(filename); } else { (void)remove(filename); - printf("file:%s is removed", filename); + printf("file:%s is removed\n", filename); } } diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index ff8186d1d7..e945e6eac4 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -104,4 +104,6 @@ void metaClose(SMeta *pMeta) { } } +void metaDestroy(const char *path) { taosRemoveDir(path); } + int metaCommit(SMeta *meta) { return 0; } \ No newline at end of file diff --git a/source/server/vnode/meta/test/metaTests.cpp b/source/server/vnode/meta/test/metaTests.cpp index caf77f720a..fcc1220273 100644 --- a/source/server/vnode/meta/test/metaTests.cpp +++ b/source/server/vnode/meta/test/metaTests.cpp @@ -9,4 +9,7 @@ TEST(MetaTest, meta_open_test) { metaClose(meta); std::cout << "Meta is closed!" << std::endl; + + metaDestroy("meta"); + std::cout << "Meta is destroyed!" << std::endl; } \ No newline at end of file From 9728ec02fe1508732c38a092201707e546128f5c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 13:42:12 +0800 Subject: [PATCH 04/14] refact --- include/common/schema.h | 76 ----------------------------- include/server/vnode/meta/meta.h | 27 +++++----- source/server/vnode/meta/src/meta.c | 13 ++++- 3 files changed, 26 insertions(+), 90 deletions(-) diff --git a/include/common/schema.h b/include/common/schema.h index ccc91f09ff..5e9057520b 100644 --- a/include/common/schema.h +++ b/include/common/schema.h @@ -16,86 +16,10 @@ #ifndef _TD_COMMON_SCHEMA_H_ #define _TD_COMMON_SCHEMA_H_ -#include "os.h" - #ifdef __cplusplus extern "C" { #endif -// ----------------- TSDB COLUMN DEFINITION -typedef struct { - int8_t type; // Column type - int16_t colId; // column ID - int16_t bytes; // column bytes (restore to int16_t in case of misuse) - uint16_t offset; // point offset in SDataRow after the header part. -} STColumn; - -#define colType(col) ((col)->type) -#define colColId(col) ((col)->colId) -#define colBytes(col) ((col)->bytes) -#define colOffset(col) ((col)->offset) - -#define colSetType(col, t) (colType(col) = (t)) -#define colSetColId(col, id) (colColId(col) = (id)) -#define colSetBytes(col, b) (colBytes(col) = (b)) -#define colSetOffset(col, o) (colOffset(col) = (o)) - -// ----------------- TSDB SCHEMA DEFINITION -typedef struct { - int version; // version - int numOfCols; // Number of columns appended - int tlen; // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + // - // (bytes)) - uint16_t flen; // First part length in a SDataRow after the header part - uint16_t vlen; // pure value part length, excluded the overhead (bytes only) - STColumn columns[]; -} STSchema; - -#define schemaNCols(s) ((s)->numOfCols) -#define schemaVersion(s) ((s)->version) -#define schemaTLen(s) ((s)->tlen) -#define schemaFLen(s) ((s)->flen) -#define schemaVLen(s) ((s)->vlen) -#define schemaColAt(s, i) ((s)->columns + i) -#define tdFreeSchema(s) tfree((s)) - -STSchema *tdDupSchema(STSchema *pSchema); -int tdEncodeSchema(void **buf, STSchema *pSchema); -void * tdDecodeSchema(void *buf, STSchema **pRSchema); - -static FORCE_INLINE int comparColId(const void *key1, const void *key2) { - if (*(int16_t *)key1 > ((STColumn *)key2)->colId) { - return 1; - } else if (*(int16_t *)key1 < ((STColumn *)key2)->colId) { - return -1; - } else { - return 0; - } -} - -static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) { - void *ptr = bsearch(&colId, (void *)pSchema->columns, schemaNCols(pSchema), sizeof(STColumn), comparColId); - if (ptr == NULL) return NULL; - return (STColumn *)ptr; -} - -// ----------------- SCHEMA BUILDER DEFINITION -typedef struct { - int tCols; - int nCols; - int tlen; - uint16_t flen; - uint16_t vlen; - int version; - STColumn *columns; -} STSchemaBuilder; - -int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); -void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); -void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); -int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); -STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); - #ifdef __cplusplus } #endif diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index bfa11b9c39..f639ed243f 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -27,33 +27,34 @@ extern "C" { typedef uint64_t tuid_t; // Types exported -typedef struct SMeta SMeta; -typedef struct SMetaOptions SMetaOptions; -typedef struct SMetaQueryHandle SMetaQueryHandle; -typedef struct SMetaQueryOptions SMetaQueryOptions; +typedef struct SMeta SMeta; +typedef struct SMetaOpts SMetaOpts; +typedef struct SMetaQueryHandle SMetaQueryHandle; +typedef struct SMetaQueryOpts SMetaQueryOpts; +typedef struct STableOpts STableOpts; // SMeta operations int metaCreate(const char *path); void metaDestroy(const char *path); -SMeta *metaOpen(SMetaOptions *); +SMeta *metaOpen(SMetaOpts *); void metaClose(SMeta *); -int metaCreateTable(SMeta *, void *); +int metaCreateTable(SMeta *, STableOpts *); int metaDropTable(SMeta *, uint64_t tuid_t); int metaAlterTable(SMeta *, void *); int metaCommit(SMeta *); // Options -SMetaOptions *metaOptionsCreate(); -void metaOptionsDestroy(SMetaOptions *); -void metaOptionsSetCache(SMetaOptions *, size_t capacity); +SMetaOpts *metaOptionsCreate(); +void metaOptionsDestroy(SMetaOpts *); +void metaOptionsSetCache(SMetaOpts *, size_t capacity); // SMetaQueryHandle -SMetaQueryHandle *metaQueryHandleCreate(SMetaQueryOptions *); +SMetaQueryHandle *metaQueryHandleCreate(SMetaQueryOpts *); void metaQueryHandleDestroy(SMetaQueryHandle *); -// SMetaQueryOptions -SMetaQueryOptions *metaQueryOptionsCreate(); -void metaQueryOptionsDestroy(SMetaQueryOptions *); +// SMetaQueryOpts +SMetaQueryOpts *metaQueryOptionsCreate(); +void metaQueryOptionsDestroy(SMetaQueryOpts *); #ifdef __cplusplus } diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index e945e6eac4..a035fef205 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -51,9 +51,15 @@ struct SMeta { size_t totalUsed; }; +struct STableOpts { + int8_t type; + char * name; + STSchema *pSchema; +}; + /* -------------------- Methods -------------------- */ -SMeta *metaOpen(SMetaOptions *options) { +SMeta *metaOpen(SMetaOpts *options) { SMeta *pMeta = NULL; char * err = NULL; @@ -104,6 +110,11 @@ void metaClose(SMeta *pMeta) { } } +int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { + // TODO + return 0; +} + void metaDestroy(const char *path) { taosRemoveDir(path); } int metaCommit(SMeta *meta) { return 0; } \ No newline at end of file From 512ad36e62b5a29d919bc00f0cdfd2ca9416e4a9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 13:49:12 +0800 Subject: [PATCH 05/14] refact --- include/common/{schema.h => tschema.h} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename include/common/{schema.h => tschema.h} (100%) diff --git a/include/common/schema.h b/include/common/tschema.h similarity index 100% rename from include/common/schema.h rename to include/common/tschema.h From b45884c15135ad04b4eb2bc5e094667d168f7e6f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 14:45:59 +0800 Subject: [PATCH 06/14] more --- include/server/vnode/meta/meta.h | 10 ++ source/server/vnode/meta/inc/metaUid.h | 34 ++++++ source/server/vnode/meta/src/meta.c | 106 +++++++++++++++++-- source/server/vnode/meta/src/metaUid.c | 23 ++++ source/server/vnode/meta/test/CMakeLists.txt | 1 + 5 files changed, 164 insertions(+), 10 deletions(-) create mode 100644 source/server/vnode/meta/inc/metaUid.h create mode 100644 source/server/vnode/meta/src/metaUid.c diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index f639ed243f..9797ed8168 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -56,6 +56,16 @@ void metaQueryHandleDestroy(SMetaQueryHandle *); SMetaQueryOpts *metaQueryOptionsCreate(); void metaQueryOptionsDestroy(SMetaQueryOpts *); +// STableOpts +void metaTableOptsInit(int8_t type, const char *name, const STSchema *pSchema); + +/* -------------------------------- Hided implementations -------------------------------- */ +struct STableOpts { + int8_t type; + char * name; + STSchema *pSchema; +}; + #ifdef __cplusplus } #endif diff --git a/source/server/vnode/meta/inc/metaUid.h b/source/server/vnode/meta/inc/metaUid.h new file mode 100644 index 0000000000..b01492cbf7 --- /dev/null +++ b/source/server/vnode/meta/inc/metaUid.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_META_UID_H_ +#define _TD_META_UID_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef uint64_t tb_uid_t; +tb_uid_t metaGenerateUid(); + +#define IVLD_TB_UID 0 + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_META_UID_H_*/ \ No newline at end of file diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index a035fef205..9cf9d7ba78 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -21,12 +21,14 @@ #include "ttypes.h" #include "meta.h" +#include "metaUid.h" /* -------------------- Structures -------------------- */ + typedef struct STable { - uint64_t uid; - tstr * name; - uint64_t suid; + tb_uid_t uid; + char * name; + tb_uid_t suid; SArray * schema; } STable; @@ -51,11 +53,8 @@ struct SMeta { size_t totalUsed; }; -struct STableOpts { - int8_t type; - char * name; - STSchema *pSchema; -}; +static STable * metaTableNew(tb_uid_t uid, const char *name, int32_t sver); +static STableObj *metaTableObjNew(); /* -------------------- Methods -------------------- */ @@ -111,10 +110,97 @@ void metaClose(SMeta *pMeta) { } int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { - // TODO + size_t vallen; + char * err = NULL; + rocksdb_readoptions_t * ropt; + STableObj * pTableObj = NULL; + rocksdb_writeoptions_t *wopt; + + // Check if table already exists + ropt = rocksdb_readoptions_create(); + + char *uidStr = rocksdb_get(pMeta->tbnameDb, ropt, pTableOpts->name, strlen(pTableOpts->name), &vallen, &err); + if (uidStr != NULL) { + // Has duplicate named table + return -1; + } + + rocksdb_readoptions_destroy(ropt); + + // Create table obj + pTableObj = metaTableObjNew(); + if (pTableObj == NULL) { + // TODO + return -1; + } + + // Create table object + pTableObj->pTable = metaTableNew(metaGenerateUid(), pTableOpts->name, schemaVersion(pTableOpts->pSchema)); + if (pTableObj->pTable == NULL) { + // TODO + } + + pthread_rwlock_rdlock(&pMeta->rwLock); + + taosHashPut(pMeta->pTableObjHash, &(pTableObj->pTable->uid), sizeof(tb_uid_t), &pTableObj, sizeof(pTableObj)); + + wopt = rocksdb_writeoptions_create(); + + rocksdb_put(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, + sizeof(tb_uid_t), &err); + rocksdb_put(pMeta->schemaDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, + sizeof(tb_uid_t), &err); + + rocksdb_writeoptions_destroy(wopt); + + pthread_rwlock_unlock(&pMeta->rwLock); + return 0; } void metaDestroy(const char *path) { taosRemoveDir(path); } -int metaCommit(SMeta *meta) { return 0; } \ No newline at end of file +int metaCommit(SMeta *meta) { return 0; } + +/* -------------------- Static Methods -------------------- */ + +static STable *metaTableNew(tb_uid_t uid, const char *name, int32_t sver) { + STable *pTable = NULL; + + pTable = (STable *)malloc(sizeof(*pTable)); + if (pTable == NULL) { + // TODO + return NULL; + } + + pTable->schema = taosArrayInit(0, sizeof(int32_t)); + if (pTable->schema == NULL) { + // TODO + return NULL; + } + + pTable->uid = uid; + pTable->name = strdup(name); + pTable->suid = IVLD_TB_UID; + taosArrayPush(pTable->schema, &sver); + + return pTable; +} + +static STableObj *metaTableObjNew() { + STableObj *pTableObj = NULL; + + pTableObj = (STableObj *)malloc(sizeof(*pTableObj)); + if (pTableObj == NULL) { + return NULL; + } + + pTableObj->pin = true; + pTableObj->ref = 1; + taosInitRWLatch(&(pTableObj->latch)); + pTableObj->offset = UINT64_MAX; + pTableObj->ctbList = NULL; + pTableObj->pTable = NULL; + + return pTableObj; +} \ No newline at end of file diff --git a/source/server/vnode/meta/src/metaUid.c b/source/server/vnode/meta/src/metaUid.c new file mode 100644 index 0000000000..4662969a2e --- /dev/null +++ b/source/server/vnode/meta/src/metaUid.c @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "metaUid.h" + +static tb_uid_t nuid = IVLD_TB_UID; + +tb_uid_t metaGenerateUid() { + // TODO: need a more complex UID generator + return ++nuid; +} \ No newline at end of file diff --git a/source/server/vnode/meta/test/CMakeLists.txt b/source/server/vnode/meta/test/CMakeLists.txt index ce00dc0a66..ee16a28687 100644 --- a/source/server/vnode/meta/test/CMakeLists.txt +++ b/source/server/vnode/meta/test/CMakeLists.txt @@ -2,6 +2,7 @@ add_executable(metaTest "") target_sources(metaTest PRIVATE "../src/meta.c" + "../src/metaUid.c" "metaTests.cpp" ) target_include_directories(metaTest From 87dcc7fd92bd2dd1a03f58de9a585569d746ae80 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 15:11:44 +0800 Subject: [PATCH 07/14] make compile and run test --- include/server/vnode/meta/meta.h | 2 +- source/server/vnode/meta/src/meta.c | 19 +++++++++++++-- source/server/vnode/meta/test/metaTests.cpp | 26 +++++++++++++++++++-- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/include/server/vnode/meta/meta.h b/include/server/vnode/meta/meta.h index 9797ed8168..3afe01511b 100644 --- a/include/server/vnode/meta/meta.h +++ b/include/server/vnode/meta/meta.h @@ -57,7 +57,7 @@ SMetaQueryOpts *metaQueryOptionsCreate(); void metaQueryOptionsDestroy(SMetaQueryOpts *); // STableOpts -void metaTableOptsInit(int8_t type, const char *name, const STSchema *pSchema); +void metaTableOptsInit(STableOpts *, int8_t type, const char *name, const STSchema *pSchema); /* -------------------------------- Hided implementations -------------------------------- */ struct STableOpts { diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index 9cf9d7ba78..b3d33c7b87 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -146,10 +146,19 @@ int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { wopt = rocksdb_writeoptions_create(); + // Add to tbname db rocksdb_put(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, sizeof(tb_uid_t), &err); - rocksdb_put(pMeta->schemaDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, - sizeof(tb_uid_t), &err); + + // Add to schema db + char id[12]; + char buf[256]; + void *pBuf = buf; + *(tb_uid_t *)id = pTableObj->pTable->uid; + *(int32_t *)(id + sizeof(tb_uid_t)) = schemaVersion(pTableOpts->pSchema); + int size = tdEncodeSchema(&pBuf, pTableOpts->pSchema); + + rocksdb_put(pMeta->schemaDb, wopt, id, 12, buf, size, &err); rocksdb_writeoptions_destroy(wopt); @@ -162,6 +171,12 @@ void metaDestroy(const char *path) { taosRemoveDir(path); } int metaCommit(SMeta *meta) { return 0; } +void metaTableOptsInit(STableOpts *pTableOpts, int8_t type, const char *name, const STSchema *pSchema) { + pTableOpts->type = type; + pTableOpts->name = strdup(name); + pTableOpts->pSchema = tdDupSchema(pSchema); +} + /* -------------------- Static Methods -------------------- */ static STable *metaTableNew(tb_uid_t uid, const char *name, int32_t sver) { diff --git a/source/server/vnode/meta/test/metaTests.cpp b/source/server/vnode/meta/test/metaTests.cpp index fcc1220273..f81302fc91 100644 --- a/source/server/vnode/meta/test/metaTests.cpp +++ b/source/server/vnode/meta/test/metaTests.cpp @@ -1,15 +1,37 @@ #include +#include #include #include "meta.h" TEST(MetaTest, meta_open_test) { + // Open Meta SMeta *meta = metaOpen(NULL); std::cout << "Meta is opened!" << std::endl; + // Create tables + STableOpts tbOpts; + char tbname[128]; + STSchema * pSchema; + STSchemaBuilder sb; + tdInitTSchemaBuilder(&sb, 0); + for (size_t i = 0; i < 10; i++) { + tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, i, 8); + } + pSchema = tdGetSchemaFromBuilder(&sb); + tdDestroyTSchemaBuilder(&sb); + for (size_t i = 0; i < 1000000; i++) { + sprintf(tbname, "tb%ld", i); + metaTableOptsInit(&tbOpts, 0, tbname, pSchema); + + metaCreateTable(meta, &tbOpts); + } + + // Close Meta metaClose(meta); std::cout << "Meta is closed!" << std::endl; - metaDestroy("meta"); - std::cout << "Meta is destroyed!" << std::endl; + // // Destroy Meta + // metaDestroy("meta"); + // std::cout << "Meta is destroyed!" << std::endl; } \ No newline at end of file From 8287a34ccb1230970be4a16c5fb4cea4c6a904ab Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 15:23:36 +0800 Subject: [PATCH 08/14] more progress --- source/server/vnode/meta/src/meta.c | 7 ++++--- source/server/vnode/meta/test/metaTests.cpp | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index b3d33c7b87..000b2ca9b0 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -145,19 +145,20 @@ int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { taosHashPut(pMeta->pTableObjHash, &(pTableObj->pTable->uid), sizeof(tb_uid_t), &pTableObj, sizeof(pTableObj)); wopt = rocksdb_writeoptions_create(); + rocksdb_writeoptions_disable_WAL(wopt, 1); // Add to tbname db rocksdb_put(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, sizeof(tb_uid_t), &err); // Add to schema db - char id[12]; - char buf[256]; + char id[12]; + char buf[256]; void *pBuf = buf; *(tb_uid_t *)id = pTableObj->pTable->uid; *(int32_t *)(id + sizeof(tb_uid_t)) = schemaVersion(pTableOpts->pSchema); int size = tdEncodeSchema(&pBuf, pTableOpts->pSchema); - + rocksdb_put(pMeta->schemaDb, wopt, id, 12, buf, size, &err); rocksdb_writeoptions_destroy(wopt); diff --git a/source/server/vnode/meta/test/metaTests.cpp b/source/server/vnode/meta/test/metaTests.cpp index f81302fc91..47ca49b70a 100644 --- a/source/server/vnode/meta/test/metaTests.cpp +++ b/source/server/vnode/meta/test/metaTests.cpp @@ -31,7 +31,7 @@ TEST(MetaTest, meta_open_test) { metaClose(meta); std::cout << "Meta is closed!" << std::endl; - // // Destroy Meta - // metaDestroy("meta"); - // std::cout << "Meta is destroyed!" << std::endl; + // Destroy Meta + metaDestroy("meta"); + std::cout << "Meta is destroyed!" << std::endl; } \ No newline at end of file From 59de14eaa48dd95d3002275897b888ed0f9fb904 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 16:15:44 +0800 Subject: [PATCH 09/14] wrap tkv --- CMakeLists.txt | 2 +- include/libs/tkv/tkv.h | 29 +++++--- source/libs/tkv/CMakeLists.txt | 4 ++ source/libs/tkv/src/tkv.c | 69 +++++++++++++++++++- source/server/vnode/meta/CMakeLists.txt | 2 +- source/server/vnode/meta/src/meta.c | 65 +++++++++--------- source/server/vnode/meta/test/CMakeLists.txt | 2 +- 7 files changed, 127 insertions(+), 46 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f27f5f1672..65d1e133d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ if(${BUILD_WITH_LEVELDB}) endif(${BUILD_WITH_LEVELDB}) ## rocksdb -option(BUILD_WITH_ROCKSDB "If build with rocksdb" ON) +option(BUILD_WITH_ROCKSDB "If build with rocksdb" OFF) if(${BUILD_WITH_ROCKSDB}) cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${DEPS_TMP_FILE}) endif(${BUILD_WITH_ROCKSDB}) diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h index c31d655e43..d9cabc1e40 100644 --- a/include/libs/tkv/tkv.h +++ b/include/libs/tkv/tkv.h @@ -16,30 +16,43 @@ #ifndef _TD_TKV_H_ #define _TD_TKV_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif // Types exported -typedef struct STkvDb STkvDb; -typedef struct STkvOptions STkvOptions; -typedef struct STkvCache STkvCache; +typedef struct STkvDb STkvDb; +typedef struct STkvOpts STkvOpts; +typedef struct STkvCache STkvCache; +typedef struct STkvReadOpts STkvReadOpts; +typedef struct STkvWriteOpts STkvWriteOpts; // DB operations -STkvDb *tkvOpen(const STkvOptions *options, const char *path); +STkvDb *tkvOpen(const STkvOpts *options, const char *path); void tkvClose(STkvDb *db); -void tkvPut(STkvDb *db, void * /*TODO*/); +void tkvPut(STkvDb *db, STkvWriteOpts *, char *key, size_t keylen, char *val, size_t vallen); +char * tkvGet(STkvDb *db, STkvReadOpts *, char *key, size_t keylen, size_t *vallen); // DB options -STkvOptions *tkvOptionsCreate(); -void tkvOptionsDestroy(STkvOptions *); -void tkvOptionsSetCache(STkvOptions *, STkvCache *); +STkvOpts *tkvOptionsCreate(); +void tkvOptionsDestroy(STkvOpts *); +void tkvOptionsSetCache(STkvOpts *, STkvCache *); // DB cache typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType; STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type); void tkvCacheDestroy(STkvCache *); +// STkvReadOpts +STkvReadOpts *tkvReadOptsCreate(); +void tkvReadOptsDestroy(STkvReadOpts *); + +// STkvWriteOpts +STkvWriteOpts *tkvWriteOptsCreate(); +void tkvWriteOptsDestroy(STkvWriteOpts *); + #ifdef __cplusplus } #endif diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tkv/CMakeLists.txt index bec359d75d..6ad553b1ca 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tkv/CMakeLists.txt @@ -4,4 +4,8 @@ target_include_directories( tkv PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +target_link_libraries( + tkv + PUBLIC os ) \ No newline at end of file diff --git a/source/libs/tkv/src/tkv.c b/source/libs/tkv/src/tkv.c index 6dea4a4e57..9c78c23db4 100644 --- a/source/libs/tkv/src/tkv.c +++ b/source/libs/tkv/src/tkv.c @@ -11,4 +11,71 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "tkv.h" + +struct STkvDb { + // TODO +}; +struct STkvOpts { + // TODO +}; +struct STkvCache { + // TODO +}; +struct STkvReadOpts { + // TODO +}; +struct STkvWriteOpts { + // TODO +}; + +STkvDb *tkvOpen(const STkvOpts *options, const char *path) { + // TODO + return NULL; +} + +void tkvClose(STkvDb *db) { + // TODO +} + +void tkvPut(STkvDb *db, STkvWriteOpts *pwopts, char *key, size_t keylen, char *val, size_t vallen) { + // TODO +} + +char *tkvGet(STkvDb *db, STkvReadOpts *propts, char *key, size_t keylen, size_t *vallen) { + // TODO + return NULL; +} + +STkvOpts *tkvOptionsCreate() { + // TODO + return NULL; +} + +void tkvOptionsDestroy(STkvOpts *popts) { + // TODO +} + +void tkvOptionsSetCache(STkvOpts *popts, STkvCache *pCache) { + // TODO +} + +STkvReadOpts *tkvReadOptsCreate() { + // TODO + return NULL; +} + +void tkvReadOptsDestroy(STkvReadOpts *propts) { + // TODO +} + +STkvWriteOpts *tkvWriteOptsCreate() { + // TODO + return NULL; +} + +void tkvWriteOptsDestroy(STkvWriteOpts *pwopts) { + // TODO +} \ No newline at end of file diff --git a/source/server/vnode/meta/CMakeLists.txt b/source/server/vnode/meta/CMakeLists.txt index 94bf9581d3..0de78074ee 100644 --- a/source/server/vnode/meta/CMakeLists.txt +++ b/source/server/vnode/meta/CMakeLists.txt @@ -8,8 +8,8 @@ target_include_directories( target_link_libraries( meta PUBLIC common + PUBLIC tkv ) -target_link_libraries(meta PUBLIC rocksdb) if(${BUILD_TEST}) add_subdirectory(test) diff --git a/source/server/vnode/meta/src/meta.c b/source/server/vnode/meta/src/meta.c index 000b2ca9b0..c63aa31b30 100644 --- a/source/server/vnode/meta/src/meta.c +++ b/source/server/vnode/meta/src/meta.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include - +#include "tkv.h" #include "thash.h" #include "tlist.h" #include "tlockfree.h" @@ -44,13 +43,13 @@ typedef struct STableObj { struct SMeta { pthread_rwlock_t rwLock; - SHashObj * pTableObjHash; // uid --> STableObj - SList * stbList; // super table list - rocksdb_t *tbnameDb; // tbname --> uid - rocksdb_t *tagDb; // uid --> tag - rocksdb_t *schemaDb; - rocksdb_t *tagIdx; - size_t totalUsed; + SHashObj *pTableObjHash; // uid --> STableObj + SList * stbList; // super table list + STkvDb * tbnameDb; // tbname --> uid + STkvDb * tagDb; // uid --> tag + STkvDb * schemaDb; + STkvDb * tagIdx; + size_t totalUsed; }; static STable * metaTableNew(tb_uid_t uid, const char *name, int32_t sver); @@ -74,34 +73,33 @@ SMeta *metaOpen(SMetaOpts *options) { pMeta->stbList = tdListNew(sizeof(STableObj *)); // Options - rocksdb_options_t *dbOptions = rocksdb_options_create(); - rocksdb_options_set_create_if_missing(dbOptions, 1); + STkvOpts *dbOptions = tkvOptionsCreate(); taosMkDir("meta"); // Open tbname DB - pMeta->tbnameDb = rocksdb_open(dbOptions, "meta/tbname_uid_db", &err); + pMeta->tbnameDb = tkvOpen(dbOptions, "meta/tbname_uid_db"); // Open tag DB - pMeta->tagDb = rocksdb_open(dbOptions, "meta/uid_tag_db", &err); + pMeta->tagDb = tkvOpen(dbOptions, "meta/uid_tag_db"); // Open schema DB - pMeta->schemaDb = rocksdb_open(dbOptions, "meta/schema_db", &err); + pMeta->schemaDb = tkvOpen(dbOptions, "meta/schema_db"); // Open tag index - pMeta->tagIdx = rocksdb_open(dbOptions, "meta/tag_idx_db", &err); + pMeta->tagIdx = tkvOpen(dbOptions, "meta/tag_idx_db"); - rocksdb_options_destroy(dbOptions); + tkvOptionsDestroy(dbOptions); return pMeta; } void metaClose(SMeta *pMeta) { if (pMeta) { - rocksdb_close(pMeta->tagIdx); - rocksdb_close(pMeta->schemaDb); - rocksdb_close(pMeta->tagDb); - rocksdb_close(pMeta->tbnameDb); + tkvClose(pMeta->tagIdx); + tkvClose(pMeta->schemaDb); + tkvClose(pMeta->tagDb); + tkvClose(pMeta->tbnameDb); tdListFree(pMeta->stbList); taosHashCleanup(pMeta->pTableObjHash); @@ -110,22 +108,21 @@ void metaClose(SMeta *pMeta) { } int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { - size_t vallen; - char * err = NULL; - rocksdb_readoptions_t * ropt; - STableObj * pTableObj = NULL; - rocksdb_writeoptions_t *wopt; + size_t vallen; + STkvReadOpts *ropt; + STableObj * pTableObj = NULL; + STkvWriteOpts *wopt; // Check if table already exists - ropt = rocksdb_readoptions_create(); + ropt = tkvReadOptsCreate(); - char *uidStr = rocksdb_get(pMeta->tbnameDb, ropt, pTableOpts->name, strlen(pTableOpts->name), &vallen, &err); + char *uidStr = tkvGet(pMeta->tbnameDb, ropt, pTableOpts->name, strlen(pTableOpts->name), &vallen); if (uidStr != NULL) { // Has duplicate named table return -1; } - rocksdb_readoptions_destroy(ropt); + tkvReadOptsDestroy(ropt); // Create table obj pTableObj = metaTableObjNew(); @@ -144,12 +141,12 @@ int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { taosHashPut(pMeta->pTableObjHash, &(pTableObj->pTable->uid), sizeof(tb_uid_t), &pTableObj, sizeof(pTableObj)); - wopt = rocksdb_writeoptions_create(); - rocksdb_writeoptions_disable_WAL(wopt, 1); + wopt = tkvWriteOptsCreate(); + // rocksdb_writeoptions_disable_WAL(wopt, 1); // Add to tbname db - rocksdb_put(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), &pTableObj->pTable->uid, - sizeof(tb_uid_t), &err); + tkvPut(pMeta->tbnameDb, wopt, pTableOpts->name, strlen(pTableOpts->name), (char *)&pTableObj->pTable->uid, + sizeof(tb_uid_t)); // Add to schema db char id[12]; @@ -159,9 +156,9 @@ int metaCreateTable(SMeta *pMeta, STableOpts *pTableOpts) { *(int32_t *)(id + sizeof(tb_uid_t)) = schemaVersion(pTableOpts->pSchema); int size = tdEncodeSchema(&pBuf, pTableOpts->pSchema); - rocksdb_put(pMeta->schemaDb, wopt, id, 12, buf, size, &err); + tkvPut(pMeta->schemaDb, wopt, id, 12, buf, size); - rocksdb_writeoptions_destroy(wopt); + tkvWriteOptsDestroy(wopt); pthread_rwlock_unlock(&pMeta->rwLock); diff --git a/source/server/vnode/meta/test/CMakeLists.txt b/source/server/vnode/meta/test/CMakeLists.txt index ee16a28687..bca02c2907 100644 --- a/source/server/vnode/meta/test/CMakeLists.txt +++ b/source/server/vnode/meta/test/CMakeLists.txt @@ -14,8 +14,8 @@ target_link_libraries(metaTest os util common - rocksdb gtest_main + tkv ) enable_testing() add_test( From 2d8aadff21d756ab37433713d667a0e6a8f99fcb Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 12 Oct 2021 16:30:40 +0800 Subject: [PATCH 10/14] put taosGetTimestampSec in osTime --- source/common/src/ttime.c | 2 -- source/os/src/osTime.c | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 0f15dcc13b..0aa2aef4be 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -94,8 +94,6 @@ static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t time parseLocaltimeWithDst }; -int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } - int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { /* parse datatime string in with tz */ if (strnchr(timestr, 'T', len, false) != NULL) { diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index bf0585e86d..7c655c0251 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -63,4 +63,6 @@ FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { return gettimeofday(tv, NULL); } +int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } + #endif From 8ce34a9096a1f7f2e0a78f5eb54198b97bd8a43a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 12 Oct 2021 16:37:44 +0800 Subject: [PATCH 11/14] fix some compile warning --- include/common/tdataformat.h | 2 +- include/os/osDir.h | 2 +- source/common/src/tdataformat.c | 2 +- source/os/src/osDir.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index c4321eb9df..2eef9db064 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -87,7 +87,7 @@ typedef struct { #define schemaColAt(s, i) ((s)->columns + i) #define tdFreeSchema(s) tfree((s)) -STSchema *tdDupSchema(STSchema *pSchema); +STSchema *tdDupSchema(const STSchema *pSchema); int tdEncodeSchema(void **buf, STSchema *pSchema); void * tdDecodeSchema(void *buf, STSchema **pRSchema); diff --git a/include/os/osDir.h b/include/os/osDir.h index 32733753a8..3ee3be2c10 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -20,7 +20,7 @@ extern "C" { #endif -void taosRemoveDir(char *dirname); +void taosRemoveDir(const char *dirname); bool taosDirExist(char *dirname); bool taosMkDir(char *dirname); void taosRemoveOldFiles(char *dirname, int32_t keepDays); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 6e76e3a8d0..94d93b651a 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -48,7 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { /** * Duplicate the schema and return a new object */ -STSchema *tdDupSchema(STSchema *pSchema) { +STSchema *tdDupSchema(const STSchema *pSchema) { int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema); STSchema *tSchema = (STSchema *)malloc(tlen); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 839db811fd..17ab88edf6 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -34,7 +34,7 @@ #include #include -void taosRemoveDir(char *dirname) { +void taosRemoveDir(const char *dirname) { DIR *dir = opendir(dirname); if (dir == NULL) return; From c635e77d4e7112a3763723e83cb3eb6675c029be Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 13 Oct 2021 13:41:04 +0800 Subject: [PATCH 12/14] add vnode's dependency on wal --- source/server/vnode/inc/vnodeInt.h | 2 +- source/server/vnode/inc/vnodeMain.h | 3 ++- source/server/vnode/inc/vnodeWrite.h | 4 ++-- source/server/vnode/tq/CMakeLists.txt | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/source/server/vnode/inc/vnodeInt.h b/source/server/vnode/inc/vnodeInt.h index b2512b2892..70796c76bd 100644 --- a/source/server/vnode/inc/vnodeInt.h +++ b/source/server/vnode/inc/vnodeInt.h @@ -83,7 +83,7 @@ typedef struct { #if 0 SSyncCfg syncCfg; #endif - SWalCfg walCfg; + //SWalCfg walCfg; void * qMgmt; char * rootDir; tsem_t sem; diff --git a/source/server/vnode/inc/vnodeMain.h b/source/server/vnode/inc/vnodeMain.h index 093d07b013..0b41812215 100644 --- a/source/server/vnode/inc/vnodeMain.h +++ b/source/server/vnode/inc/vnodeMain.h @@ -16,10 +16,11 @@ #ifndef _TD_VNODE_MAIN_H_ #define _TD_VNODE_MAIN_H_ +#include "vnodeInt.h" + #ifdef __cplusplus extern "C" { #endif -#include "vnodeInt.h" int32_t vnodeInitMain(); void vnodeCleanupMain(); diff --git a/source/server/vnode/inc/vnodeWrite.h b/source/server/vnode/inc/vnodeWrite.h index 48acf750c1..4bbe0d88fa 100644 --- a/source/server/vnode/inc/vnodeWrite.h +++ b/source/server/vnode/inc/vnodeWrite.h @@ -27,7 +27,7 @@ taos_queue vnodeAllocWriteQueue(SVnode *pVnode); void vnodeFreeWriteQueue(taos_queue pQueue); void vnodeProcessWriteMsg(SRpcMsg *pRpcMsg); -int32_t vnodeProcessWalMsg(SVnode *pVnode, SWalHead *pHead); +//int32_t vnodeProcessWalMsg(SVnode *pVnode, SWalHead *pHead); void vnodeStartWrite(SVnode *pVnode); void vnodeStopWrite(SVnode *pVnode); @@ -37,4 +37,4 @@ void vnodeWaitWriteCompleted(SVnode *pVnode); } #endif -#endif /*_TD_VNODE_WRITE_H_*/ \ No newline at end of file +#endif /*_TD_VNODE_WRITE_H_*/ diff --git a/source/server/vnode/tq/CMakeLists.txt b/source/server/vnode/tq/CMakeLists.txt index 9577007400..0c15e23d33 100644 --- a/source/server/vnode/tq/CMakeLists.txt +++ b/source/server/vnode/tq/CMakeLists.txt @@ -3,10 +3,11 @@ add_library(tq ${TQ_SRC}) target_include_directories( tq PUBLIC "${CMAKE_SOURCE_DIR}/include/server/vnode/tq" + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/wal" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_SOURCE_DIR}/include/os" ) target_link_libraries( - os + wal ) From fd80f0b232f0ad16b04370537e38b09133e011a6 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 13 Oct 2021 13:45:19 +0800 Subject: [PATCH 13/14] add wal back to vnode --- source/server/vnode/inc/vnodeInt.h | 2 +- source/server/vnode/inc/vnodeWrite.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/server/vnode/inc/vnodeInt.h b/source/server/vnode/inc/vnodeInt.h index 70796c76bd..b2512b2892 100644 --- a/source/server/vnode/inc/vnodeInt.h +++ b/source/server/vnode/inc/vnodeInt.h @@ -83,7 +83,7 @@ typedef struct { #if 0 SSyncCfg syncCfg; #endif - //SWalCfg walCfg; + SWalCfg walCfg; void * qMgmt; char * rootDir; tsem_t sem; diff --git a/source/server/vnode/inc/vnodeWrite.h b/source/server/vnode/inc/vnodeWrite.h index 4bbe0d88fa..0bb670de5b 100644 --- a/source/server/vnode/inc/vnodeWrite.h +++ b/source/server/vnode/inc/vnodeWrite.h @@ -27,7 +27,7 @@ taos_queue vnodeAllocWriteQueue(SVnode *pVnode); void vnodeFreeWriteQueue(taos_queue pQueue); void vnodeProcessWriteMsg(SRpcMsg *pRpcMsg); -//int32_t vnodeProcessWalMsg(SVnode *pVnode, SWalHead *pHead); +int32_t vnodeProcessWalMsg(SVnode *pVnode, SWalHead *pHead); void vnodeStartWrite(SVnode *pVnode); void vnodeStopWrite(SVnode *pVnode); From dd9ac9bbb90011b81f49c170da85a48843c9a5fb Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 13 Oct 2021 14:25:11 +0800 Subject: [PATCH 14/14] refine wal interface --- include/libs/wal/wal.h | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 37cd263783..b6fd5a70d9 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -58,18 +58,25 @@ void walStop(twalh); void walClose(twalh); //write -int64_t walWrite(twalh, int8_t msgType, void* body, uint32_t bodyLen); -void walFsync(twalh, bool forceHint); -//int32_t walCommit(twalh, int64_t ver); -//int32_t walRollback(twalh, int64_t ver); +int64_t walWriteWithMsgType(twalh, int8_t msgType, void* body, int32_t bodyLen); +int64_t walWrite(twalh, void* body, int32_t bodyLen); +int64_t walWriteBatch(twalh, void* body, int32_t* bodyLen, int32_t batchSize); + +//apis for lifecycle management +void walFsync(twalh, bool force); +int32_t walCommit(twalh, int64_t ver); +//truncate after +int32_t walRollback(twalh, int64_t ver); +//notify that previous log can be pruned safely +int32_t walPrune(twalh, int64_t ver); //read int32_t walRead(twalh, SWalHead **, int64_t ver); int32_t walReadWithFp(twalh, FWalWrite writeFp, int64_t verStart, int readNum); -//life cycle -int32_t walDataPersisted(twalh, int64_t ver); +//lifecycle check int32_t walFirstVer(twalh); +int32_t walPersistedVer(twalh); int32_t walLastVer(twalh); //int32_t walDataCorrupted(twalh);