From cb470d7afe15bca43a03448eb7c4db185c67287d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 2 Dec 2021 18:10:06 +0800 Subject: [PATCH] [TD-11760]: use sqlite to store suid to uid list --- deps/CMakeLists.txt | 5 +- source/dnode/vnode/meta/CMakeLists.txt | 1 + source/dnode/vnode/meta/inc/metaDB.h | 3 +- source/dnode/vnode/meta/src/metaDB.c | 77 ++++++++++++++++---------- 4 files changed, 54 insertions(+), 32 deletions(-) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index afd606c403..0ef8c7e3be 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -82,7 +82,7 @@ endif(${BUILD_WITH_NURAFT}) # BDB if(${BUILD_WITH_BDB}) - add_library(bdb STATIC IMPORTED) + add_library(bdb STATIC IMPORTED GLOBAL) set_target_properties(bdb PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/bdb/libdb.a" INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/bdb" @@ -93,8 +93,9 @@ if(${BUILD_WITH_BDB}) endif(${BUILD_WITH_BDB}) # SQLite +# see https://stackoverflow.com/questions/8774593/cmake-link-to-external-library#comment58570736_10550334 if(${BUILD_WITH_SQLITE}) - add_library(sqlite STATIC IMPORTED) + add_library(sqlite STATIC IMPORTED GLOBAL) set_target_properties(sqlite PROPERTIES IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/sqlite/.libs/libsqlite3.a" INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/sqlite" diff --git a/source/dnode/vnode/meta/CMakeLists.txt b/source/dnode/vnode/meta/CMakeLists.txt index 8d37f5842e..34de051441 100644 --- a/source/dnode/vnode/meta/CMakeLists.txt +++ b/source/dnode/vnode/meta/CMakeLists.txt @@ -7,6 +7,7 @@ target_include_directories( ) target_link_libraries( meta + PUBLIC sqlite PUBLIC common PUBLIC tkv ) diff --git a/source/dnode/vnode/meta/inc/metaDB.h b/source/dnode/vnode/meta/inc/metaDB.h index f758af0673..babff30e94 100644 --- a/source/dnode/vnode/meta/inc/metaDB.h +++ b/source/dnode/vnode/meta/inc/metaDB.h @@ -17,6 +17,7 @@ #define _TD_META_DB_H_ #include "rocksdb/c.h" +#include "sqlite3.h" #include "meta.h" @@ -29,7 +30,7 @@ typedef struct { rocksdb_t *nameDb; // name -> uid rocksdb_t *tagDb; // uid -> tag rocksdb_t *schemaDb; // uid+version -> schema - rocksdb_t *mapDb; // suid -> uid_list + sqlite3 * mapDb; // suid -> uid_list } meta_db_t; int metaOpenDB(SMeta *pMeta); diff --git a/source/dnode/vnode/meta/src/metaDB.c b/source/dnode/vnode/meta/src/metaDB.c index d1fb65d2ed..eec0910c07 100644 --- a/source/dnode/vnode/meta/src/metaDB.c +++ b/source/dnode/vnode/meta/src/metaDB.c @@ -17,7 +17,7 @@ static void metaSaveSchemaDB(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema); static void metaGetSchemaDBKey(char key[], tb_uid_t uid, int sversion); -static int metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid); +// static int metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid); #define SCHEMA_KEY_LEN (sizeof(tb_uid_t) + sizeof(int)) @@ -65,8 +65,14 @@ int metaOpenDB(SMeta *pMeta) { META_OPEN_DB_IMPL(pMeta->pDB->schemaDb, options, dir, err); // mapDb - sprintf(dir, "%s/map_db", pMeta->path); - META_OPEN_DB_IMPL(pMeta->pDB->mapDb, options, dir, err); + sprintf(dir, "%s/meta.db", pMeta->path); + if (sqlite3_open(dir, &(pMeta->pDB->mapDb)) != SQLITE_OK) { + // TODO + } + + // // set read uncommitted + sqlite3_exec(pMeta->pDB->mapDb, "PRAGMA read_uncommitted=true;", 0, 0, 0); + sqlite3_exec(pMeta->pDB->mapDb, "BEGIN;", 0, 0, 0); rocksdb_options_destroy(options); return 0; @@ -82,7 +88,12 @@ int metaOpenDB(SMeta *pMeta) { void metaCloseDB(SMeta *pMeta) { if (pMeta->pDB) { - META_CLOSE_DB_IMPL(pMeta->pDB->mapDb); + if (pMeta->pDB->mapDb) { + sqlite3_exec(pMeta->pDB->mapDb, "COMMIT;", 0, 0, 0); + sqlite3_close(pMeta->pDB->mapDb); + pMeta->pDB->mapDb = NULL; + } + META_CLOSE_DB_IMPL(pMeta->pDB->schemaDb); META_CLOSE_DB_IMPL(pMeta->pDB->tagDb); META_CLOSE_DB_IMPL(pMeta->pDB->nameDb); @@ -97,6 +108,7 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { char * err = NULL; size_t size; char pBuf[1024]; // TODO + char sql[128]; rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create(); @@ -124,8 +136,12 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { // save schemaDB metaSaveSchemaDB(pMeta, uid, pTbOptions->stbCfg.pSchema); - // save mapDB (really need?) - rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&uid), sizeof(uid), "", 0, &err); + // // save mapDB (really need?) + // rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&uid), sizeof(uid), "", 0, &err); + sprintf(sql, "create table st_%" PRIu64 " (uid BIGINT);", uid); + if (sqlite3_exec(pMeta->pDB->mapDb, sql, NULL, NULL, &err) != SQLITE_OK) { + // fprintf(stderr, "Failed to create table, since %s\n", err); + } break; case META_CHILD_TABLE: // save tagDB @@ -133,7 +149,10 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { kvRowLen(pTbOptions->ctbCfg.pTag), &err); // save mapDB - metaSaveMapDB(pMeta, pTbOptions->ctbCfg.suid, uid); + sprintf(sql, "insert into st_%" PRIu64 " values (%" PRIu64 ");", pTbOptions->ctbCfg.suid, uid); + if (sqlite3_exec(pMeta->pDB->mapDb, sql, NULL, NULL, &err) != SQLITE_OK) { + fprintf(stderr, "failed to insert data, since %s\n", err); + } break; default: ASSERT(0); @@ -172,32 +191,32 @@ static void metaGetSchemaDBKey(char *key, tb_uid_t uid, int sversion) { *(int *)POINTER_SHIFT(key, sizeof(tb_uid_t)) = sversion; } -static int metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid) { - size_t vlen; - char * val; - char * err = NULL; +// static int metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid) { +// size_t vlen; +// char * val; +// char * err = NULL; - rocksdb_readoptions_t *ropt = rocksdb_readoptions_create(); - val = rocksdb_get(pMeta->pDB->mapDb, ropt, (char *)(&suid), sizeof(suid), &vlen, &err); - rocksdb_readoptions_destroy(ropt); +// rocksdb_readoptions_t *ropt = rocksdb_readoptions_create(); +// val = rocksdb_get(pMeta->pDB->mapDb, ropt, (char *)(&suid), sizeof(suid), &vlen, &err); +// rocksdb_readoptions_destroy(ropt); - void *nval = malloc(vlen + sizeof(uid)); - if (nval == NULL) { - return -1; - } +// void *nval = malloc(vlen + sizeof(uid)); +// if (nval == NULL) { +// return -1; +// } - if (vlen) { - memcpy(nval, val, vlen); - } - memcpy(POINTER_SHIFT(nval, vlen), (void *)(&uid), sizeof(uid)); +// if (vlen) { +// memcpy(nval, val, vlen); +// } +// memcpy(POINTER_SHIFT(nval, vlen), (void *)(&uid), sizeof(uid)); - rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create(); - rocksdb_writeoptions_disable_WAL(wopt, 1); +// rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create(); +// rocksdb_writeoptions_disable_WAL(wopt, 1); - rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&suid), sizeof(suid), nval, vlen + sizeof(uid), &err); +// rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&suid), sizeof(suid), nval, vlen + sizeof(uid), &err); - rocksdb_writeoptions_destroy(wopt); - free(nval); +// rocksdb_writeoptions_destroy(wopt); +// free(nval); - return 0; -} \ No newline at end of file +// return 0; +// } \ No newline at end of file