From 4bf754b1bce7707c0bf88aeff548161c1986e667 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 3 Dec 2021 18:27:16 +0800 Subject: [PATCH 01/27] try to use SQLite --- source/dnode/vnode/impl/CMakeLists.txt | 2 +- source/dnode/vnode/meta/inc/metaDB.h | 6 + source/dnode/vnode/meta/src/metaDB.c | 169 +++++++++++++++++++------ source/dnode/vnode/meta/src/metaIdx.c | 4 + 4 files changed, 138 insertions(+), 43 deletions(-) diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 3886308ec9..040b02d2b6 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -19,5 +19,5 @@ target_link_libraries( # test if(${BUILD_TEST}) - # add_subdirectory(test) + add_subdirectory(test) endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/meta/inc/metaDB.h b/source/dnode/vnode/meta/inc/metaDB.h index babff30e94..c61dd1e20f 100644 --- a/source/dnode/vnode/meta/inc/metaDB.h +++ b/source/dnode/vnode/meta/inc/metaDB.h @@ -16,6 +16,8 @@ #ifndef _TD_META_DB_H_ #define _TD_META_DB_H_ +#define USE_SQLITE_IMPL 1 + #include "rocksdb/c.h" #include "sqlite3.h" @@ -25,6 +27,7 @@ extern "C" { #endif +#if !USE_SQLITE_IMPL typedef struct { rocksdb_t *tbDb; // uid -> tb obj rocksdb_t *nameDb; // name -> uid @@ -32,6 +35,9 @@ typedef struct { rocksdb_t *schemaDb; // uid+version -> schema sqlite3 * mapDb; // suid -> uid_list } meta_db_t; +#else +typedef sqlite3 meta_db_t; +#endif int metaOpenDB(SMeta *pMeta); void metaCloseDB(SMeta *pMeta); diff --git a/source/dnode/vnode/meta/src/metaDB.c b/source/dnode/vnode/meta/src/metaDB.c index eec0910c07..7ad2b82d30 100644 --- a/source/dnode/vnode/meta/src/metaDB.c +++ b/source/dnode/vnode/meta/src/metaDB.c @@ -15,9 +15,9 @@ #include "metaDef.h" +#if !USE_SQLITE_IMPL static void metaSaveSchemaDB(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema); static void metaGetSchemaDBKey(char key[], tb_uid_t uid, int sversion); -// static int metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid); #define SCHEMA_KEY_LEN (sizeof(tb_uid_t) + sizeof(int)) @@ -31,9 +31,20 @@ static void metaGetSchemaDBKey(char key[], tb_uid_t uid, int sversion); } \ } while (0) +#define META_CLOSE_DB_IMPL(pDB) \ + do { \ + if (pDB) { \ + rocksdb_close(pDB); \ + pDB = NULL; \ + } \ + } while (0) +#endif + int metaOpenDB(SMeta *pMeta) { - char dir[128]; - char * err = NULL; + char dir[128]; + int rc; + char *err = NULL; +#if !USE_SQLITE_IMPL rocksdb_options_t *options = rocksdb_options_create(); if (pMeta->pCache) { @@ -75,18 +86,63 @@ int metaOpenDB(SMeta *pMeta) { sqlite3_exec(pMeta->pDB->mapDb, "BEGIN;", 0, 0, 0); rocksdb_options_destroy(options); +#else + sprintf(dir, "%s/meta.db", pMeta->path); + rc = sqlite3_open(dir, &(pMeta->pDB)); + if (rc != SQLITE_OK) { + // TODO: handle error + printf("failed to open meta.db\n"); + } + + // For all tables + rc = sqlite3_exec(pMeta->pDB, + "CREATE TABLE IF NOT EXISTS tb (" + " tbname VARCHAR(256) NOT NULL UNIQUE," + " tb_uid INTEGER NOT NULL UNIQUE " + ");", + NULL, NULL, &err); + if (rc != SQLITE_OK) { + // TODO: handle error + printf("failed to create meta table tb since %s\n", err); + } + + // For super tables + rc = sqlite3_exec(pMeta->pDB, + "CREATE TABLE IF NOT EXISTS stb (" + " tb_uid INTEGER NOT NULL UNIQUE," + " tbname VARCHAR(256) NOT NULL UNIQUE," + " tb_schema BLOB NOT NULL," + " tag_schema BLOB NOT NULL" + ");", + NULL, NULL, &err); + if (rc != SQLITE_OK) { + // TODO: handle error + printf("failed to create meta table stb since %s\n", err); + } + + // For normal tables + rc = sqlite3_exec(pMeta->pDB, + "CREATE TABLE IF NOT EXISTS ntb (" + " tb_uid INTEGER NOT NULL UNIQUE," + " tbname VARCHAR(256) NOT NULL," + " tb_schema BLOB NOT NULL" + ");", + NULL, NULL, &err); + if (rc != SQLITE_OK) { + // TODO: handle error + printf("failed to create meta table ntb since %s\n", err); + } + + sqlite3_exec(pMeta->pDB, "BEGIN;", NULL, NULL, &err); + + tfree(err); +#endif + return 0; } -#define META_CLOSE_DB_IMPL(pDB) \ - do { \ - if (pDB) { \ - rocksdb_close(pDB); \ - pDB = NULL; \ - } \ - } while (0) - void metaCloseDB(SMeta *pMeta) { +#if !USE_SQLITE_IMPL if (pMeta->pDB) { if (pMeta->pDB->mapDb) { sqlite3_exec(pMeta->pDB->mapDb, "COMMIT;", 0, 0, 0); @@ -101,9 +157,19 @@ void metaCloseDB(SMeta *pMeta) { free(pMeta->pDB); pMeta->pDB = NULL; } +#else + if (pMeta->pDB) { + sqlite3_exec(pMeta->pDB, "BEGIN;", NULL, NULL, NULL); + sqlite3_close(pMeta->pDB); + pMeta->pDB = NULL; + } + + // TODO +#endif } int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { +#if !USE_SQLITE_IMPL tb_uid_t uid; char * err = NULL; size_t size; @@ -140,7 +206,7 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { // rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&uid), sizeof(uid), "", 0, &err); sprintf(sql, "create table st_%" PRIu64 " (uid BIGINT);", uid); if (sqlite3_exec(pMeta->pDB->mapDb, sql, NULL, NULL, &err) != SQLITE_OK) { - // fprintf(stderr, "Failed to create table, since %s\n", err); + // fprintf(stderr,"Failed to create table, since %s\n", err); } break; case META_CHILD_TABLE: @@ -159,6 +225,53 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { } rocksdb_writeoptions_destroy(wopt); +#else + char sql[256]; + char *err = NULL; + int rc; + + switch (pTbOptions->type) { + case META_SUPER_TABLE: + // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 + // ");" + // "INSERT INTO stb VALUES (%" PRIu64 + // ", \'%s\', );" + // "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 + // " (" + // " tb_uid INTEGER NOT NULL UNIQUE," + // " tbname VARCHAR(256)," + // " tag1 INTEGER" + // ");" + // "CREATE INDEX IF NOT EXISTS stb_%" PRIu64 "_tag1_idx ON stb_1638517480 (tag1);"); + rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); + if (rc != SQLITE_OK) { + printf("failed to create normal table since %s\n", err); + } + break; + case META_NORMAL_TABLE: + // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 + // ");" + // "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );"); + rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); + if (rc != SQLITE_OK) { + printf("failed to create normal table since %s\n", err); + } + break; + case META_CHILD_TABLE: + // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 + // ");" + // "INSERT INTO stb_%" PRIu64 " VALUES (%" PRIu64 ", \'%s\', );"); + rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); + if (rc != SQLITE_OK) { + printf("failed to create normal table since %s\n", err); + } + break; + default: + break; + } + + tfree(err); +#endif return 0; } @@ -169,6 +282,7 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { } /* ------------------------ STATIC METHODS ------------------------ */ +#if !USE_SQLITE_IMPL static void metaSaveSchemaDB(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema) { char key[64]; char pBuf[1024]; @@ -190,33 +304,4 @@ static void metaGetSchemaDBKey(char *key, tb_uid_t uid, int sversion) { *(tb_uid_t *)key = uid; *(int *)POINTER_SHIFT(key, sizeof(tb_uid_t)) = sversion; } - -// static int metaSaveMapDB(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid) { -// size_t vlen; -// char * val; -// char * err = NULL; - -// rocksdb_readoptions_t *ropt = rocksdb_readoptions_create(); -// val = rocksdb_get(pMeta->pDB->mapDb, ropt, (char *)(&suid), sizeof(suid), &vlen, &err); -// rocksdb_readoptions_destroy(ropt); - -// void *nval = malloc(vlen + sizeof(uid)); -// if (nval == NULL) { -// return -1; -// } - -// if (vlen) { -// memcpy(nval, val, vlen); -// } -// memcpy(POINTER_SHIFT(nval, vlen), (void *)(&uid), sizeof(uid)); - -// rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create(); -// rocksdb_writeoptions_disable_WAL(wopt, 1); - -// rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&suid), sizeof(suid), nval, vlen + sizeof(uid), &err); - -// rocksdb_writeoptions_destroy(wopt); -// free(nval); - -// return 0; -// } \ No newline at end of file +#endif \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaIdx.c b/source/dnode/vnode/meta/src/metaIdx.c index a3c0d3540e..078c7e7d08 100644 --- a/source/dnode/vnode/meta/src/metaIdx.c +++ b/source/dnode/vnode/meta/src/metaIdx.c @@ -16,6 +16,7 @@ #include "metaDef.h" int metaOpenIdx(SMeta *pMeta) { +#if 0 char idxDir[128]; // TODO char * err = NULL; rocksdb_options_t *options = rocksdb_options_create(); @@ -36,15 +37,18 @@ int metaOpenIdx(SMeta *pMeta) { } rocksdb_options_destroy(options); +#endif return 0; } void metaCloseIdx(SMeta *pMeta) { /* TODO */ +#if 0 if (pMeta->pIdx) { rocksdb_close(pMeta->pIdx); pMeta->pIdx = NULL; } +#endif } int metaSaveTableToIdx(SMeta *pMeta, const STbCfg *pTbOptions) { From d86a5ca3e894e0468a218f195bc7807ae2de35bf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 3 Dec 2021 18:28:34 +0800 Subject: [PATCH 02/27] refact --- source/dnode/vnode/meta/src/metaMain.c | 187 +------------------------ 1 file changed, 1 insertion(+), 186 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaMain.c b/source/dnode/vnode/meta/src/metaMain.c index a936002328..53055dcea3 100644 --- a/source/dnode/vnode/meta/src/metaMain.c +++ b/source/dnode/vnode/meta/src/metaMain.c @@ -129,189 +129,4 @@ static void metaCloseImpl(SMeta *pMeta) { metaCloseIdx(pMeta); metaCloseDB(pMeta); metaCloseCache(pMeta); -} - -// OLD ------------------------------------------------------------------- -#if 0 -static int metaCreateSuperTable(SMeta *pMeta, const char *tbname, const SSuperTableOpts *pSuperTableOpts); -static int metaCreateChildTable(SMeta *pMeta, const char *tbname, const SChildTableOpts *pChildTableOpts); -static int metaCreateNormalTable(SMeta *pMeta, const char *tbname, const SNormalTableOpts *pNormalTableOpts); - -int metaCreateTable(SMeta *pMeta, const STableOptions *pTableOpts) { - size_t vallen; - char * pUid; - - // Check if table already exists - pUid = tkvGet(pMeta->tbnameDb, NULL, pTableOpts->name, strlen(pTableOpts->name), &vallen); - if (pUid) { - free(pUid); - // Table already exists, return error code - return -1; - } - - switch (pTableOpts->type) { - case META_SUPER_TABLE: - return metaCreateSuperTable(pMeta, pTableOpts->name, &(pTableOpts->superOpts)); - case META_CHILD_TABLE: - return metaCreateChildTable(pMeta, pTableOpts->name, &(pTableOpts->childOpts)); - case META_NORMAL_TABLE: - return metaCreateNormalTable(pMeta, pTableOpts->name, &(pTableOpts->normalOpts)); - default: - ASSERT(0); - } - - return 0; -} - -static int metaCreateSuperTable(SMeta *pMeta, const char *tbname, const SSuperTableOpts *pSuperTableOpts) { - size_t vallen; - size_t keylen; - char * pVal = NULL; - char schemaKey[sizeof(tb_uid_t) * 2]; - char buffer[1024]; /* TODO */ - void * pBuf = NULL; - - pVal = tkvGet(pMeta->tableDb, NULL, (char *)(&(pSuperTableOpts->uid)), sizeof(pSuperTableOpts->uid), &vallen); - if (pVal) { - free(pVal); - // TODO: table with same uid exists, just return error - return -1; - } - - // Put uid -> tbObj - vallen = 0; - pBuf = (void *)buffer; - vallen += taosEncodeString(&pBuf, tbname); // ENCODE TABLE NAME - vallen += taosEncodeFixedI32(&pBuf, 1); // ENCODE SCHEMA, SCHEMA HERE IS AN ARRAY OF VERSIONS - vallen += taosEncodeFixedI32(&pBuf, schemaVersion(pSuperTableOpts->pSchema)); - vallen += tdEncodeSchema(&pBuf, pSuperTableOpts->pTagSchema); // ENCODE TAG SCHEMA - tkvPut(pMeta->tableDb, NULL, (char *)(&(pSuperTableOpts->uid)), sizeof(pSuperTableOpts->uid), buffer, vallen); - - // Put tbname -> uid - tkvPut(pMeta->tbnameDb, NULL, tbname, strlen(tbname), (char *)(&(pSuperTableOpts->uid)), - sizeof(pSuperTableOpts->uid)); - - // Put uid+sversion -> schema - *(tb_uid_t *)schemaKey = pSuperTableOpts->uid; - *(int32_t *)(POINTER_SHIFT(schemaKey, sizeof(tb_uid_t))) = schemaVersion(pSuperTableOpts->pSchema); - keylen = sizeof(tb_uid_t) + sizeof(int32_t); - pBuf = (void *)buffer; - vallen = tdEncodeSchema(&pBuf, pSuperTableOpts->pSchema); - tkvPut(pMeta->schemaDb, NULL, schemaKey, keylen, buffer, vallen); - - return 0; -} - -static int metaCreateChildTable(SMeta *pMeta, const char *tbname, const SChildTableOpts *pChildTableOpts) { - size_t vallen; - char buffer[1024]; /* TODO */ - void * pBuf = NULL; - char * pTable; - tb_uid_t uid; - - // Check if super table exists - pTable = tkvGet(pMeta->tableDb, NULL, (char *)(&(pChildTableOpts->suid)), sizeof(pChildTableOpts->suid), &vallen); - if (pTable == NULL) { - // Super table not exists, just return error - return -1; - } - - // Generate a uid to the new table - uid = generateUid(&pMeta->uidGenerator); - - // Put uid -> tbObj - vallen = 0; - pBuf = (void *)buffer; - vallen += taosEncodeString(&pBuf, tbname); - vallen += taosEncodeFixedU64(&pBuf, pChildTableOpts->suid); - tkvPut(pMeta->tableDb, NULL, (char *)(&uid), sizeof(uid), buffer, vallen); - - // Put tbname -> uid - tkvPut(pMeta->tbnameDb, NULL, tbname, strlen(tbname), (char *)(&uid), sizeof(uid)); - - // Put uid-> tags - tkvPut(pMeta->tagDb, NULL, (char *)(&uid), sizeof(uid), (char *)pChildTableOpts->tags, - (size_t)kvRowLen(pChildTableOpts->tags)); - - // TODO: Put tagIdx - - return 0; -} - -static int metaCreateNormalTable(SMeta *pMeta, const char *tbname, const SNormalTableOpts *pNormalTableOpts) { - size_t vallen; - char keyBuf[sizeof(tb_uid_t) + sizeof(int32_t)]; - char buffer[1024]; /* TODO */ - void * pBuf = NULL; - tb_uid_t uid; - - // Generate a uid to the new table - uid = generateUid(&pMeta->uidGenerator); - - // Put uid -> tbObj - vallen = 0; - pBuf = (void *)buffer; - vallen += taosEncodeString(&pBuf, tbname); - vallen += taosEncodeFixedI32(&pBuf, 1); - vallen += taosEncodeFixedI32(&pBuf, schemaVersion(pNormalTableOpts->pSchema)); - tkvPut(pMeta->tableDb, NULL, (char *)(&uid), sizeof(uid), buffer, vallen); - - // Put tbname -> uid - tkvPut(pMeta->tbnameDb, NULL, tbname, strlen(tbname), (char *)(&(uid)), sizeof(uid)); - - // Put uid+sversion -> schema - vallen = 0; - pBuf = (void *)buffer; - vallen += tdEncodeSchema(&pBuf, pNormalTableOpts->pSchema); - tkvPut(pMeta->schemaDb, NULL, keyBuf, sizeof(tb_uid_t) + sizeof(int32_t), buffer, vallen); - - return 0; -} - -void metaNormalTableOptsInit(STableOptions *pTableOpts, const char *name, const STSchema *pSchema) { - pTableOpts->type = META_NORMAL_TABLE; - pTableOpts->name = strdup(name); - pTableOpts->normalOpts.pSchema = tdDupSchema(pSchema); -} - -void metaSuperTableOptsInit(STableOptions *pTableOpts, const char *name, tb_uid_t uid, const STSchema *pSchema, - const STSchema *pTagSchema) { - pTableOpts->type = META_SUPER_TABLE; - pTableOpts->name = strdup(name); - pTableOpts->superOpts.uid = uid; - pTableOpts->superOpts.pSchema = tdDupSchema(pSchema); - pTableOpts->superOpts.pTagSchema = tdDupSchema(pTagSchema); -} - -void metaChildTableOptsInit(STableOptions *pTableOpts, const char *name, tb_uid_t suid, const SKVRow tags) { - pTableOpts->type = META_CHILD_TABLE; - pTableOpts->name = strdup(name); - pTableOpts->childOpts.suid = suid; - pTableOpts->childOpts.tags = tdKVRowDup(tags); -} - -void metaTableOptsClear(STableOptions *pTableOpts) { - switch (pTableOpts->type) { - case META_NORMAL_TABLE: - tfree(pTableOpts->name); - tdFreeSchema(pTableOpts->normalOpts.pSchema); - break; - case META_SUPER_TABLE: - tdFreeSchema(pTableOpts->superOpts.pTagSchema); - tdFreeSchema(pTableOpts->superOpts.pSchema); - tfree(pTableOpts->name); - break; - case META_CHILD_TABLE: - kvRowFree(pTableOpts->childOpts.tags); - tfree(pTableOpts->name); - break; - default: - break; - } - - memset(pTableOpts, 0, sizeof(*pTableOpts)); -} - -void metaDestroy(const char *path) { taosRemoveDir(path); } - -#endif \ No newline at end of file +} \ No newline at end of file From 0953c45dccfea8c931f2708327c13b64584edd04 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Dec 2021 15:40:22 +0800 Subject: [PATCH 03/27] fix log --- source/dnode/vnode/meta/src/metaDB.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/meta/src/metaDB.c b/source/dnode/vnode/meta/src/metaDB.c index 7ad2b82d30..302e5dc6ae 100644 --- a/source/dnode/vnode/meta/src/metaDB.c +++ b/source/dnode/vnode/meta/src/metaDB.c @@ -263,7 +263,7 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { // "INSERT INTO stb_%" PRIu64 " VALUES (%" PRIu64 ", \'%s\', );"); rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); if (rc != SQLITE_OK) { - printf("failed to create normal table since %s\n", err); + printf("failed to create child table since %s\n", err); } break; default: From 4bcc7778db9947f39d2ddfac715deb74e0d02475 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Dec 2021 16:24:30 +0800 Subject: [PATCH 04/27] more --- contrib/test/bdb/bdbTest.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/contrib/test/bdb/bdbTest.c b/contrib/test/bdb/bdbTest.c index ab80b33fd0..ce249a457d 100644 --- a/contrib/test/bdb/bdbTest.c +++ b/contrib/test/bdb/bdbTest.c @@ -6,23 +6,24 @@ // refer: https://docs.oracle.com/cd/E17076_05/html/gsg/C/BerkeleyDB-Core-C-GSG.pdf int main(int argc, char const *argv[]) { - DB * db; - int ret; - uint32_t flags; + DB * dbp; + int ret; + u_int32_t flags; + DBT key, value; - ret = db_create(&db, NULL, 0); + ret = db_create(&dbp, NULL, 0); if (ret != 0) { exit(1); } flags = DB_CREATE; - ret = db->open(db, NULL, "test.db", NULL, DB_BTREE, flags, 0); + ret = dbp->open(dbp, NULL, "meta.db", NULL, DB_BTREE, flags, 0); if (ret != 0) { exit(1); } - db->close(db, 0); + dbp->close(dbp, 0); return 0; } From 4cf826825ddbc6d2eeb17e94255249fdeb88b6dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Dec 2021 20:38:41 +0800 Subject: [PATCH 05/27] bdb test --- contrib/test/bdb/bdbTest.c | 56 ++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/contrib/test/bdb/bdbTest.c b/contrib/test/bdb/bdbTest.c index ce249a457d..3780227ec6 100644 --- a/contrib/test/bdb/bdbTest.c +++ b/contrib/test/bdb/bdbTest.c @@ -5,25 +5,65 @@ // refer: https://docs.oracle.com/cd/E17076_05/html/gsg/C/BerkeleyDB-Core-C-GSG.pdf +// Access Methods: +// 1. BTree +// 2. Hash +// 3. Queue +// 4. Recno + +#define USE_ENV 0 + int main(int argc, char const *argv[]) { - DB * dbp; + DB * dbp = NULL; + u_int32_t db_flags; + DB_ENV * envp = NULL; + u_int32_t env_flags; int ret; - u_int32_t flags; - DBT key, value; + DBT key, value; // Database records - ret = db_create(&dbp, NULL, 0); +#if USE_ENV + // Initialize an env object and open it for + ret = db_env_create(&envp, 0); + if (ret != 0) { + fprintf(stderr, "Error creating env handle: %s\n", db_strerror(ret)); + return -1; + } + + env_flags = DB_CREATE | DB_INIT_MPOOL; + + ret = envp->open(envp, "./meta", env_flags, 0); + if (ret != 0) { + fprintf(stderr, "Error opening env handle: %s\n", db_strerror(ret)); + return -1; + } +#endif + + // Initialize a DB handle and open the DB + ret = db_create(&dbp, envp, 0); if (ret != 0) { exit(1); } - flags = DB_CREATE; - - ret = dbp->open(dbp, NULL, "meta.db", NULL, DB_BTREE, flags, 0); + db_flags = DB_CREATE; + ret = dbp->open(dbp, /* DB structure pointer */ + NULL, /* Transaction pointer */ + "meta.db", /* On-disk file that holds the database */ + NULL, /* Optional logical database name */ + DB_BTREE, /* Database access method */ + db_flags, /* Open flags */ + 0); /* File mode */ if (ret != 0) { exit(1); } - dbp->close(dbp, 0); + // Close the database + if (dbp != NULL) { + dbp->close(dbp, 0); + } + + if (envp != NULL) { + envp->close(envp, 0); + } return 0; } From 37405fa0e962a1cd96c762fd9be71db68dd932c6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Dec 2021 21:15:56 +0800 Subject: [PATCH 06/27] more --- contrib/test/bdb/bdbTest.c | 47 +++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/contrib/test/bdb/bdbTest.c b/contrib/test/bdb/bdbTest.c index 3780227ec6..0d89605202 100644 --- a/contrib/test/bdb/bdbTest.c +++ b/contrib/test/bdb/bdbTest.c @@ -1,5 +1,6 @@ #include #include +#include #include "db.h" @@ -13,13 +14,51 @@ #define USE_ENV 0 +#define DESCRIPTION_SIZE 128 +float money = 122.45; +char *description = "Grocery bill."; + +static void put_value(DB *dbp) { + DBT key = {0}; + DBT value = {0}; + int ret; + + key.data = &money; + key.size = sizeof(money); + + value.data = description; + value.size = strlen(description) + 1; + + ret = dbp->put(dbp, NULL, &key, &value, DB_NOOVERWRITE); + if (ret != 0) { + fprintf(stderr, "Failed to put DB record: %s\n", db_strerror(ret)); + } +} + +static void get_value(DB *dbp) { + char desp[DESCRIPTION_SIZE]; + DBT key = {0}; + DBT value = {0}; + + key.data = &money; + key.size = sizeof(money); + + value.data = desp; + value.ulen = DESCRIPTION_SIZE; + value.flags = DB_DBT_USERMEM; + + dbp->get(dbp, NULL, &key, &value, 0); + printf("The value is \"%s\"\n", value.data); +} + int main(int argc, char const *argv[]) { DB * dbp = NULL; u_int32_t db_flags; DB_ENV * envp = NULL; u_int32_t env_flags; int ret; - DBT key, value; // Database records + DBT key = {0}; + DBT value = {0}; #if USE_ENV // Initialize an env object and open it for @@ -56,6 +95,12 @@ int main(int argc, char const *argv[]) { exit(1); } + // Insert a key-value record + put_value(dbp); + + // Read the key-value record + get_value(dbp); + // Close the database if (dbp != NULL) { dbp->close(dbp, 0); From d27914960913376726979738414b3ad5c12238d0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 5 Dec 2021 21:36:48 +0800 Subject: [PATCH 07/27] more --- contrib/test/bdb/bdbTest.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/contrib/test/bdb/bdbTest.c b/contrib/test/bdb/bdbTest.c index 0d89605202..96ea9ee1fc 100644 --- a/contrib/test/bdb/bdbTest.c +++ b/contrib/test/bdb/bdbTest.c @@ -18,6 +18,12 @@ float money = 122.45; char *description = "Grocery bill."; +typedef struct { + int id; + char *family_name; + char *surname; +} SPersion; + static void put_value(DB *dbp) { DBT key = {0}; DBT value = {0}; @@ -37,8 +43,8 @@ static void put_value(DB *dbp) { static void get_value(DB *dbp) { char desp[DESCRIPTION_SIZE]; - DBT key = {0}; - DBT value = {0}; + DBT key = {0}; + DBT value = {0}; key.data = &money; key.size = sizeof(money); @@ -83,7 +89,7 @@ int main(int argc, char const *argv[]) { exit(1); } - db_flags = DB_CREATE; + db_flags = DB_CREATE | DB_TRUNCATE; ret = dbp->open(dbp, /* DB structure pointer */ NULL, /* Transaction pointer */ "meta.db", /* On-disk file that holds the database */ From 26e1504ac4a383bf9b64a097f2567ba6cf656859 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 6 Dec 2021 10:14:30 +0800 Subject: [PATCH 08/27] more --- contrib/test/bdb/bdbTest.c | 78 +++++++++++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/contrib/test/bdb/bdbTest.c b/contrib/test/bdb/bdbTest.c index 96ea9ee1fc..cbdc07be91 100644 --- a/contrib/test/bdb/bdbTest.c +++ b/contrib/test/bdb/bdbTest.c @@ -12,6 +12,11 @@ // 3. Queue // 4. Recno +// Use secondary database work as index. +// Any attemp to write to a secondary database results in a non-zero status return. + +static int idx_callback(DB *dbp, const DBT *keyp, const DBT *valuep, DBT *resvp); + #define USE_ENV 0 #define DESCRIPTION_SIZE 128 @@ -54,11 +59,12 @@ static void get_value(DB *dbp) { value.flags = DB_DBT_USERMEM; dbp->get(dbp, NULL, &key, &value, 0); - printf("The value is \"%s\"\n", value.data); + printf("The value is \"%s\"\n", desp); } int main(int argc, char const *argv[]) { DB * dbp = NULL; + DB * sdbp = NULL; u_int32_t db_flags; DB_ENV * envp = NULL; u_int32_t env_flags; @@ -89,6 +95,16 @@ int main(int argc, char const *argv[]) { exit(1); } + ret = db_create(&sdbp, envp, 0); + if (ret != 0) { + exit(1); + } + + ret = sdbp->set_flags(sdbp, DB_DUPSORT); + if (ret != 0) { + exit(1); + } + db_flags = DB_CREATE | DB_TRUNCATE; ret = dbp->open(dbp, /* DB structure pointer */ NULL, /* Transaction pointer */ @@ -101,13 +117,37 @@ int main(int argc, char const *argv[]) { exit(1); } - // Insert a key-value record - put_value(dbp); + ret = sdbp->open(sdbp, /* DB structure pointer */ + NULL, /* Transaction pointer */ + "index.db", /* On-disk file that holds the database */ + NULL, /* Optional logical database name */ + DB_BTREE, /* Database access method */ + db_flags, /* Open flags */ + 0); /* File mode */ + if (ret != 0) { + exit(1); + } - // Read the key-value record - get_value(dbp); + // Associate the secondary database to the primary + dbp->associate(dbp, /* Primary database */ + NULL, /* TXN id */ + sdbp, /* Secondary database */ + idx_callback, /* Callback used for key creation */ + 0); /* Flags */ + + { + // Insert a key-value record + put_value(dbp); + + // Read the key-value record + get_value(dbp); + } // Close the database + if (sdbp != NULL) { + sdbp->close(sdbp, 0); + } + if (dbp != NULL) { dbp->close(dbp, 0); } @@ -118,3 +158,31 @@ int main(int argc, char const *argv[]) { return 0; } + +static int idx_callback(DB * sdbp, /* secondary db handle */ + const DBT *keyp, /* primary db record's key */ + const DBT *valuep, /* primary db record's value */ + DBT * skeyp /* secondary db record's key*/ +) { + DBT *tmpdbt; + + tmpdbt = (DBT *)calloc(2, sizeof(DBT)); + + { // TODO + tmpdbt[0].data = NULL; + tmpdbt[0].size = 0; + + tmpdbt[1].data = NULL; + tmpdbt[1].size = 0; + } + + /** + * DB_DBT_MULTIPLE means DBT references an array + * DB_DBT_APPMALLOC means we dynamically allocated memory for the DBT's data field. + */ + skeyp->flags = DB_DBT_MULTIPLE | DB_DBT_APPMALLOC; + skeyp->size = 2; + skeyp->data = tmpdbt; + + return 0; +} \ No newline at end of file From c66a3ba0ee11a45335e485b2312dfd37702ab01c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 6 Dec 2021 15:23:53 +0800 Subject: [PATCH 09/27] use bdb to implement meta --- source/dnode/vnode/meta/CMakeLists.txt | 33 +- source/dnode/vnode/meta/inc/metaCache.h | 4 +- source/dnode/vnode/meta/inc/metaDB.h | 17 +- source/dnode/vnode/meta/inc/metaDef.h | 6 +- source/dnode/vnode/meta/inc/metaIdx.h | 4 +- source/dnode/vnode/meta/src/metaBDBImpl.c | 92 ++++++ source/dnode/vnode/meta/src/metaCache.c | 26 +- source/dnode/vnode/meta/src/metaDB.c | 307 ------------------- source/dnode/vnode/meta/src/metaIdx.c | 4 + source/dnode/vnode/meta/src/metaSQLiteImpl.c | 140 +++++++++ 10 files changed, 288 insertions(+), 345 deletions(-) create mode 100644 source/dnode/vnode/meta/src/metaBDBImpl.c delete mode 100644 source/dnode/vnode/meta/src/metaDB.c create mode 100644 source/dnode/vnode/meta/src/metaSQLiteImpl.c diff --git a/source/dnode/vnode/meta/CMakeLists.txt b/source/dnode/vnode/meta/CMakeLists.txt index 34de051441..7e5acc6450 100644 --- a/source/dnode/vnode/meta/CMakeLists.txt +++ b/source/dnode/vnode/meta/CMakeLists.txt @@ -1,4 +1,22 @@ +set(META_DB_IMPL_LIST "BDB" "SQLITE") +set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") +set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) + +if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) + message(STATUS "META DB Impl: ${META_DB_IMPL}==============") +else() + message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============") +endif() + aux_source_directory(src META_SRC) +if(${META_DB_IMPL} STREQUAL "BDB") + list(REMOVE_ITEM META_SRC "src/metaSQLiteImpl.c") +elseif(${META_DB_IMPL} STREQUAL "SQLITE") + list(REMOVE_ITEM META_SRC "src/metaBDBImpl.c") +endif() + +message("${META_SRC}") + add_library(meta STATIC ${META_SRC}) target_include_directories( meta @@ -7,11 +25,22 @@ target_include_directories( ) target_link_libraries( meta - PUBLIC sqlite PUBLIC common - PUBLIC tkv ) +if(${META_DB_IMPL} STREQUAL "BDB") + target_link_libraries( + meta + PUBLIC bdb + ) +elseif(${META_DB_IMPL} STREQUAL "SQLITE") + target_link_libraries( + meta + PUBLIC sqlite + ) +endif() + + if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/meta/inc/metaCache.h b/source/dnode/vnode/meta/inc/metaCache.h index 92c5a09c0c..46798f0de4 100644 --- a/source/dnode/vnode/meta/inc/metaCache.h +++ b/source/dnode/vnode/meta/inc/metaCache.h @@ -16,15 +16,13 @@ #ifndef _TD_META_CACHE_H_ #define _TD_META_CACHE_H_ -#include "rocksdb/c.h" - #include "meta.h" #ifdef __cplusplus extern "C" { #endif -typedef rocksdb_cache_t meta_cache_t; +typedef struct SMetaCache SMetaCache; int metaOpenCache(SMeta *pMeta); void metaCloseCache(SMeta *pMeta); diff --git a/source/dnode/vnode/meta/inc/metaDB.h b/source/dnode/vnode/meta/inc/metaDB.h index c61dd1e20f..49a3f25c89 100644 --- a/source/dnode/vnode/meta/inc/metaDB.h +++ b/source/dnode/vnode/meta/inc/metaDB.h @@ -16,28 +16,13 @@ #ifndef _TD_META_DB_H_ #define _TD_META_DB_H_ -#define USE_SQLITE_IMPL 1 - -#include "rocksdb/c.h" -#include "sqlite3.h" - #include "meta.h" #ifdef __cplusplus extern "C" { #endif -#if !USE_SQLITE_IMPL -typedef struct { - rocksdb_t *tbDb; // uid -> tb obj - rocksdb_t *nameDb; // name -> uid - rocksdb_t *tagDb; // uid -> tag - rocksdb_t *schemaDb; // uid+version -> schema - sqlite3 * mapDb; // suid -> uid_list -} meta_db_t; -#else -typedef sqlite3 meta_db_t; -#endif +typedef struct SMetaDB SMetaDB; int metaOpenDB(SMeta *pMeta); void metaCloseDB(SMeta *pMeta); diff --git a/source/dnode/vnode/meta/inc/metaDef.h b/source/dnode/vnode/meta/inc/metaDef.h index fd14efd50b..e1c15af5aa 100644 --- a/source/dnode/vnode/meta/inc/metaDef.h +++ b/source/dnode/vnode/meta/inc/metaDef.h @@ -34,9 +34,9 @@ extern "C" { struct SMeta { char* path; SMetaCfg options; - meta_db_t* pDB; - meta_index_t* pIdx; - meta_cache_t* pCache; + SMetaDB* pDB; + SMetaIdx* pIdx; + SMetaCache* pCache; STbUidGenerator uidGnrt; SMemAllocatorFactory* pmaf; }; diff --git a/source/dnode/vnode/meta/inc/metaIdx.h b/source/dnode/vnode/meta/inc/metaIdx.h index 28d58cb4f1..b6afc4cc97 100644 --- a/source/dnode/vnode/meta/inc/metaIdx.h +++ b/source/dnode/vnode/meta/inc/metaIdx.h @@ -16,15 +16,13 @@ #ifndef _TD_META_IDX_H_ #define _TD_META_IDX_H_ -#include "rocksdb/c.h" - #include "meta.h" #ifdef __cplusplus extern "C" { #endif -typedef rocksdb_t meta_index_t; +typedef struct SMetaIdx SMetaIdx; int metaOpenIdx(SMeta *pMeta); void metaCloseIdx(SMeta *pMeta); diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c new file mode 100644 index 0000000000..1cc197820b --- /dev/null +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "metaDef.h" + +#include "db.h" + +struct SMetaDB { + DB * pDB; + DB * pIdx; + DB_ENV *pEvn; +}; + +int metaOpenDB(SMeta *pMeta) { + int ret; + char dbname[128]; + + pMeta->pDB = (SMetaDB *)calloc(1, sizeof(SMetaDB)); + if (pMeta->pDB == NULL) { + // TODO: handle error + return -1; + } + + // TODO: create the env + + ret = db_create(&(pMeta->pDB->pDB), pMeta->pDB->pEvn, 0); + if (ret != 0) { + // TODO: handle error + return -1; + } + + ret = db_create(&(pMeta->pDB->pIdx), pMeta->pDB->pEvn, 0); + if (ret != 0) { + // TODO: handle error + return -1; + } + + ret = pMeta->pDB->pDB->open(pMeta->pDB->pDB, /* DB structure pointer */ + NULL, /* Transaction pointer */ + "meta.db", /* On-disk file that holds the database */ + NULL, /* Optional logical database name */ + DB_BTREE, /* Database access method */ + DB_CREATE, /* Open flags */ + 0); /* File mode */ + if (ret != 0) { + // TODO: handle error + return -1; + } + + ret = pMeta->pDB->pIdx->open(pMeta->pDB->pIdx, /* DB structure pointer */ + NULL, /* Transaction pointer */ + "index.db", /* On-disk file that holds the database */ + NULL, /* Optional logical database name */ + DB_BTREE, /* Database access method */ + DB_CREATE, /* Open flags */ + 0); /* File mode */ + if (ret != 0) { + // TODO: handle error + return -1; + } + + // TODO + return 0; +} + +void metaCloseDB(SMeta *pMeta) { + if (pMeta->pDB) { + /* TODO */ + free(pMeta->pDB); + } +} + +int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { + // TODO + return 0; +} + +int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { + // TODO +} \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaCache.c b/source/dnode/vnode/meta/src/metaCache.c index aaa97caea0..266305cbab 100644 --- a/source/dnode/vnode/meta/src/metaCache.c +++ b/source/dnode/vnode/meta/src/metaCache.c @@ -16,22 +16,26 @@ #include "meta.h" #include "metaDef.h" +struct SMetaCache { + // TODO +}; + int metaOpenCache(SMeta *pMeta) { // TODO - if (pMeta->options.lruSize) { - pMeta->pCache = rocksdb_cache_create_lru(pMeta->options.lruSize); - if (pMeta->pCache == NULL) { - // TODO: handle error - return -1; - } - } + // if (pMeta->options.lruSize) { + // pMeta->pCache = rocksdb_cache_create_lru(pMeta->options.lruSize); + // if (pMeta->pCache == NULL) { + // // TODO: handle error + // return -1; + // } + // } return 0; } void metaCloseCache(SMeta *pMeta) { - if (pMeta->pCache) { - rocksdb_cache_destroy(pMeta->pCache); - pMeta->pCache = NULL; - } + // if (pMeta->pCache) { + // rocksdb_cache_destroy(pMeta->pCache); + // pMeta->pCache = NULL; + // } } \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaDB.c b/source/dnode/vnode/meta/src/metaDB.c deleted file mode 100644 index 302e5dc6ae..0000000000 --- a/source/dnode/vnode/meta/src/metaDB.c +++ /dev/null @@ -1,307 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "metaDef.h" - -#if !USE_SQLITE_IMPL -static void metaSaveSchemaDB(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema); -static void metaGetSchemaDBKey(char key[], tb_uid_t uid, int sversion); - -#define SCHEMA_KEY_LEN (sizeof(tb_uid_t) + sizeof(int)) - -#define META_OPEN_DB_IMPL(pDB, options, dir, err) \ - do { \ - pDB = rocksdb_open(options, dir, &err); \ - if (pDB == NULL) { \ - metaCloseDB(pMeta); \ - rocksdb_options_destroy(options); \ - return -1; \ - } \ - } while (0) - -#define META_CLOSE_DB_IMPL(pDB) \ - do { \ - if (pDB) { \ - rocksdb_close(pDB); \ - pDB = NULL; \ - } \ - } while (0) -#endif - -int metaOpenDB(SMeta *pMeta) { - char dir[128]; - int rc; - char *err = NULL; -#if !USE_SQLITE_IMPL - rocksdb_options_t *options = rocksdb_options_create(); - - if (pMeta->pCache) { - rocksdb_options_set_row_cache(options, pMeta->pCache); - } - rocksdb_options_set_create_if_missing(options, 1); - - pMeta->pDB = (meta_db_t *)calloc(1, sizeof(*(pMeta->pDB))); - if (pMeta->pDB == NULL) { - // TODO: handle error - rocksdb_options_destroy(options); - return -1; - } - - // tbDb - sprintf(dir, "%s/tb_db", pMeta->path); - META_OPEN_DB_IMPL(pMeta->pDB->tbDb, options, dir, err); - - // nameDb - sprintf(dir, "%s/name_db", pMeta->path); - META_OPEN_DB_IMPL(pMeta->pDB->nameDb, options, dir, err); - - // tagDb - sprintf(dir, "%s/tag_db", pMeta->path); - META_OPEN_DB_IMPL(pMeta->pDB->tagDb, options, dir, err); - - // schemaDb - sprintf(dir, "%s/schema_db", pMeta->path); - META_OPEN_DB_IMPL(pMeta->pDB->schemaDb, options, dir, err); - - // mapDb - sprintf(dir, "%s/meta.db", pMeta->path); - if (sqlite3_open(dir, &(pMeta->pDB->mapDb)) != SQLITE_OK) { - // TODO - } - - // // set read uncommitted - sqlite3_exec(pMeta->pDB->mapDb, "PRAGMA read_uncommitted=true;", 0, 0, 0); - sqlite3_exec(pMeta->pDB->mapDb, "BEGIN;", 0, 0, 0); - - rocksdb_options_destroy(options); -#else - sprintf(dir, "%s/meta.db", pMeta->path); - rc = sqlite3_open(dir, &(pMeta->pDB)); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to open meta.db\n"); - } - - // For all tables - rc = sqlite3_exec(pMeta->pDB, - "CREATE TABLE IF NOT EXISTS tb (" - " tbname VARCHAR(256) NOT NULL UNIQUE," - " tb_uid INTEGER NOT NULL UNIQUE " - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table tb since %s\n", err); - } - - // For super tables - rc = sqlite3_exec(pMeta->pDB, - "CREATE TABLE IF NOT EXISTS stb (" - " tb_uid INTEGER NOT NULL UNIQUE," - " tbname VARCHAR(256) NOT NULL UNIQUE," - " tb_schema BLOB NOT NULL," - " tag_schema BLOB NOT NULL" - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table stb since %s\n", err); - } - - // For normal tables - rc = sqlite3_exec(pMeta->pDB, - "CREATE TABLE IF NOT EXISTS ntb (" - " tb_uid INTEGER NOT NULL UNIQUE," - " tbname VARCHAR(256) NOT NULL," - " tb_schema BLOB NOT NULL" - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table ntb since %s\n", err); - } - - sqlite3_exec(pMeta->pDB, "BEGIN;", NULL, NULL, &err); - - tfree(err); -#endif - - return 0; -} - -void metaCloseDB(SMeta *pMeta) { -#if !USE_SQLITE_IMPL - if (pMeta->pDB) { - if (pMeta->pDB->mapDb) { - sqlite3_exec(pMeta->pDB->mapDb, "COMMIT;", 0, 0, 0); - sqlite3_close(pMeta->pDB->mapDb); - pMeta->pDB->mapDb = NULL; - } - - META_CLOSE_DB_IMPL(pMeta->pDB->schemaDb); - META_CLOSE_DB_IMPL(pMeta->pDB->tagDb); - META_CLOSE_DB_IMPL(pMeta->pDB->nameDb); - META_CLOSE_DB_IMPL(pMeta->pDB->tbDb); - free(pMeta->pDB); - pMeta->pDB = NULL; - } -#else - if (pMeta->pDB) { - sqlite3_exec(pMeta->pDB, "BEGIN;", NULL, NULL, NULL); - sqlite3_close(pMeta->pDB); - pMeta->pDB = NULL; - } - - // TODO -#endif -} - -int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { -#if !USE_SQLITE_IMPL - tb_uid_t uid; - char * err = NULL; - size_t size; - char pBuf[1024]; // TODO - char sql[128]; - - rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create(); - - // Generate a uid for child and normal table - if (pTbOptions->type == META_SUPER_TABLE) { - uid = pTbOptions->stbCfg.suid; - } else { - uid = metaGenerateUid(pMeta); - } - - // Save tbname -> uid to tbnameDB - rocksdb_put(pMeta->pDB->nameDb, wopt, pTbOptions->name, strlen(pTbOptions->name), (char *)(&uid), sizeof(uid), &err); - rocksdb_writeoptions_disable_WAL(wopt, 1); - - // Save uid -> tb_obj to tbDB - size = metaEncodeTbObjFromTbOptions(pTbOptions, pBuf, 1024); - rocksdb_put(pMeta->pDB->tbDb, wopt, (char *)(&uid), sizeof(uid), pBuf, size, &err); - - switch (pTbOptions->type) { - case META_NORMAL_TABLE: - // save schemaDB - metaSaveSchemaDB(pMeta, uid, pTbOptions->ntbCfg.pSchema); - break; - case META_SUPER_TABLE: - // save schemaDB - metaSaveSchemaDB(pMeta, uid, pTbOptions->stbCfg.pSchema); - - // // save mapDB (really need?) - // rocksdb_put(pMeta->pDB->mapDb, wopt, (char *)(&uid), sizeof(uid), "", 0, &err); - sprintf(sql, "create table st_%" PRIu64 " (uid BIGINT);", uid); - if (sqlite3_exec(pMeta->pDB->mapDb, sql, NULL, NULL, &err) != SQLITE_OK) { - // fprintf(stderr,"Failed to create table, since %s\n", err); - } - break; - case META_CHILD_TABLE: - // save tagDB - rocksdb_put(pMeta->pDB->tagDb, wopt, (char *)(&uid), sizeof(uid), pTbOptions->ctbCfg.pTag, - kvRowLen(pTbOptions->ctbCfg.pTag), &err); - - // save mapDB - sprintf(sql, "insert into st_%" PRIu64 " values (%" PRIu64 ");", pTbOptions->ctbCfg.suid, uid); - if (sqlite3_exec(pMeta->pDB->mapDb, sql, NULL, NULL, &err) != SQLITE_OK) { - fprintf(stderr, "failed to insert data, since %s\n", err); - } - break; - default: - ASSERT(0); - } - - rocksdb_writeoptions_destroy(wopt); -#else - char sql[256]; - char *err = NULL; - int rc; - - switch (pTbOptions->type) { - case META_SUPER_TABLE: - // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - // ");" - // "INSERT INTO stb VALUES (%" PRIu64 - // ", \'%s\', );" - // "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 - // " (" - // " tb_uid INTEGER NOT NULL UNIQUE," - // " tbname VARCHAR(256)," - // " tag1 INTEGER" - // ");" - // "CREATE INDEX IF NOT EXISTS stb_%" PRIu64 "_tag1_idx ON stb_1638517480 (tag1);"); - rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); - if (rc != SQLITE_OK) { - printf("failed to create normal table since %s\n", err); - } - break; - case META_NORMAL_TABLE: - // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - // ");" - // "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );"); - rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); - if (rc != SQLITE_OK) { - printf("failed to create normal table since %s\n", err); - } - break; - case META_CHILD_TABLE: - // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - // ");" - // "INSERT INTO stb_%" PRIu64 " VALUES (%" PRIu64 ", \'%s\', );"); - rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); - if (rc != SQLITE_OK) { - printf("failed to create child table since %s\n", err); - } - break; - default: - break; - } - - tfree(err); -#endif - - return 0; -} - -int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { - /* TODO */ - return 0; -} - -/* ------------------------ STATIC METHODS ------------------------ */ -#if !USE_SQLITE_IMPL -static void metaSaveSchemaDB(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema) { - char key[64]; - char pBuf[1024]; - char * ppBuf = pBuf; - size_t vsize; - char * err = NULL; - - rocksdb_writeoptions_t *wopt = rocksdb_writeoptions_create(); - rocksdb_writeoptions_disable_WAL(wopt, 1); - - metaGetSchemaDBKey(key, uid, schemaVersion(pSchema)); - vsize = tdEncodeSchema((void **)(&ppBuf), pSchema); - rocksdb_put(pMeta->pDB->schemaDb, wopt, key, SCHEMA_KEY_LEN, pBuf, vsize, &err); - - rocksdb_writeoptions_destroy(wopt); -} - -static void metaGetSchemaDBKey(char *key, tb_uid_t uid, int sversion) { - *(tb_uid_t *)key = uid; - *(int *)POINTER_SHIFT(key, sizeof(tb_uid_t)) = sversion; -} -#endif \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaIdx.c b/source/dnode/vnode/meta/src/metaIdx.c index 078c7e7d08..fe07f5ced4 100644 --- a/source/dnode/vnode/meta/src/metaIdx.c +++ b/source/dnode/vnode/meta/src/metaIdx.c @@ -15,6 +15,10 @@ #include "metaDef.h" +struct SMetaIdx { + /* data */ +}; + int metaOpenIdx(SMeta *pMeta) { #if 0 char idxDir[128]; // TODO diff --git a/source/dnode/vnode/meta/src/metaSQLiteImpl.c b/source/dnode/vnode/meta/src/metaSQLiteImpl.c new file mode 100644 index 0000000000..a8692f05fe --- /dev/null +++ b/source/dnode/vnode/meta/src/metaSQLiteImpl.c @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "metaDef.h" +#include "sqlite3.h" + +int metaOpenDB(SMeta *pMeta) { + char dir[128]; + int rc; + char *err = NULL; + + sprintf(dir, "%s/meta.db", pMeta->path); + rc = sqlite3_open(dir, &(pMeta->pDB)); + if (rc != SQLITE_OK) { + // TODO: handle error + printf("failed to open meta.db\n"); + } + + // For all tables + rc = sqlite3_exec(pMeta->pDB, + "CREATE TABLE IF NOT EXISTS tb (" + " tbname VARCHAR(256) NOT NULL UNIQUE," + " tb_uid INTEGER NOT NULL UNIQUE " + ");", + NULL, NULL, &err); + if (rc != SQLITE_OK) { + // TODO: handle error + printf("failed to create meta table tb since %s\n", err); + } + + // For super tables + rc = sqlite3_exec(pMeta->pDB, + "CREATE TABLE IF NOT EXISTS stb (" + " tb_uid INTEGER NOT NULL UNIQUE," + " tbname VARCHAR(256) NOT NULL UNIQUE," + " tb_schema BLOB NOT NULL," + " tag_schema BLOB NOT NULL" + ");", + NULL, NULL, &err); + if (rc != SQLITE_OK) { + // TODO: handle error + printf("failed to create meta table stb since %s\n", err); + } + + // For normal tables + rc = sqlite3_exec(pMeta->pDB, + "CREATE TABLE IF NOT EXISTS ntb (" + " tb_uid INTEGER NOT NULL UNIQUE," + " tbname VARCHAR(256) NOT NULL," + " tb_schema BLOB NOT NULL" + ");", + NULL, NULL, &err); + if (rc != SQLITE_OK) { + // TODO: handle error + printf("failed to create meta table ntb since %s\n", err); + } + + sqlite3_exec(pMeta->pDB, "BEGIN;", NULL, NULL, &err); + + tfree(err); + + return 0; +} + +void metaCloseDB(SMeta *pMeta) { + if (pMeta->pDB) { + sqlite3_exec(pMeta->pDB, "BEGIN;", NULL, NULL, NULL); + sqlite3_close(pMeta->pDB); + pMeta->pDB = NULL; + } + + // TODO +} + +int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { + char sql[256]; + char *err = NULL; + int rc; + + switch (pTbOptions->type) { + case META_SUPER_TABLE: + // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 + // ");" + // "INSERT INTO stb VALUES (%" PRIu64 + // ", \'%s\', );" + // "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 + // " (" + // " tb_uid INTEGER NOT NULL UNIQUE," + // " tbname VARCHAR(256)," + // " tag1 INTEGER" + // ");" + // "CREATE INDEX IF NOT EXISTS stb_%" PRIu64 "_tag1_idx ON stb_1638517480 (tag1);"); + rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); + if (rc != SQLITE_OK) { + printf("failed to create normal table since %s\n", err); + } + break; + case META_NORMAL_TABLE: + // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 + // ");" + // "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );"); + rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); + if (rc != SQLITE_OK) { + printf("failed to create normal table since %s\n", err); + } + break; + case META_CHILD_TABLE: + // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 + // ");" + // "INSERT INTO stb_%" PRIu64 " VALUES (%" PRIu64 ", \'%s\', );"); + rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); + if (rc != SQLITE_OK) { + printf("failed to create child table since %s\n", err); + } + break; + default: + break; + } + + tfree(err); + + return 0; +} + +int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { + /* TODO */ + return 0; +} \ No newline at end of file From c8f23a801de2141c4c6ffbd4b15b30dc125f1074 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 6 Dec 2021 15:35:39 +0800 Subject: [PATCH 10/27] more --- source/dnode/vnode/meta/src/metaBDBImpl.c | 30 ++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 1cc197820b..e0c21dba64 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -24,8 +24,7 @@ struct SMetaDB { }; int metaOpenDB(SMeta *pMeta) { - int ret; - char dbname[128]; + int ret; pMeta->pDB = (SMetaDB *)calloc(1, sizeof(SMetaDB)); if (pMeta->pDB == NULL) { @@ -34,6 +33,17 @@ int metaOpenDB(SMeta *pMeta) { } // TODO: create the env + ret = db_env_create(&(pMeta->pDB->pEvn), 0); + if (ret != 0) { + // TODO: handle error + return -1; + } + + ret = pMeta->pDB->pEvn->open(pMeta->pDB->pEvn, pMeta->path, DB_CREATE | DB_INIT_MPOOL, 0); + if (ret != 0) { + // TODO: handle error + return -1; + } ret = db_create(&(pMeta->pDB->pDB), pMeta->pDB->pEvn, 0); if (ret != 0) { @@ -77,7 +87,21 @@ int metaOpenDB(SMeta *pMeta) { void metaCloseDB(SMeta *pMeta) { if (pMeta->pDB) { - /* TODO */ + if (pMeta->pDB->pIdx) { + pMeta->pDB->pIdx->close(pMeta->pDB->pIdx, 0); + pMeta->pDB->pIdx = NULL; + } + + if (pMeta->pDB->pDB) { + pMeta->pDB->pDB->close(pMeta->pDB->pDB, 0); + pMeta->pDB->pDB = NULL; + } + + if (pMeta->pDB->pEvn) { + pMeta->pDB->pEvn->close(pMeta->pDB->pEvn, 0); + pMeta->pDB->pEvn = NULL; + } + free(pMeta->pDB); } } From 2ab7dc06bfb185a445d757beeccd59a7f088e689 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 6 Dec 2021 15:49:51 +0800 Subject: [PATCH 11/27] more --- source/dnode/vnode/meta/inc/metaDef.h | 2 +- source/dnode/vnode/meta/src/metaSQLiteImpl.c | 31 +++++++++++++------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/meta/inc/metaDef.h b/source/dnode/vnode/meta/inc/metaDef.h index e1c15af5aa..0204031e76 100644 --- a/source/dnode/vnode/meta/inc/metaDef.h +++ b/source/dnode/vnode/meta/inc/metaDef.h @@ -34,7 +34,7 @@ extern "C" { struct SMeta { char* path; SMetaCfg options; - SMetaDB* pDB; + SMetaDB* pDB; SMetaIdx* pIdx; SMetaCache* pCache; STbUidGenerator uidGnrt; diff --git a/source/dnode/vnode/meta/src/metaSQLiteImpl.c b/source/dnode/vnode/meta/src/metaSQLiteImpl.c index a8692f05fe..37ec1cef47 100644 --- a/source/dnode/vnode/meta/src/metaSQLiteImpl.c +++ b/source/dnode/vnode/meta/src/metaSQLiteImpl.c @@ -16,20 +16,30 @@ #include "metaDef.h" #include "sqlite3.h" +struct SMetaDB { + sqlite3 *pDB; +}; + int metaOpenDB(SMeta *pMeta) { char dir[128]; int rc; char *err = NULL; + pMeta->pDB = (SMetaDB *)calloc(1, sizeof(SMetaDB)); + if (pMeta->pDB == NULL) { + // TODO: handle error + return -1; + } + sprintf(dir, "%s/meta.db", pMeta->path); - rc = sqlite3_open(dir, &(pMeta->pDB)); + rc = sqlite3_open(dir, &(pMeta->pDB->pDB)); if (rc != SQLITE_OK) { // TODO: handle error printf("failed to open meta.db\n"); } // For all tables - rc = sqlite3_exec(pMeta->pDB, + rc = sqlite3_exec(pMeta->pDB->pDB, "CREATE TABLE IF NOT EXISTS tb (" " tbname VARCHAR(256) NOT NULL UNIQUE," " tb_uid INTEGER NOT NULL UNIQUE " @@ -41,7 +51,7 @@ int metaOpenDB(SMeta *pMeta) { } // For super tables - rc = sqlite3_exec(pMeta->pDB, + rc = sqlite3_exec(pMeta->pDB->pDB, "CREATE TABLE IF NOT EXISTS stb (" " tb_uid INTEGER NOT NULL UNIQUE," " tbname VARCHAR(256) NOT NULL UNIQUE," @@ -55,7 +65,7 @@ int metaOpenDB(SMeta *pMeta) { } // For normal tables - rc = sqlite3_exec(pMeta->pDB, + rc = sqlite3_exec(pMeta->pDB->pDB, "CREATE TABLE IF NOT EXISTS ntb (" " tb_uid INTEGER NOT NULL UNIQUE," " tbname VARCHAR(256) NOT NULL," @@ -67,7 +77,7 @@ int metaOpenDB(SMeta *pMeta) { printf("failed to create meta table ntb since %s\n", err); } - sqlite3_exec(pMeta->pDB, "BEGIN;", NULL, NULL, &err); + sqlite3_exec(pMeta->pDB->pDB, "BEGIN;", NULL, NULL, &err); tfree(err); @@ -76,8 +86,9 @@ int metaOpenDB(SMeta *pMeta) { void metaCloseDB(SMeta *pMeta) { if (pMeta->pDB) { - sqlite3_exec(pMeta->pDB, "BEGIN;", NULL, NULL, NULL); - sqlite3_close(pMeta->pDB); + sqlite3_exec(pMeta->pDB->pDB, "COMMIT;", NULL, NULL, NULL); + sqlite3_close(pMeta->pDB->pDB); + free(pMeta->pDB); pMeta->pDB = NULL; } @@ -102,7 +113,7 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { // " tag1 INTEGER" // ");" // "CREATE INDEX IF NOT EXISTS stb_%" PRIu64 "_tag1_idx ON stb_1638517480 (tag1);"); - rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); + rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); if (rc != SQLITE_OK) { printf("failed to create normal table since %s\n", err); } @@ -111,7 +122,7 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 // ");" // "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );"); - rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); + rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); if (rc != SQLITE_OK) { printf("failed to create normal table since %s\n", err); } @@ -120,7 +131,7 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 // ");" // "INSERT INTO stb_%" PRIu64 " VALUES (%" PRIu64 ", \'%s\', );"); - rc = sqlite3_exec(pMeta->pDB, sql, NULL, NULL, &err); + rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); if (rc != SQLITE_OK) { printf("failed to create child table since %s\n", err); } From bb404285073138d07c2ce72101279f3e092be00c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 6 Dec 2021 17:25:07 +0800 Subject: [PATCH 12/27] more --- source/dnode/vnode/meta/CMakeLists.txt | 2 +- source/dnode/vnode/meta/inc/metaDB.h | 2 +- source/dnode/vnode/meta/src/metaBDBImpl.c | 4 +++- source/dnode/vnode/meta/src/metaSQLiteImpl.c | 22 +++++++++++++------- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/meta/CMakeLists.txt b/source/dnode/vnode/meta/CMakeLists.txt index 7e5acc6450..f0aa28d7a6 100644 --- a/source/dnode/vnode/meta/CMakeLists.txt +++ b/source/dnode/vnode/meta/CMakeLists.txt @@ -1,5 +1,5 @@ set(META_DB_IMPL_LIST "BDB" "SQLITE") -set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") +set(META_DB_IMPL "SQLITE" CACHE STRING "Use BDB as the default META implementation") set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) diff --git a/source/dnode/vnode/meta/inc/metaDB.h b/source/dnode/vnode/meta/inc/metaDB.h index 49a3f25c89..62b91ed48c 100644 --- a/source/dnode/vnode/meta/inc/metaDB.h +++ b/source/dnode/vnode/meta/inc/metaDB.h @@ -26,7 +26,7 @@ typedef struct SMetaDB SMetaDB; int metaOpenDB(SMeta *pMeta); void metaCloseDB(SMeta *pMeta); -int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions); +int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg); int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid); #ifdef __cplusplus diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index e0c21dba64..76c779fe2c 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -39,6 +39,8 @@ int metaOpenDB(SMeta *pMeta) { return -1; } + // pMeta->pDB->pEvn->set_cachesize(pMeta->pDB->pEvn, ) + ret = pMeta->pDB->pEvn->open(pMeta->pDB->pEvn, pMeta->path, DB_CREATE | DB_INIT_MPOOL, 0); if (ret != 0) { // TODO: handle error @@ -106,7 +108,7 @@ void metaCloseDB(SMeta *pMeta) { } } -int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { +int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { // TODO return 0; } diff --git a/source/dnode/vnode/meta/src/metaSQLiteImpl.c b/source/dnode/vnode/meta/src/metaSQLiteImpl.c index 37ec1cef47..12aa0a6c5d 100644 --- a/source/dnode/vnode/meta/src/metaSQLiteImpl.c +++ b/source/dnode/vnode/meta/src/metaSQLiteImpl.c @@ -95,13 +95,15 @@ void metaCloseDB(SMeta *pMeta) { // TODO } -int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { - char sql[256]; - char *err = NULL; - int rc; +int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { + char sql[256]; + char * err = NULL; + int rc; + tb_uid_t uid; - switch (pTbOptions->type) { + switch (pTbCfg->type) { case META_SUPER_TABLE: + uid = pTbCfg->stbCfg.suid; // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 // ");" // "INSERT INTO stb VALUES (%" PRIu64 @@ -119,15 +121,19 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbOptions) { } break; case META_NORMAL_TABLE: - // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - // ");" - // "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );"); + uid = metaGenerateUid(pMeta); + // sprintf(sql, + // "INSERT INTO tb VALUES (\'%s\', %" PRIu64 + // ");" + // "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );", + // pTbCfg->name, uid, uid, pTbCfg->name, ); rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); if (rc != SQLITE_OK) { printf("failed to create normal table since %s\n", err); } break; case META_CHILD_TABLE: + uid = metaGenerateUid(pMeta); // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 // ");" // "INSERT INTO stb_%" PRIu64 " VALUES (%" PRIu64 ", \'%s\', );"); From 1d5a1fbf6dd69f0db1b2c738a2fa709ed0ed748f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 6 Dec 2021 17:53:59 +0800 Subject: [PATCH 13/27] more --- source/dnode/vnode/meta/src/metaSQLiteImpl.c | 66 +++++++++++++------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaSQLiteImpl.c b/source/dnode/vnode/meta/src/metaSQLiteImpl.c index 12aa0a6c5d..9c2a757652 100644 --- a/source/dnode/vnode/meta/src/metaSQLiteImpl.c +++ b/source/dnode/vnode/meta/src/metaSQLiteImpl.c @@ -96,43 +96,64 @@ void metaCloseDB(SMeta *pMeta) { } int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { - char sql[256]; - char * err = NULL; - int rc; - tb_uid_t uid; + char sql[256]; + char * err = NULL; + int rc; + tb_uid_t uid; + sqlite3_stmt *stmt; + char buf[256]; + void * pBuf; switch (pTbCfg->type) { case META_SUPER_TABLE: uid = pTbCfg->stbCfg.suid; - // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - // ");" - // "INSERT INTO stb VALUES (%" PRIu64 - // ", \'%s\', );" - // "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 - // " (" - // " tb_uid INTEGER NOT NULL UNIQUE," - // " tbname VARCHAR(256)," - // " tag1 INTEGER" - // ");" - // "CREATE INDEX IF NOT EXISTS stb_%" PRIu64 "_tag1_idx ON stb_1638517480 (tag1);"); - rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); + strcpy(sql, "INSERT INTO tb VALUES (?, ?);"); + // sprintf(sql, + // "INSERT INTO tb VALUES (?, ?);" + // // "INSERT INTO stb VALUES (?, ?, ?, ?);" + // // "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 + // // " (" + // // " tb_uid INTEGER NOT NULL UNIQUE," + // // " tbname VARCHAR(256)," + // // " tag1 INTEGER);" + // , + // uid); + rc = sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL); if (rc != SQLITE_OK) { - printf("failed to create normal table since %s\n", err); + return -1; } + sqlite3_bind_text(stmt, 1, pTbCfg->name, -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(stmt, 2, uid); + // sqlite3_bind_int64(stmt, 3, uid); + // sqlite3_bind_text(stmt, 4, pTbCfg->name, -1, SQLITE_TRANSIENT); + // pBuf = buf; + // tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema); + // sqlite3_bind_blob(stmt, 5, buf, POINTER_DISTANCE(pBuf, buf), NULL); + // pBuf = buf; + // tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema); + // sqlite3_bind_blob(stmt, 6, buf, POINTER_DISTANCE(pBuf, buf), NULL); + + rc = sqlite3_step(stmt); + if (rc != SQLITE_OK) { + printf("failed to create normal table since %s\n", sqlite3_errmsg(pMeta->pDB->pDB)); + } + sqlite3_finalize(stmt); break; case META_NORMAL_TABLE: - uid = metaGenerateUid(pMeta); + // uid = metaGenerateUid(pMeta); // sprintf(sql, // "INSERT INTO tb VALUES (\'%s\', %" PRIu64 // ");" // "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );", // pTbCfg->name, uid, uid, pTbCfg->name, ); - rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); - if (rc != SQLITE_OK) { - printf("failed to create normal table since %s\n", err); - } + + // rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); + // if (rc != SQLITE_OK) { + // printf("failed to create normal table since %s\n", err); + // } break; case META_CHILD_TABLE: +#if 0 uid = metaGenerateUid(pMeta); // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 // ");" @@ -141,6 +162,7 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { if (rc != SQLITE_OK) { printf("failed to create child table since %s\n", err); } +#endif break; default: break; From 2fdc55c29e68e9480c983edf5038678d28b1a8af Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 6 Dec 2021 18:15:03 +0800 Subject: [PATCH 14/27] more --- source/dnode/vnode/meta/src/metaSQLiteImpl.c | 57 +++++++++++++++----- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaSQLiteImpl.c b/source/dnode/vnode/meta/src/metaSQLiteImpl.c index 9c2a757652..fe9ef22fb3 100644 --- a/source/dnode/vnode/meta/src/metaSQLiteImpl.c +++ b/source/dnode/vnode/meta/src/metaSQLiteImpl.c @@ -107,23 +107,55 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { switch (pTbCfg->type) { case META_SUPER_TABLE: uid = pTbCfg->stbCfg.suid; - strcpy(sql, "INSERT INTO tb VALUES (?, ?);"); - // sprintf(sql, - // "INSERT INTO tb VALUES (?, ?);" - // // "INSERT INTO stb VALUES (?, ?, ?, ?);" - // // "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 - // // " (" - // // " tb_uid INTEGER NOT NULL UNIQUE," - // // " tbname VARCHAR(256)," - // // " tag1 INTEGER);" - // , - // uid); + sprintf(sql, + "INSERT INTO tb VALUES (\'%s\', %" PRIu64 + ");" + "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 + " (" + " tb_uid INTEGER NOT NULL UNIQUE," + " tbname VARCHAR(256)," + " tag1 INTEGER);", + pTbCfg->name, uid, uid); + rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); + if (rc != SQLITE_OK) { + printf("failed to create normal table since %s\n", err); + } + + sprintf(sql, "INSERT INTO stb VALUES (%" PRIu64 ", %s, ?, ?)", uid, pTbCfg->name); + sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL); + + pBuf = buf; + tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema); + sqlite3_bind_blob(stmt, 1, buf, POINTER_DISTANCE(pBuf, buf), NULL); + pBuf = buf; + tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema); + sqlite3_bind_blob(stmt, 2, buf, POINTER_DISTANCE(pBuf, buf), NULL); + + sqlite3_step(stmt); + + sqlite3_finalize(stmt); + +#if 0 + sprintf(sql, + "INSERT INTO tb VALUES (?, ?);" + // "INSERT INTO stb VALUES (?, ?, ?, ?);" + // "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 + // " (" + // " tb_uid INTEGER NOT NULL UNIQUE," + // " tbname VARCHAR(256)," + // " tag1 INTEGER);" + , + uid); rc = sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL); if (rc != SQLITE_OK) { return -1; } sqlite3_bind_text(stmt, 1, pTbCfg->name, -1, SQLITE_TRANSIENT); sqlite3_bind_int64(stmt, 2, uid); + sqlite3_step(stmt); + sqlite3_finalize(stmt); + + // sqlite3_bind_int64(stmt, 3, uid); // sqlite3_bind_text(stmt, 4, pTbCfg->name, -1, SQLITE_TRANSIENT); // pBuf = buf; @@ -133,11 +165,12 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { // tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema); // sqlite3_bind_blob(stmt, 6, buf, POINTER_DISTANCE(pBuf, buf), NULL); - rc = sqlite3_step(stmt); + rc = sqliteVjj3_step(stmt); if (rc != SQLITE_OK) { printf("failed to create normal table since %s\n", sqlite3_errmsg(pMeta->pDB->pDB)); } sqlite3_finalize(stmt); +#endif break; case META_NORMAL_TABLE: // uid = metaGenerateUid(pMeta); From 3df6971514b3ffec5ace23d30c403acefe21b37a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 6 Dec 2021 20:42:14 +0800 Subject: [PATCH 15/27] use bdb by default --- source/dnode/vnode/meta/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/meta/CMakeLists.txt b/source/dnode/vnode/meta/CMakeLists.txt index f0aa28d7a6..7e5acc6450 100644 --- a/source/dnode/vnode/meta/CMakeLists.txt +++ b/source/dnode/vnode/meta/CMakeLists.txt @@ -1,5 +1,5 @@ set(META_DB_IMPL_LIST "BDB" "SQLITE") -set(META_DB_IMPL "SQLITE" CACHE STRING "Use BDB as the default META implementation") +set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) From 53d4fe80fec185311e2ccaccc2dc029aaf064307 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 6 Dec 2021 21:17:10 +0800 Subject: [PATCH 16/27] more --- source/dnode/vnode/meta/src/metaBDBImpl.c | 99 +++++++++++++++++++---- 1 file changed, 83 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 76c779fe2c..0565859688 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -18,7 +18,9 @@ #include "db.h" struct SMetaDB { - DB * pDB; + DB * pStbDB; + DB * pCtbDB; + DB * pNtbDB; DB * pIdx; DB_ENV *pEvn; }; @@ -32,22 +34,32 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - // TODO: create the env + // create the env ret = db_env_create(&(pMeta->pDB->pEvn), 0); if (ret != 0) { // TODO: handle error return -1; } - // pMeta->pDB->pEvn->set_cachesize(pMeta->pDB->pEvn, ) - ret = pMeta->pDB->pEvn->open(pMeta->pDB->pEvn, pMeta->path, DB_CREATE | DB_INIT_MPOOL, 0); if (ret != 0) { // TODO: handle error return -1; } - ret = db_create(&(pMeta->pDB->pDB), pMeta->pDB->pEvn, 0); + ret = db_create(&(pMeta->pDB->pStbDB), pMeta->pDB->pEvn, 0); + if (ret != 0) { + // TODO: handle error + return -1; + } + + ret = db_create(&(pMeta->pDB->pCtbDB), pMeta->pDB->pEvn, 0); + if (ret != 0) { + // TODO: handle error + return -1; + } + + ret = db_create(&(pMeta->pDB->pNtbDB), pMeta->pDB->pEvn, 0); if (ret != 0) { // TODO: handle error return -1; @@ -59,13 +71,37 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - ret = pMeta->pDB->pDB->open(pMeta->pDB->pDB, /* DB structure pointer */ - NULL, /* Transaction pointer */ - "meta.db", /* On-disk file that holds the database */ - NULL, /* Optional logical database name */ - DB_BTREE, /* Database access method */ - DB_CREATE, /* Open flags */ - 0); /* File mode */ + ret = pMeta->pDB->pStbDB->open(pMeta->pDB->pStbDB, /* DB structure pointer */ + NULL, /* Transaction pointer */ + "meta.db", /* On-disk file that holds the database */ + NULL, /* Optional logical database name */ + DB_BTREE, /* Database access method */ + DB_CREATE, /* Open flags */ + 0); /* File mode */ + if (ret != 0) { + // TODO: handle error + return -1; + } + + ret = pMeta->pDB->pCtbDB->open(pMeta->pDB->pCtbDB, /* DB structure pointer */ + NULL, /* Transaction pointer */ + "meta.db", /* On-disk file that holds the database */ + NULL, /* Optional logical database name */ + DB_BTREE, /* Database access method */ + DB_CREATE, /* Open flags */ + 0); /* File mode */ + if (ret != 0) { + // TODO: handle error + return -1; + } + + ret = pMeta->pDB->pNtbDB->open(pMeta->pDB->pNtbDB, /* DB structure pointer */ + NULL, /* Transaction pointer */ + "meta.db", /* On-disk file that holds the database */ + NULL, /* Optional logical database name */ + DB_BTREE, /* Database access method */ + DB_CREATE, /* Open flags */ + 0); /* File mode */ if (ret != 0) { // TODO: handle error return -1; @@ -94,9 +130,19 @@ void metaCloseDB(SMeta *pMeta) { pMeta->pDB->pIdx = NULL; } - if (pMeta->pDB->pDB) { - pMeta->pDB->pDB->close(pMeta->pDB->pDB, 0); - pMeta->pDB->pDB = NULL; + if (pMeta->pDB->pNtbDB) { + pMeta->pDB->pNtbDB->close(pMeta->pDB->pNtbDB, 0); + pMeta->pDB->pNtbDB = NULL; + } + + if (pMeta->pDB->pCtbDB) { + pMeta->pDB->pCtbDB->close(pMeta->pDB->pCtbDB, 0); + pMeta->pDB->pCtbDB = NULL; + } + + if (pMeta->pDB->pStbDB) { + pMeta->pDB->pStbDB->close(pMeta->pDB->pStbDB, 0); + pMeta->pDB->pStbDB = NULL; } if (pMeta->pDB->pEvn) { @@ -109,7 +155,28 @@ void metaCloseDB(SMeta *pMeta) { } int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { - // TODO + tb_uid_t uid; + DBT key = {0}; + DBT value = {0}; + char buf[256]; + void * pBuf; + int bsize; + + if (pTbCfg->type == META_SUPER_TABLE) { + uid = pTbCfg->stbCfg.suid; + } else { + uid = metaGenerateUid(pMeta); + } + + key.size = sizeof(uid); + key.data = &uid; + + pBuf = buf; + value.size = metaEncodeTbCfg(&pBuf, pTbCfg); + value.data = buf; + + pMeta->pDB->pStbDB->put(pMeta->pDB->pStbDB, NULL, &key, &value, 0); + return 0; } From 53eebdce883acc9443b4d04a07edad07e4f662b1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 6 Dec 2021 21:24:14 +0800 Subject: [PATCH 17/27] more --- include/dnode/vnode/meta/meta.h | 2 +- source/dnode/vnode/meta/src/metaTbCfg.c | 53 +++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index 1de1f9fa2c..44ae1bb79f 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -105,7 +105,7 @@ void metaOptionsClear(SMetaCfg *pOptions); #define META_CLEAR_TB_CFG(pTbCfg) int metaEncodeTbCfg(void **pBuf, STbCfg *pTbCfg); -void *metaDecodeTbCfg(void *pBuf, STbCfg **pTbCfg); +void *metaDecodeTbCfg(void *pBuf, STbCfg *pTbCfg); #ifdef __cplusplus } diff --git a/source/dnode/vnode/meta/src/metaTbCfg.c b/source/dnode/vnode/meta/src/metaTbCfg.c index 8485d82a8d..da181a68fc 100644 --- a/source/dnode/vnode/meta/src/metaTbCfg.c +++ b/source/dnode/vnode/meta/src/metaTbCfg.c @@ -49,10 +49,55 @@ size_t metaEncodeTbObjFromTbOptions(const STbCfg *pTbOptions, void *pBuf, size_t } int metaEncodeTbCfg(void **pBuf, STbCfg *pTbCfg) { - // TODO - return 0; + int tsize = 0; + + tsize += taosEncodeString(pBuf, pTbCfg->name); + tsize += taosEncodeFixedU32(pBuf, pTbCfg->ttl); + tsize += taosEncodeFixedU32(pBuf, pTbCfg->keep); + tsize += taosEncodeFixedU8(pBuf, pTbCfg->type); + + switch (pTbCfg->type) { + case META_SUPER_TABLE: + tsize += taosEncodeFixedU64(pBuf, pTbCfg->stbCfg.suid); + tsize += tdEncodeSchema(pBuf, pTbCfg->stbCfg.pSchema); + tsize += tdEncodeSchema(pBuf, pTbCfg->stbCfg.pTagSchema); + break; + case META_CHILD_TABLE: + tsize += taosEncodeFixedU64(pBuf, pTbCfg->ctbCfg.suid); + tsize += tdEncodeKVRow(pBuf, pTbCfg->ctbCfg.pTag); + break; + case META_NORMAL_TABLE: + tsize += tdEncodeSchema(pBuf, pTbCfg->ntbCfg.pSchema); + break; + default: + break; + } + + return tsize; } -void *metaDecodeTbCfg(void *pBuf, STbCfg **pTbCfg) { - // TODO +void *metaDecodeTbCfg(void *pBuf, STbCfg *pTbCfg) { + pBuf = taosDecodeString(pBuf, &(pTbCfg->name)); + pBuf = taosDecodeFixedU32(pBuf, &(pTbCfg->ttl)); + pBuf = taosDecodeFixedU32(pBuf, &(pTbCfg->keep)); + pBuf = taosDecodeFixedU8(pBuf, &(pTbCfg->type)); + + switch (pTbCfg->type) { + case META_SUPER_TABLE: + pBuf = taosDecodeFixedU64(pBuf, &(pTbCfg->stbCfg.suid)); + pBuf = tdDecodeSchema(pBuf, &(pTbCfg->stbCfg.pSchema)); + pBuf = tdDecodeSchema(pBuf, &(pTbCfg->stbCfg.pTagSchema)); + break; + case META_CHILD_TABLE: + pBuf = taosDecodeFixedU64(pBuf, &(pTbCfg->ctbCfg.suid)); + pBuf = tdDecodeKVRow(pBuf, &(pTbCfg->ctbCfg.pTag)); + break; + case META_NORMAL_TABLE: + pBuf = tdDecodeSchema(pBuf, &(pTbCfg->ntbCfg.pSchema)); + break; + default: + break; + } + + return pBuf; } \ No newline at end of file From 30922ff223bab4cb33f4e6b4b5302b320f2f06fb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 7 Dec 2021 10:08:56 +0800 Subject: [PATCH 18/27] more --- cmake/cmake.options | 2 +- source/dnode/vnode/meta/CMakeLists.txt | 2 -- src/tkv/CMakeLists.txt | 7 ------- 3 files changed, 1 insertion(+), 10 deletions(-) delete mode 100644 src/tkv/CMakeLists.txt diff --git a/cmake/cmake.options b/cmake/cmake.options index 105ff17c58..b3f8ea7bbd 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -16,7 +16,7 @@ option( option( BUILD_WITH_ROCKSDB "If build with rocksdb" - ON + OFF ) option( diff --git a/source/dnode/vnode/meta/CMakeLists.txt b/source/dnode/vnode/meta/CMakeLists.txt index 7e5acc6450..bb48d1acad 100644 --- a/source/dnode/vnode/meta/CMakeLists.txt +++ b/source/dnode/vnode/meta/CMakeLists.txt @@ -15,8 +15,6 @@ elseif(${META_DB_IMPL} STREQUAL "SQLITE") list(REMOVE_ITEM META_SRC "src/metaBDBImpl.c") endif() -message("${META_SRC}") - add_library(meta STATIC ${META_SRC}) target_include_directories( meta diff --git a/src/tkv/CMakeLists.txt b/src/tkv/CMakeLists.txt deleted file mode 100644 index 264a95685f..0000000000 --- a/src/tkv/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -aux_source_directory(source TKV_SRC) -add_library(tkv ${TKV_SRC}) -target_include_directories( - tkv - PUBLIC "${CMAKE_SOURCE_DIR}/include/tkv" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" -) \ No newline at end of file From 40d12ec48e15973c9a8c8b75df1b73d056a368f0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 7 Dec 2021 11:31:49 +0800 Subject: [PATCH 19/27] more --- source/dnode/vnode/meta/inc/metaDB.h | 2 +- source/dnode/vnode/meta/src/metaBDBImpl.c | 159 ++++++++++++++-------- 2 files changed, 100 insertions(+), 61 deletions(-) diff --git a/source/dnode/vnode/meta/inc/metaDB.h b/source/dnode/vnode/meta/inc/metaDB.h index 62b91ed48c..b4b7e5d19e 100644 --- a/source/dnode/vnode/meta/inc/metaDB.h +++ b/source/dnode/vnode/meta/inc/metaDB.h @@ -26,7 +26,7 @@ typedef struct SMetaDB SMetaDB; int metaOpenDB(SMeta *pMeta); void metaCloseDB(SMeta *pMeta); -int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg); +int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg); int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid); #ifdef __cplusplus diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 0565859688..093c849e55 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -13,27 +13,36 @@ * along with this program. If not, see . */ -#include "metaDef.h" - #include "db.h" +#include "metaDef.h" + +#include "thash.h" + struct SMetaDB { - DB * pStbDB; - DB * pCtbDB; - DB * pNtbDB; - DB * pIdx; + // DB + DB * pStbDB; + DB * pNtbDB; + SHashObj *pCtbMap; + DB * pSchemaDB; + // IDX + SHashObj *pIdxMap; + DB * pNameIdx; + DB * pUidIdx; + // ENV DB_ENV *pEvn; }; +static SMetaDB *metaNewDB(); +static void metaFreeDB(SMetaDB *pDB); + int metaOpenDB(SMeta *pMeta) { - int ret; + int ret; + SMetaDB *pDB; - pMeta->pDB = (SMetaDB *)calloc(1, sizeof(SMetaDB)); - if (pMeta->pDB == NULL) { - // TODO: handle error - return -1; - } + pMeta->pDB = metaNewDB(); +#if 0 // create the env ret = db_env_create(&(pMeta->pDB->pEvn), 0); if (ret != 0) { @@ -47,19 +56,13 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - ret = db_create(&(pMeta->pDB->pStbDB), pMeta->pDB->pEvn, 0); + ret = db_create(&(pMeta->pDB->pDB), pMeta->pDB->pEvn, 0); if (ret != 0) { // TODO: handle error return -1; } - ret = db_create(&(pMeta->pDB->pCtbDB), pMeta->pDB->pEvn, 0); - if (ret != 0) { - // TODO: handle error - return -1; - } - - ret = db_create(&(pMeta->pDB->pNtbDB), pMeta->pDB->pEvn, 0); + ret = db_create(&(pMeta->pDB->pSchemaDB), pMeta->pDB->pEvn, 0); if (ret != 0) { // TODO: handle error return -1; @@ -71,37 +74,25 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - ret = pMeta->pDB->pStbDB->open(pMeta->pDB->pStbDB, /* DB structure pointer */ - NULL, /* Transaction pointer */ - "meta.db", /* On-disk file that holds the database */ - NULL, /* Optional logical database name */ - DB_BTREE, /* Database access method */ - DB_CREATE, /* Open flags */ - 0); /* File mode */ + ret = pMeta->pDB->pDB->open(pMeta->pDB->pDB, /* DB structure pointer */ + NULL, /* Transaction pointer */ + "meta.db", /* On-disk file that holds the database */ + NULL, /* Optional logical database name */ + DB_BTREE, /* Database access method */ + DB_CREATE, /* Open flags */ + 0); /* File mode */ if (ret != 0) { // TODO: handle error return -1; } - ret = pMeta->pDB->pCtbDB->open(pMeta->pDB->pCtbDB, /* DB structure pointer */ - NULL, /* Transaction pointer */ - "meta.db", /* On-disk file that holds the database */ - NULL, /* Optional logical database name */ - DB_BTREE, /* Database access method */ - DB_CREATE, /* Open flags */ - 0); /* File mode */ - if (ret != 0) { - // TODO: handle error - return -1; - } - - ret = pMeta->pDB->pNtbDB->open(pMeta->pDB->pNtbDB, /* DB structure pointer */ - NULL, /* Transaction pointer */ - "meta.db", /* On-disk file that holds the database */ - NULL, /* Optional logical database name */ - DB_BTREE, /* Database access method */ - DB_CREATE, /* Open flags */ - 0); /* File mode */ + ret = pMeta->pDB->pSchemaDB->open(pMeta->pDB->pSchemaDB, /* DB structure pointer */ + NULL, /* Transaction pointer */ + "meta.db", /* On-disk file that holds the database */ + NULL, /* Optional logical database name */ + DB_BTREE, /* Database access method */ + DB_CREATE, /* Open flags */ + 0); /* File mode */ if (ret != 0) { // TODO: handle error return -1; @@ -119,30 +110,38 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - // TODO + ret = pMeta->pDB->pDB->associate(pMeta->pDB->pDB, /* Primary database */ + NULL, /* TXN id */ + pMeta->pDB->pIdx, /* Secondary database */ + metaIdxCallback, /* Callback used for key creation */ + 0); /* Flags */ + if (ret != 0) { + // TODO: handle error + return -1; + } +#endif + return 0; } void metaCloseDB(SMeta *pMeta) { + metaFreeDB(pMeta->pDB); + pMeta->pDB = NULL; +#if 0 if (pMeta->pDB) { if (pMeta->pDB->pIdx) { pMeta->pDB->pIdx->close(pMeta->pDB->pIdx, 0); pMeta->pDB->pIdx = NULL; } - if (pMeta->pDB->pNtbDB) { - pMeta->pDB->pNtbDB->close(pMeta->pDB->pNtbDB, 0); - pMeta->pDB->pNtbDB = NULL; + if (pMeta->pDB->pSchemaDB) { + pMeta->pDB->pSchemaDB->close(pMeta->pDB->pSchemaDB, 0); + pMeta->pDB->pSchemaDB = NULL; } - if (pMeta->pDB->pCtbDB) { - pMeta->pDB->pCtbDB->close(pMeta->pDB->pCtbDB, 0); - pMeta->pDB->pCtbDB = NULL; - } - - if (pMeta->pDB->pStbDB) { - pMeta->pDB->pStbDB->close(pMeta->pDB->pStbDB, 0); - pMeta->pDB->pStbDB = NULL; + if (pMeta->pDB->pDB) { + pMeta->pDB->pDB->close(pMeta->pDB->pDB, 0); + pMeta->pDB->pDB = NULL; } if (pMeta->pDB->pEvn) { @@ -152,9 +151,11 @@ void metaCloseDB(SMeta *pMeta) { free(pMeta->pDB); } +#endif } -int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { +int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { +#if 0 tb_uid_t uid; DBT key = {0}; DBT value = {0}; @@ -175,11 +176,49 @@ int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { value.size = metaEncodeTbCfg(&pBuf, pTbCfg); value.data = buf; - pMeta->pDB->pStbDB->put(pMeta->pDB->pStbDB, NULL, &key, &value, 0); + pMeta->pDB->pDB->put(pMeta->pDB->pDB, NULL, &key, &value, 0); +#endif return 0; } int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { // TODO +} + +/* ------------------------ STATIC METHODS ------------------------ */ +static SMetaDB *metaNewDB() { + SMetaDB *pDB; + pDB = (SMetaDB *)calloc(1, sizeof(*pDB)); + if (pDB == NULL) { + return NULL; + } + + pDB->pCtbMap = taosHashInit(0, MurmurHash3_32, false, HASH_NO_LOCK); + if (pDB->pCtbMap == NULL) { + metaFreeDB(pDB); + return NULL; + } + + pDB->pIdxMap = taosHashInit(0, MurmurHash3_32, false, HASH_NO_LOCK); + if (pDB->pIdxMap == NULL) { + metaFreeDB(pDB); + return NULL; + } + + return pDB; +} + +static void metaFreeDB(SMetaDB *pDB) { + if (pDB == NULL) { + if (pDB->pIdxMap) { + taosHashCleanup(pDB->pIdxMap); + } + + if (pDB->pCtbMap) { + taosHashCleanup(pDB->pCtbMap); + } + + free(pDB); + } } \ No newline at end of file From e7ea5a0e903abdd6adbfeeedc9183d151bf05563 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 7 Dec 2021 13:26:32 +0800 Subject: [PATCH 20/27] more --- source/dnode/vnode/meta/src/metaBDBImpl.c | 95 ++++++++++++++--------- 1 file changed, 57 insertions(+), 38 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 093c849e55..d219c71eda 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -33,29 +33,44 @@ struct SMetaDB { DB_ENV *pEvn; }; +#define P_ERROR(info, code) fprintf(stderr, info "reason: %s", db_strerror(code)) + static SMetaDB *metaNewDB(); static void metaFreeDB(SMetaDB *pDB); +static int metaCreateDBEnv(SMetaDB *pDB, const char *path); +static void metaDestroyDBEnv(SMetaDB *pDB); int metaOpenDB(SMeta *pMeta) { int ret; SMetaDB *pDB; pMeta->pDB = metaNewDB(); + if (pMeta->pDB == NULL) { + return -1; + } + + pDB = pMeta->pDB; + + if (metaCreateDBEnv(pDB, pMeta->path) < 0) { + metaCloseDB(pMeta); + return -1; + } + + // ret = db_create(&(pDB->pStbDB), pDB->pEvn, 0); + // if (ret != 0) { + // P_ERROR("failed to create META super table DB", ret); + // metaCloseDB(pMeta); + // return -1; + // } + + // ret = pDB->pStbDB->open(pDB->pStbDB, NULL, "meta.db", NULL, DB_BTREE, DB_CREATE, 0); + // if (ret != 0) { + // P_ERROR("failed to create META super table DB", ret); + // metaCloseDB(pMeta); + // return -1; + // } #if 0 - // create the env - ret = db_env_create(&(pMeta->pDB->pEvn), 0); - if (ret != 0) { - // TODO: handle error - return -1; - } - - ret = pMeta->pDB->pEvn->open(pMeta->pDB->pEvn, pMeta->path, DB_CREATE | DB_INIT_MPOOL, 0); - if (ret != 0) { - // TODO: handle error - return -1; - } - ret = db_create(&(pMeta->pDB->pDB), pMeta->pDB->pEvn, 0); if (ret != 0) { // TODO: handle error @@ -125,33 +140,11 @@ int metaOpenDB(SMeta *pMeta) { } void metaCloseDB(SMeta *pMeta) { - metaFreeDB(pMeta->pDB); - pMeta->pDB = NULL; -#if 0 if (pMeta->pDB) { - if (pMeta->pDB->pIdx) { - pMeta->pDB->pIdx->close(pMeta->pDB->pIdx, 0); - pMeta->pDB->pIdx = NULL; - } - - if (pMeta->pDB->pSchemaDB) { - pMeta->pDB->pSchemaDB->close(pMeta->pDB->pSchemaDB, 0); - pMeta->pDB->pSchemaDB = NULL; - } - - if (pMeta->pDB->pDB) { - pMeta->pDB->pDB->close(pMeta->pDB->pDB, 0); - pMeta->pDB->pDB = NULL; - } - - if (pMeta->pDB->pEvn) { - pMeta->pDB->pEvn->close(pMeta->pDB->pEvn, 0); - pMeta->pDB->pEvn = NULL; - } - - free(pMeta->pDB); + metaDestroyDBEnv(pMeta->pDB); + metaFreeDB(pMeta->pDB); + pMeta->pDB = NULL; } -#endif } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { @@ -221,4 +214,30 @@ static void metaFreeDB(SMetaDB *pDB) { free(pDB); } +} + +static int metaCreateDBEnv(SMetaDB *pDB, const char *path) { + int ret; + + if (path == NULL) return 0; + + ret = db_env_create(&(pDB->pEvn), 0); + if (ret != 0) { + P_ERROR("Failed to create META DB ENV", ret); + return -1; + } + + ret = pDB->pEvn->open(pDB->pEvn, path, DB_CREATE | DB_INIT_MPOOL, 0); + if (ret != 0) { + P_ERROR("failed to open META DB ENV", ret); + return -1; + } + + return 0; +} + +static void metaDestroyDBEnv(SMetaDB *pDB) { + if (pDB->pEvn) { + pDB->pEvn->close(pDB->pEvn, 0); + } } \ No newline at end of file From 9048c77ecaf0ee382288be6f8aa130d883601210 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 7 Dec 2021 16:18:26 +0800 Subject: [PATCH 21/27] more --- source/dnode/vnode/meta/src/metaBDBImpl.c | 127 ++++++---------------- 1 file changed, 31 insertions(+), 96 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index d219c71eda..8aefeaf1d8 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -40,6 +40,25 @@ static void metaFreeDB(SMetaDB *pDB); static int metaCreateDBEnv(SMetaDB *pDB, const char *path); static void metaDestroyDBEnv(SMetaDB *pDB); +#define META_OPEN_DB(pDB, pEnv, fName) \ + do { \ + ret = db_create(&((pDB)), (pEnv), 0); \ + if (ret != 0) { \ + P_ERROR("Failed to create META DB", ret); \ + metaCloseDB(pMeta); \ + return -1; \ + } \ + \ + ret = (pDB)->open((pDB), NULL, (fName), NULL, DB_BTREE, DB_CREATE, 0); \ + if (ret != 0) { \ + P_ERROR("Failed to open META DB", ret); \ + metaCloseDB(pMeta); \ + return -1; \ + } \ + } while (0) + +#define META_CLOSE_DB(pDB) + int metaOpenDB(SMeta *pMeta) { int ret; SMetaDB *pDB; @@ -56,86 +75,25 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - // ret = db_create(&(pDB->pStbDB), pDB->pEvn, 0); - // if (ret != 0) { - // P_ERROR("failed to create META super table DB", ret); - // metaCloseDB(pMeta); - // return -1; - // } + META_OPEN_DB(pDB->pStbDB, pDB->pEvn, "meta.db"); - // ret = pDB->pStbDB->open(pDB->pStbDB, NULL, "meta.db", NULL, DB_BTREE, DB_CREATE, 0); - // if (ret != 0) { - // P_ERROR("failed to create META super table DB", ret); - // metaCloseDB(pMeta); - // return -1; - // } + META_OPEN_DB(pDB->pNtbDB, pDB->pEvn, "meta.db"); -#if 0 - ret = db_create(&(pMeta->pDB->pDB), pMeta->pDB->pEvn, 0); - if (ret != 0) { - // TODO: handle error - return -1; + META_OPEN_DB(pDB->pSchemaDB, pDB->pEvn, "meta.db"); + + { + // TODO: Loop to open each super table db } - ret = db_create(&(pMeta->pDB->pSchemaDB), pMeta->pDB->pEvn, 0); - if (ret != 0) { - // TODO: handle error - return -1; - } + META_OPEN_DB(pDB->pNameIdx, pDB->pEvn, "index.db"); - ret = db_create(&(pMeta->pDB->pIdx), pMeta->pDB->pEvn, 0); - if (ret != 0) { - // TODO: handle error - return -1; - } + META_OPEN_DB(pDB->pUidIdx, pDB->pEvn, "index.db"); - ret = pMeta->pDB->pDB->open(pMeta->pDB->pDB, /* DB structure pointer */ - NULL, /* Transaction pointer */ - "meta.db", /* On-disk file that holds the database */ - NULL, /* Optional logical database name */ - DB_BTREE, /* Database access method */ - DB_CREATE, /* Open flags */ - 0); /* File mode */ - if (ret != 0) { - // TODO: handle error - return -1; + { + // TODO: Loop to open index DB for each super table + // and create the association between main DB and index } - ret = pMeta->pDB->pSchemaDB->open(pMeta->pDB->pSchemaDB, /* DB structure pointer */ - NULL, /* Transaction pointer */ - "meta.db", /* On-disk file that holds the database */ - NULL, /* Optional logical database name */ - DB_BTREE, /* Database access method */ - DB_CREATE, /* Open flags */ - 0); /* File mode */ - if (ret != 0) { - // TODO: handle error - return -1; - } - - ret = pMeta->pDB->pIdx->open(pMeta->pDB->pIdx, /* DB structure pointer */ - NULL, /* Transaction pointer */ - "index.db", /* On-disk file that holds the database */ - NULL, /* Optional logical database name */ - DB_BTREE, /* Database access method */ - DB_CREATE, /* Open flags */ - 0); /* File mode */ - if (ret != 0) { - // TODO: handle error - return -1; - } - - ret = pMeta->pDB->pDB->associate(pMeta->pDB->pDB, /* Primary database */ - NULL, /* TXN id */ - pMeta->pDB->pIdx, /* Secondary database */ - metaIdxCallback, /* Callback used for key creation */ - 0); /* Flags */ - if (ret != 0) { - // TODO: handle error - return -1; - } -#endif - return 0; } @@ -148,30 +106,7 @@ void metaCloseDB(SMeta *pMeta) { } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { -#if 0 - tb_uid_t uid; - DBT key = {0}; - DBT value = {0}; - char buf[256]; - void * pBuf; - int bsize; - - if (pTbCfg->type == META_SUPER_TABLE) { - uid = pTbCfg->stbCfg.suid; - } else { - uid = metaGenerateUid(pMeta); - } - - key.size = sizeof(uid); - key.data = &uid; - - pBuf = buf; - value.size = metaEncodeTbCfg(&pBuf, pTbCfg); - value.data = buf; - - pMeta->pDB->pDB->put(pMeta->pDB->pDB, NULL, &key, &value, 0); -#endif - + // TODO return 0; } From c448c156739069715edf2b38b2fde33298281ae7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 7 Dec 2021 17:57:17 +0800 Subject: [PATCH 22/27] more work --- source/dnode/vnode/meta/src/metaBDBImpl.c | 161 +++++++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 8aefeaf1d8..857eff793e 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -17,8 +17,14 @@ #include "metaDef.h" +#include "tcoding.h" #include "thash.h" +typedef struct { + tb_uid_t uid; + int32_t sver; +} SSchemaKey; + struct SMetaDB { // DB DB * pStbDB; @@ -39,9 +45,18 @@ static SMetaDB *metaNewDB(); static void metaFreeDB(SMetaDB *pDB); static int metaCreateDBEnv(SMetaDB *pDB, const char *path); static void metaDestroyDBEnv(SMetaDB *pDB); +static int metaEncodeSchemaKey(void **buf, SSchemaKey *pSchemaKey); +static void * metaDecodeSchemaKey(void *buf, SSchemaKey *pSchemaKey); +static int metaNameIdxCb(DB *sdbp, const DBT *pKey, const DBT *pValue, DBT *pSKey); +static int metaUidIdxCb(DB *sdbp, const DBT *pKey, const DBT *pValue, DBT *pSKey); +static void metaPutSchema(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema); +static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); +static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); +static int metaSaveTbInfo(DB *pDB, tb_uid_t uid, STbCfg *pTbCfg); #define META_OPEN_DB(pDB, pEnv, fName) \ do { \ + int ret; \ ret = db_create(&((pDB)), (pEnv), 0); \ if (ret != 0) { \ P_ERROR("Failed to create META DB", ret); \ @@ -59,6 +74,15 @@ static void metaDestroyDBEnv(SMetaDB *pDB); #define META_CLOSE_DB(pDB) +#define META_ASSOCIATE_IDX(pDB, pNameIdx, cbf) \ + do { \ + int ret = (pDB)->associate((pDB), NULL, (pNameIdx), (cbf), 0); \ + if (ret != 0) { \ + P_ERROR("Failed to associate META DB", ret); \ + metaCloseDB(pMeta); \ + } \ + } while (0) + int metaOpenDB(SMeta *pMeta) { int ret; SMetaDB *pDB; @@ -89,6 +113,17 @@ int metaOpenDB(SMeta *pMeta) { META_OPEN_DB(pDB->pUidIdx, pDB->pEvn, "index.db"); + // Associate name index + META_ASSOCIATE_IDX(pDB->pStbDB, pDB->pNameIdx, metaNameIdxCb); + META_ASSOCIATE_IDX(pDB->pStbDB, pDB->pUidIdx, metaUidIdxCb); + META_ASSOCIATE_IDX(pDB->pNtbDB, pDB->pNameIdx, metaNameIdxCb); + META_ASSOCIATE_IDX(pDB->pNtbDB, pDB->pUidIdx, metaUidIdxCb); + + for (;;) { + // Loop to associate each super table db + break; + } + { // TODO: Loop to open index DB for each super table // and create the association between main DB and index @@ -106,7 +141,56 @@ void metaCloseDB(SMeta *pMeta) { } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { - // TODO + char buf[512]; + void * pBuf; + DBT key = {0}; + DBT value = {0}; + SSchemaKey schemaKey; + tb_uid_t uid; + + if (pTbCfg->type == META_SUPER_TABLE) { + // Handle SUPER table + uid = pTbCfg->stbCfg.suid; + + // Same table info + metaSaveTbInfo(pMeta->pDB->pStbDB, uid, pTbCfg); + + // save schema + metaPutSchema(pMeta, uid, pTbCfg->stbCfg.pSchema); + + { + // Create a super table DB and corresponding index DB + DB *pStbDB; + DB *pStbIdxDB; + + META_OPEN_DB(pStbDB, pMeta->pDB->pEvn, "meta.db"); + + META_OPEN_DB(pStbIdxDB, pMeta->pDB->pEvn, "index.db"); + + // TODO META_ASSOCIATE_IDX(); + } + } else if (pTbCfg->type == META_CHILD_TABLE) { + // Handle CHILD table + uid = metaGenerateUid(pMeta); + + DB *pCTbDB = taosHashGet(pMeta->pDB->pCtbMap, &(pTbCfg->ctbCfg.suid), sizeof(pTbCfg->ctbCfg.suid)); + if (pCTbDB == NULL) { + ASSERT(0); + } + + metaSaveTbInfo(pCTbDB, uid, pTbCfg); + + } else if (pTbCfg->type == META_NORMAL_TABLE) { + // Handle NORMAL table + uid = metaGenerateUid(pMeta); + + metaSaveTbInfo(pMeta->pDB->pNtbDB, uid, pTbCfg); + + metaPutSchema(pMeta, uid, pTbCfg->stbCfg.pSchema); + } else { + ASSERT(0); + } + return 0; } @@ -175,4 +259,79 @@ static void metaDestroyDBEnv(SMetaDB *pDB) { if (pDB->pEvn) { pDB->pEvn->close(pDB->pEvn, 0); } +} + +static int metaEncodeSchemaKey(void **buf, SSchemaKey *pSchemaKey) { + int tsize = 0; + + tsize += taosEncodeFixedU64(buf, pSchemaKey->uid); + tsize += taosEncodeFixedI32(buf, pSchemaKey->sver); + + return tsize; +} + +static void *metaDecodeSchemaKey(void *buf, SSchemaKey *pSchemaKey) { + buf = taosDecodeFixedU64(buf, &(pSchemaKey->uid)); + buf = taosDecodeFixedI32(buf, &(pSchemaKey->sver)); + + return buf; +} + +static int metaNameIdxCb(DB *sdbp, const DBT *pKey, const DBT *pValue, DBT *pSKey) { + // TODO + return 0; +} + +static int metaUidIdxCb(DB *sdbp, const DBT *pKey, const DBT *pValue, DBT *pSKey) { + // TODO + return 0; +} + +static void metaPutSchema(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema) { + SSchemaKey skey; + char buf[256]; + void * pBuf = buf; + DBT key = {0}; + DBT value = {0}; + + skey.uid = uid; + skey.sver = schemaVersion(pSchema); + + key.data = &skey; + key.size = sizeof(skey); + + tdEncodeSchema(&pBuf, pSchema); + value.data = buf; + value.size = POINTER_DISTANCE(pBuf, buf); + + pMeta->pDB->pSchemaDB->put(pMeta->pDB->pSchemaDB, NULL, &key, &value, 0); +} + +static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { + // TODO + return 0; +} + +static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { + // TODO + return NULL; +} + +static int metaSaveTbInfo(DB *pDB, tb_uid_t uid, STbCfg *pTbCfg) { + DBT key = {0}; + DBT value = {0}; + char buf[512]; + void *pBuf = buf; + + key.data = &uid; + key.size = sizeof(uid); + + metaEncodeTbInfo(&pBuf, pTbCfg); + + value.data = buf; + value.size = POINTER_DISTANCE(pBuf, buf); + + pDB->put(pDB, NULL, &key, &value, 0); + + return 0; } \ No newline at end of file From f66db4b04846af43edc2ae758133e4bac41782cc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 7 Dec 2021 18:21:16 +0800 Subject: [PATCH 23/27] more --- source/dnode/vnode/meta/src/metaBDBImpl.c | 46 ++++++++++++++++++----- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 857eff793e..f57145b119 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -74,9 +74,9 @@ static int metaSaveTbInfo(DB *pDB, tb_uid_t uid, STbCfg *pTbCfg); #define META_CLOSE_DB(pDB) -#define META_ASSOCIATE_IDX(pDB, pNameIdx, cbf) \ +#define META_ASSOCIATE_IDX(pDB, pIdx, cbf) \ do { \ - int ret = (pDB)->associate((pDB), NULL, (pNameIdx), (cbf), 0); \ + int ret = (pDB)->associate((pDB), NULL, (pIdx), (cbf), 0); \ if (ret != 0) { \ P_ERROR("Failed to associate META DB", ret); \ metaCloseDB(pMeta); \ @@ -111,13 +111,13 @@ int metaOpenDB(SMeta *pMeta) { META_OPEN_DB(pDB->pNameIdx, pDB->pEvn, "index.db"); - META_OPEN_DB(pDB->pUidIdx, pDB->pEvn, "index.db"); + // META_OPEN_DB(pDB->pUidIdx, pDB->pEvn, "index.db"); // Associate name index META_ASSOCIATE_IDX(pDB->pStbDB, pDB->pNameIdx, metaNameIdxCb); - META_ASSOCIATE_IDX(pDB->pStbDB, pDB->pUidIdx, metaUidIdxCb); - META_ASSOCIATE_IDX(pDB->pNtbDB, pDB->pNameIdx, metaNameIdxCb); - META_ASSOCIATE_IDX(pDB->pNtbDB, pDB->pUidIdx, metaUidIdxCb); + // META_ASSOCIATE_IDX(pDB->pStbDB, pDB->pUidIdx, metaUidIdxCb); + // META_ASSOCIATE_IDX(pDB->pNtbDB, pDB->pNameIdx, metaNameIdxCb); + // META_ASSOCIATE_IDX(pDB->pNtbDB, pDB->pUidIdx, metaUidIdxCb); for (;;) { // Loop to associate each super table db @@ -308,13 +308,41 @@ static void metaPutSchema(SMeta *pMeta, tb_uid_t uid, STSchema *pSchema) { } static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { - // TODO - return 0; + int tsize = 0; + + tsize += taosEncodeString(buf, pTbCfg->name); + tsize += taosEncodeFixedU32(buf, pTbCfg->ttl); + tsize += taosEncodeFixedU32(buf, pTbCfg->keep); + + if (pTbCfg->type == META_SUPER_TABLE) { + tsize += tdEncodeSchema(buf, pTbCfg->stbCfg.pTagSchema); + } else if (pTbCfg->type == META_CHILD_TABLE) { + tsize += taosEncodeFixedU64(buf, pTbCfg->ctbCfg.suid); + tsize += tdEncodeKVRow(buf, pTbCfg->ctbCfg.pTag); + } else if (pTbCfg->type == META_NORMAL_TABLE) { + } else { + ASSERT(0); + } + + return tsize; } static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { // TODO - return NULL; + buf = taosDecodeString(buf, &(pTbCfg->name)); + buf = taosDecodeFixedU32(buf, &(pTbCfg->ttl)); + buf = taosDecodeFixedU32(buf, &(pTbCfg->keep)); + + if (pTbCfg->type == META_SUPER_TABLE) { + buf = tdDecodeSchema(buf, &(pTbCfg->stbCfg.pTagSchema)); + } else if (pTbCfg->type == META_CHILD_TABLE) { + buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid)); + buf = tdDecodeKVRow(buf, &(pTbCfg->ctbCfg.pTag)); + } else if (pTbCfg->type == META_NORMAL_TABLE) { + } else { + ASSERT(0); + } + return buf; } static int metaSaveTbInfo(DB *pDB, tb_uid_t uid, STbCfg *pTbCfg) { From abfdaddec4eb99657590373680361ea0e6cad271 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 7 Dec 2021 18:28:33 +0800 Subject: [PATCH 24/27] more --- source/dnode/vnode/meta/src/metaBDBImpl.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index f57145b119..026bcfdafb 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -75,12 +75,12 @@ static int metaSaveTbInfo(DB *pDB, tb_uid_t uid, STbCfg *pTbCfg); #define META_CLOSE_DB(pDB) #define META_ASSOCIATE_IDX(pDB, pIdx, cbf) \ - do { \ + do { \ int ret = (pDB)->associate((pDB), NULL, (pIdx), (cbf), 0); \ - if (ret != 0) { \ - P_ERROR("Failed to associate META DB", ret); \ - metaCloseDB(pMeta); \ - } \ + if (ret != 0) { \ + P_ERROR("Failed to associate META DB", ret); \ + metaCloseDB(pMeta); \ + } \ } while (0) int metaOpenDB(SMeta *pMeta) { From 35e5120c96716985e60abbbe8653179b7d7b0b2a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 8 Dec 2021 16:27:56 +0800 Subject: [PATCH 25/27] more --- cmake/cmake.options | 2 +- source/dnode/vnode/meta/src/metaBDBImpl.c | 322 +++++++++++----------- 2 files changed, 169 insertions(+), 155 deletions(-) diff --git a/cmake/cmake.options b/cmake/cmake.options index b3f8ea7bbd..edaab3bd45 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -22,7 +22,7 @@ option( option( BUILD_WITH_SQLITE "If build with sqlite" - ON + OFF ) option( diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 026bcfdafb..7bbe7ddc71 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -20,26 +20,177 @@ #include "tcoding.h" #include "thash.h" +struct SMetaDB { + // DB + DB *pTbDB; + DB *pSchemaDB; + // IDX + DB *pNameIdx; + DB *pStbIdx; + DB *pNtbIdx; + DB *pCtbIdx; + // ENV + DB_ENV *pEvn; +}; + +static SMetaDB *metaNewDB(); +static void metaFreeDB(SMetaDB *pDB); +static int metaOpenBDBEnv(DB_ENV **ppEnv, const char *path); +static void metaCloseBDBEnv(DB_ENV *pEnv); +static int metaOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName); +static void metaCloseBDBDb(DB *pDB); + +#define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) +#define metaOpenBDBIdx metaOpenBDBDb + +int metaOpenDB(SMeta *pMeta) { + SMetaDB *pDB; + + // Create DB object + pDB = metaNewDB(); + if (pDB == NULL) { + return -1; + } + + pMeta->pDB = pDB; + + // Open DB Env + if (metaOpenBDBEnv(&(pDB->pEvn), pMeta->path) < 0) { + metaCloseDB(pMeta); + return -1; + } + + // Open DBs + if (metaOpenBDBDb(&(pDB->pTbDB), pDB->pEvn, "meta.db") < 0) { + metaCloseDB(pMeta); + return -1; + } + + if (metaOpenBDBDb(&(pDB->pSchemaDB), pDB->pEvn, "meta.db") < 0) { + metaCloseDB(pMeta); + return -1; + } + + // Open Indices + if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "index.db") < 0) { + metaCloseDB(pMeta); + return -1; + } + + if (metaOpenBDBIdx(&(pDB->pStbIdx), pDB->pEvn, "index.db") < 0) { + metaCloseDB(pMeta); + return -1; + } + + if (metaOpenBDBIdx(&(pDB->pNtbIdx), pDB->pEvn, "index.db") < 0) { + metaCloseDB(pMeta); + return -1; + } + + if (metaOpenBDBIdx(&(pDB->pCtbIdx), pDB->pEvn, "index.db") < 0) { + metaCloseDB(pMeta); + return -1; + } + // Associate Indices + + return 0; +} + +void metaCloseDB(SMeta *pMeta) { + if (pMeta->pDB) { + metaCloseBDBEnv(pMeta->pDB->pEvn); + metaFreeDB(pMeta->pDB); + pMeta->pDB = NULL; + } +} + +int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { + // TODO + return 0; +} + +int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { + // TODO + return 0; +} + +/* ------------------------ STATIC METHODS ------------------------ */ +static SMetaDB *metaNewDB() { + SMetaDB *pDB = NULL; + pDB = (SMetaDB *)calloc(1, sizeof(*pDB)); + if (pDB == NULL) { + return NULL; + } + + return pDB; +} + +static void metaFreeDB(SMetaDB *pDB) { + if (pDB) { + free(pDB); + } +} + +static int metaOpenBDBEnv(DB_ENV **ppEnv, const char *path) { + int ret; + DB_ENV *pEnv; + + if (path == NULL) return 0; + + ret = db_env_create(&pEnv, 0); + if (ret != 0) { + BDB_PERR("Failed to create META env", ret); + return -1; + } + + ret = pEnv->open(pEnv, path, DB_CREATE | DB_INIT_MPOOL, 0); + if (ret != 0) { + BDB_PERR("Failed to open META env", ret); + return -1; + } + + *ppEnv = pEnv; + + return 0; +} + +static void metaCloseBDBEnv(DB_ENV *pEnv) { + if (pEnv) { + pEnv->close(pEnv, 0); + } +} + +static int metaOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName) { + int ret; + DB *pDB; + + ret = db_create(&((pDB)), (pEnv), 0); + if (ret != 0) { + BDB_PERR("Failed to create META DB", ret); + return -1; + } + + ret = pDB->open(pDB, NULL, pFName, NULL, DB_BTREE, DB_CREATE, 0); + if (ret) { + BDB_PERR("Failed to open META DB", ret); + return -1; + } + + return 0; +} + +static void metaCloseBDBDb(DB *pDB) { + if (pDB) { + pDB->close(pDB, 0); + } +} + +#if 0 typedef struct { tb_uid_t uid; int32_t sver; } SSchemaKey; -struct SMetaDB { - // DB - DB * pStbDB; - DB * pNtbDB; - SHashObj *pCtbMap; - DB * pSchemaDB; - // IDX - SHashObj *pIdxMap; - DB * pNameIdx; - DB * pUidIdx; - // ENV - DB_ENV *pEvn; -}; - -#define P_ERROR(info, code) fprintf(stderr, info "reason: %s", db_strerror(code)) static SMetaDB *metaNewDB(); static void metaFreeDB(SMetaDB *pDB); @@ -54,26 +205,6 @@ static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static int metaSaveTbInfo(DB *pDB, tb_uid_t uid, STbCfg *pTbCfg); -#define META_OPEN_DB(pDB, pEnv, fName) \ - do { \ - int ret; \ - ret = db_create(&((pDB)), (pEnv), 0); \ - if (ret != 0) { \ - P_ERROR("Failed to create META DB", ret); \ - metaCloseDB(pMeta); \ - return -1; \ - } \ - \ - ret = (pDB)->open((pDB), NULL, (fName), NULL, DB_BTREE, DB_CREATE, 0); \ - if (ret != 0) { \ - P_ERROR("Failed to open META DB", ret); \ - metaCloseDB(pMeta); \ - return -1; \ - } \ - } while (0) - -#define META_CLOSE_DB(pDB) - #define META_ASSOCIATE_IDX(pDB, pIdx, cbf) \ do { \ int ret = (pDB)->associate((pDB), NULL, (pIdx), (cbf), 0); \ @@ -83,62 +214,6 @@ static int metaSaveTbInfo(DB *pDB, tb_uid_t uid, STbCfg *pTbCfg); } \ } while (0) -int metaOpenDB(SMeta *pMeta) { - int ret; - SMetaDB *pDB; - - pMeta->pDB = metaNewDB(); - if (pMeta->pDB == NULL) { - return -1; - } - - pDB = pMeta->pDB; - - if (metaCreateDBEnv(pDB, pMeta->path) < 0) { - metaCloseDB(pMeta); - return -1; - } - - META_OPEN_DB(pDB->pStbDB, pDB->pEvn, "meta.db"); - - META_OPEN_DB(pDB->pNtbDB, pDB->pEvn, "meta.db"); - - META_OPEN_DB(pDB->pSchemaDB, pDB->pEvn, "meta.db"); - - { - // TODO: Loop to open each super table db - } - - META_OPEN_DB(pDB->pNameIdx, pDB->pEvn, "index.db"); - - // META_OPEN_DB(pDB->pUidIdx, pDB->pEvn, "index.db"); - - // Associate name index - META_ASSOCIATE_IDX(pDB->pStbDB, pDB->pNameIdx, metaNameIdxCb); - // META_ASSOCIATE_IDX(pDB->pStbDB, pDB->pUidIdx, metaUidIdxCb); - // META_ASSOCIATE_IDX(pDB->pNtbDB, pDB->pNameIdx, metaNameIdxCb); - // META_ASSOCIATE_IDX(pDB->pNtbDB, pDB->pUidIdx, metaUidIdxCb); - - for (;;) { - // Loop to associate each super table db - break; - } - - { - // TODO: Loop to open index DB for each super table - // and create the association between main DB and index - } - - return 0; -} - -void metaCloseDB(SMeta *pMeta) { - if (pMeta->pDB) { - metaDestroyDBEnv(pMeta->pDB); - metaFreeDB(pMeta->pDB); - pMeta->pDB = NULL; - } -} int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { char buf[512]; @@ -199,68 +274,6 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { } /* ------------------------ STATIC METHODS ------------------------ */ -static SMetaDB *metaNewDB() { - SMetaDB *pDB; - pDB = (SMetaDB *)calloc(1, sizeof(*pDB)); - if (pDB == NULL) { - return NULL; - } - - pDB->pCtbMap = taosHashInit(0, MurmurHash3_32, false, HASH_NO_LOCK); - if (pDB->pCtbMap == NULL) { - metaFreeDB(pDB); - return NULL; - } - - pDB->pIdxMap = taosHashInit(0, MurmurHash3_32, false, HASH_NO_LOCK); - if (pDB->pIdxMap == NULL) { - metaFreeDB(pDB); - return NULL; - } - - return pDB; -} - -static void metaFreeDB(SMetaDB *pDB) { - if (pDB == NULL) { - if (pDB->pIdxMap) { - taosHashCleanup(pDB->pIdxMap); - } - - if (pDB->pCtbMap) { - taosHashCleanup(pDB->pCtbMap); - } - - free(pDB); - } -} - -static int metaCreateDBEnv(SMetaDB *pDB, const char *path) { - int ret; - - if (path == NULL) return 0; - - ret = db_env_create(&(pDB->pEvn), 0); - if (ret != 0) { - P_ERROR("Failed to create META DB ENV", ret); - return -1; - } - - ret = pDB->pEvn->open(pDB->pEvn, path, DB_CREATE | DB_INIT_MPOOL, 0); - if (ret != 0) { - P_ERROR("failed to open META DB ENV", ret); - return -1; - } - - return 0; -} - -static void metaDestroyDBEnv(SMetaDB *pDB) { - if (pDB->pEvn) { - pDB->pEvn->close(pDB->pEvn, 0); - } -} - static int metaEncodeSchemaKey(void **buf, SSchemaKey *pSchemaKey) { int tsize = 0; @@ -362,4 +375,5 @@ static int metaSaveTbInfo(DB *pDB, tb_uid_t uid, STbCfg *pTbCfg) { pDB->put(pDB, NULL, &key, &value, 0); return 0; -} \ No newline at end of file +} +#endif \ No newline at end of file From b59a375654572fd709955d32647bfd69e6a9158e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 8 Dec 2021 17:17:03 +0800 Subject: [PATCH 26/27] more work --- source/dnode/vnode/meta/src/metaBDBImpl.c | 62 ++++++++++++++++++++--- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/meta/src/metaBDBImpl.c index 7bbe7ddc71..9e52a7151d 100644 --- a/source/dnode/vnode/meta/src/metaBDBImpl.c +++ b/source/dnode/vnode/meta/src/metaBDBImpl.c @@ -33,15 +33,21 @@ struct SMetaDB { DB_ENV *pEvn; }; +typedef int (*bdbIdxCbPtr)(DB *, const DBT *, const DBT *, DBT *); + static SMetaDB *metaNewDB(); static void metaFreeDB(SMetaDB *pDB); static int metaOpenBDBEnv(DB_ENV **ppEnv, const char *path); static void metaCloseBDBEnv(DB_ENV *pEnv); static int metaOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName); static void metaCloseBDBDb(DB *pDB); +static int metaOpenBDBIdx(DB **ppIdx, DB_ENV *pEnv, const char *pFName, DB *pDB, bdbIdxCbPtr cbf); +static int metaNameIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); +static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); +static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); +static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey); #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) -#define metaOpenBDBIdx metaOpenBDBDb int metaOpenDB(SMeta *pMeta) { SMetaDB *pDB; @@ -72,26 +78,25 @@ int metaOpenDB(SMeta *pMeta) { } // Open Indices - if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "index.db") < 0) { + if (metaOpenBDBIdx(&(pDB->pNameIdx), pDB->pEvn, "index.db", pDB->pTbDB, &metaNameIdxCb) < 0) { metaCloseDB(pMeta); return -1; } - if (metaOpenBDBIdx(&(pDB->pStbIdx), pDB->pEvn, "index.db") < 0) { + if (metaOpenBDBIdx(&(pDB->pStbIdx), pDB->pEvn, "index.db", pDB->pTbDB, &metaStbIdxCb) < 0) { metaCloseDB(pMeta); return -1; } - if (metaOpenBDBIdx(&(pDB->pNtbIdx), pDB->pEvn, "index.db") < 0) { + if (metaOpenBDBIdx(&(pDB->pNtbIdx), pDB->pEvn, "index.db", pDB->pTbDB, &metaNtbIdxCb) < 0) { metaCloseDB(pMeta); return -1; } - if (metaOpenBDBIdx(&(pDB->pCtbIdx), pDB->pEvn, "index.db") < 0) { + if (metaOpenBDBIdx(&(pDB->pCtbIdx), pDB->pEvn, "index.db", pDB->pTbDB, &metaCtbIdxCb) < 0) { metaCloseDB(pMeta); return -1; } - // Associate Indices return 0; } @@ -176,6 +181,8 @@ static int metaOpenBDBDb(DB **ppDB, DB_ENV *pEnv, const char *pFName) { return -1; } + *ppDB = pDB; + return 0; } @@ -185,6 +192,49 @@ static void metaCloseBDBDb(DB *pDB) { } } +static int metaOpenBDBIdx(DB **ppIdx, DB_ENV *pEnv, const char *pFName, DB *pDB, bdbIdxCbPtr cbf) { + DB *pIdx; + int ret; + + if (metaOpenBDBDb(ppIdx, pEnv, pFName) < 0) { + return -1; + } + + pIdx = *ppIdx; + ret = pDB->associate(pDB, NULL, pIdx, cbf, 0); + if (ret) { + BDB_PERR("Failed to associate META DB and Index", ret); + } + + return 0; +} + +static void metaCloseBDBIdx(DB *pIdx) { + if (pIdx) { + pIdx->close(pIdx, 0); + } +} + +static int metaNameIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { + // TODO + return 0; +} + +static int metaStbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { + // TODO + return 0; +} + +static int metaNtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { + // TODO + return 0; +} + +static int metaCtbIdxCb(DB *pIdx, const DBT *pKey, const DBT *pValue, DBT *pSKey) { + // TODO + return 0; +} + #if 0 typedef struct { tb_uid_t uid; From 73d938b09a4e48faff6062e080164c8380d9cfb5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Dec 2021 18:07:13 +0800 Subject: [PATCH 27/27] add test case --- source/libs/index/src/index_fst.c | 14 ++++++++------ source/libs/index/test/indexTests.cpp | 7 ++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 0cc95b738d..40e35306a2 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -167,7 +167,10 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, break; } if (addPrefix != 0) { - fstBuilderNodeUnfinishedAddOutputPrefix(un, addPrefix); + if (i + 1 < ssz) { + FstBuilderNodeUnfinished *unf = taosArrayGet(node->stack, i + 1); + fstBuilderNodeUnfinishedAddOutputPrefix(unf, addPrefix); + } } } return i; @@ -581,14 +584,13 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { - 1 // pack size - node->nTrans; uint64_t end = start + node->nTrans; - uint64_t len = end - start; - int32_t dlen = 0; - uint8_t *data = fstSliceData(slice, &dlen); + FstSlice t = fstSliceCopy(slice, start, end - 1); + int32_t len = 0; + uint8_t *data = fstSliceData(&t, &len); for(int i = 0; i < len; i++) { //uint8_t v = slice->data[slice->start + i]; ////slice->data[slice->start + i]; uint8_t v = data[i]; - if (v == b) { return node->nTrans - i - 1; // bug } @@ -1060,7 +1062,7 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { } *out = tOut; - return false; + return true; } FstNode *fstGetRoot(Fst *fst) { diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 858861529d..9135a7a173 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -83,7 +83,7 @@ int main(int argc, char** argv) { std::string str("aaa"); str[2] = 'a' + i ; FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); - Output val = 2; + Output val = 0; fstBuilderInsert(b, key, val); } @@ -108,13 +108,14 @@ int main(int argc, char** argv) { Fst *fst = fstCreate(&s); { - std::string str("aaa"); + std::string str("aax"); uint64_t out; - + FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); bool ok = fstGet(fst, &key, &out); if (ok == true) { + printf("val = %d\n", out); //indexInfo("Get key-value success, %s, %d", str.c_str(), out); } else { //indexError("Get key-value failed, %s", str.c_str());