From 8ff248a308bb4135e62537bf418bfaeaeb1493b6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 06:35:00 +0000 Subject: [PATCH 01/22] meta with tdb compile --- source/dnode/vnode/CMakeLists.txt | 11 +- source/dnode/vnode/src/meta/metaTDBImpl.c | 134 ++++++++++++---------- 2 files changed, 79 insertions(+), 66 deletions(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index bdc8a71b04..6b21f935c4 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,5 +1,5 @@ set(META_DB_IMPL_LIST "BDB" "TDB") -set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") +set(META_DB_IMPL "TDB" CACHE STRING "Use BDB as the default META implementation") set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) @@ -46,14 +46,11 @@ target_link_libraries( PUBLIC stream PUBLIC qworker PUBLIC sync + # TODO: get rid of BDB + PUBLIC bdb + PUBLIC tdb ) -if(${META_DB_IMPL} STREQUAL "BDB") - target_link_libraries(vnode PUBLIC bdb) -elseif(${META_DB_IMPL} STREQUAL "TDB") - target_link_libraries(vnode PUBLIC tdb) -endif() - if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index f4b450b4a8..344fcbe97c 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -15,75 +15,27 @@ #include "metaDef.h" -#include "tdb.h" +#include "tdbInt.h" struct SMetaDB { TENV *pEnv; - TDB * pTbDB; - TDB * pSchemaDB; - TDB * pNameIdx; - TDB * pStbIdx; - TDB * pNtbIdx; - TDB * pCtbIdx; - // tag index hash table - // suid+colid --> TDB * - struct { - } tagIdxHt; + TDB *pTbDB; + TDB *pSchemaDB; + TDB *pNameIdx; + TDB *pStbIdx; + TDB *pNtbIdx; + TDB *pCtbIdx; }; -#define A(op, flag) \ - do { \ - if ((ret = op) != 0) goto flag; \ - } while (0) - int metaOpenDB(SMeta *pMeta) { - SMetaDB *pDb; - TENV * pEnv; - TDB * pTbDB; - TDB * pSchemaDB; - TDB * pNameIdx; - TDB * pStbIdx; - TDB * pNtbIdx; - TDB * pCtbIdx; - int ret; - - pDb = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDb)); - if (pDb == NULL) { - return -1; - } - - // Create and open the ENV - A((tdbEnvCreate(&pEnv)), _err); -#if 0 - // Set options of the environment - A(tdbEnvSetPageSize(pEnv, 8192), _err); - A(tdbEnvSetCacheSize(pEnv, 16 * 1024 * 1024), _err); -#endif - A((tdbEnvOpen(&pEnv)), _err); - - // Create and open each DB - A(tdbCreate(&pTbDB), _err); - A(tdbOpen(&pTbDB, "table.db", NULL, pEnv), _err); - - A(tdbCreate(&pSchemaDB), _err); - A(tdbOpen(&pSchemaDB, "schema.db", NULL, pEnv), _err); - - A(tdbCreate(&pNameIdx), _err); - A(tdbOpen(&pNameIdx, "name.db", NULL, pEnv), _err); - // tdbAssociate(); - - pDb->pEnv = pEnv; - pDb->pTbDB = pTbDB; - pDb->pSchemaDB = pSchemaDB; - pMeta->pDB = pDb; + // TODO + ASSERT(0); return 0; - -_err: - return -1; } void metaCloseDB(SMeta *pMeta) { // TODO + ASSERT(0); } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { @@ -93,53 +45,117 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { // TODO + ASSERT(0); return 0; } STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { // TODO + ASSERT(0); return NULL; } STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { // TODO + ASSERT(0); return NULL; } SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { // TODO + ASSERT(0); return NULL; } STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { // TODO + ASSERT(0); return NULL; } SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { // TODO + ASSERT(0); return NULL; } void metaCloseTbCursor(SMTbCursor *pTbCur) { // TODO + ASSERT(0); } char *metaTbCursorNext(SMTbCursor *pTbCur) { // TODO + ASSERT(0); return NULL; } SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { // TODO + ASSERT(0); return NULL; } void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { // TODO + ASSERT(0); } tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { // TODO + ASSERT(0); return 0; -} \ No newline at end of file +} + +int metaGetTbNum(SMeta *pMeta) { + // TODO + ASSERT(0); + return 0; +} + +STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { + // TODO + ASSERT(0); + return NULL; +} + +int metaRemoveSmaFromDb(SMeta *pMeta, int64_t indexUid) { + // TODO + ASSERT(0); + return 0; +} + +int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { + // TODO + ASSERT(0); + return 0; +} + +STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { + // TODO + ASSERT(0); + return NULL; +} + +const char *metaSmaCursorNext(SMSmaCursor *pCur) { + // TODO + ASSERT(0); + return NULL; +} + +void metaCloseSmaCurosr(SMSmaCursor *pCur) { + // TODO + ASSERT(0); +} + +SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { + // TODO + ASSERT(0); + return NULL; +} + +SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { + // TODO + ASSERT(0); + return NULL; +} From 2003240959127460142c2cab0103d4e4afef82e9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 07:12:03 +0000 Subject: [PATCH 02/22] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 66 ++++++++++++++++++++++- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 344fcbe97c..5f35ac6efb 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -28,8 +28,70 @@ struct SMetaDB { }; int metaOpenDB(SMeta *pMeta) { - // TODO - ASSERT(0); + SMetaDB *pMetaDb; + int ret; + + // allocate DB handle + pMetaDb = taosMemoryCalloc(1, sizeof(*pMetaDb)); + if (pMetaDb == NULL) { + // TODO + ASSERT(0); + return -1; + } + + // open the ENV + ret = tdbEnvOpen(pMeta->path, 4096, 256, &(pMetaDb->pEnv)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + // open table DB + ret = tdbDbOpen("table.db", sizeof(tb_uid_t), TDB_VARIANT_LEN, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pTbDB)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + // open schema DB + ret = tdbDbOpen("schema.db", sizeof(tb_uid_t) + sizeof(int32_t), TDB_VARIANT_LEN, NULL /*TODO*/, pMetaDb->pEnv, + &(pMetaDb->pSchemaDB)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + ret = tdbDbOpen("name.idx", TDB_VARIANT_LEN, 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pNameIdx)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + ret = tdbDbOpen("stb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pStbIdx)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + ret = tdbDbOpen("ntb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pNtbIdx)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + ret = tdbDbOpen("ctb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pCtbIdx)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + return 0; } From 27a567543b903bae3369dfe263fd1d7e2086e9f5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 08:44:04 +0000 Subject: [PATCH 03/22] fix mutex problem --- source/libs/tdb/src/db/tdbPCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 981dd63593..07c267a15c 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -122,7 +122,7 @@ static void tdbPCacheClearLock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mute static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); } -static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mutex)); } +static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); } static bool tdbPCacheLocked(SPCache *pCache) { assert(0); From 387936e4378f0e735705d4a589c0b6220a3b299e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 08:58:09 +0000 Subject: [PATCH 04/22] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 14 +++----------- source/libs/tdb/src/inc/tdbPage.h | 1 + source/libs/tdb/src/page/tdbPage.c | 12 ++++++++++++ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index faced8e839..b568de848e 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -30,7 +30,6 @@ struct SBTree { int valLen; SPager *pPager; FKeyComparator kcmpr; - u8 fanout; int pageSize; int maxLocal; int minLocal; @@ -100,21 +99,14 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S pBt->pPager = pPager; // pBt->kcmpr pBt->kcmpr = kcmpr ? kcmpr : tdbDefaultKeyCmprFn; - // pBt->fanout - if (keyLen == TDB_VARIANT_LEN) { - pBt->fanout = TDB_DEFAULT_FANOUT; - } else { - ASSERT(0); - // TODO: pBt->fanout = 0; - } // pBt->pageSize pBt->pageSize = tdbPagerGetPageSize(pPager); // pBt->maxLocal - pBt->maxLocal = (pBt->pageSize - 14) / pBt->fanout; + pBt->maxLocal = tdbPageCapacity(pBt->pageSize, sizeof(SIntHdr)) / 4; // pBt->minLocal: Should not be allowed smaller than 15, which is [nPayload][nKey][nData] - pBt->minLocal = (pBt->pageSize - 14) / pBt->fanout / 2; + pBt->minLocal = pBt->maxLocal / 2; // pBt->maxLeaf - pBt->maxLeaf = pBt->pageSize - 14; + pBt->maxLeaf = tdbPageCapacity(pBt->pageSize, sizeof(SLeafHdr)); // pBt->minLeaf pBt->minLeaf = pBt->minLocal; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 49aa9f4398..563fb53e98 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -111,6 +111,7 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); int tdbPageDropCell(SPage *pPage, int idx); void tdbPageCopy(SPage *pFromPage, SPage *pToPage); +int tdbPageCapacity(int pageSize, int amHdrSize); static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { SCell *pCell; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 3301202a33..accd174e55 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -242,6 +242,18 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { pToPage->nOverflow = pFromPage->nOverflow; } +int tdbPageCapacity(int pageSize, int amHdrSize) { + int szPageHdr; + + if (pageSize < 65536) { + szPageHdr = pageMethods.szPageHdr; + } else { + szPageHdr = pageLargeMethods.szPageHdr; + } + + return pageSize - szPageHdr - amHdrSize; +} + static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { SCell *pFreeCell; u8 *pOffset; From 71e04b29ec5e06ba5ea434080f4f712523308a3d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 09:14:50 +0000 Subject: [PATCH 05/22] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index b568de848e..4ea4d6a8f1 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -84,6 +84,8 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S SBTree *pBt; int ret; + ASSERT(keyLen != 0); + *ppBt = NULL; pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt)); @@ -921,7 +923,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo } // 2. Encode payload part - if (leaf) { + if (leaf && vLen > 0) { ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, pVal, vLen, &nPayload); } else { ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, NULL, 0, &nPayload); @@ -960,6 +962,7 @@ static int tdbBtreeDecodePayload(SPage *pPage, const u8 *pPayload, SCellDecoder return 0; } +// TODO: here has problem static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder) { u8 flags; u8 leaf; From 0b336e693961edf7118a5c3ca1a62bb743e27025 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 09:19:19 +0000 Subject: [PATCH 06/22] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 5f35ac6efb..61f62320c4 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -171,7 +171,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { int metaGetTbNum(SMeta *pMeta) { // TODO - ASSERT(0); + // ASSERT(0); return 0; } From 0d3a30049c685b68036370cf92c3db5cd1d99af3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 09:50:00 +0000 Subject: [PATCH 07/22] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 64 +++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 61f62320c4..9940ae166d 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -96,12 +96,70 @@ int metaOpenDB(SMeta *pMeta) { } void metaCloseDB(SMeta *pMeta) { - // TODO - ASSERT(0); + if (pMeta->pDB) { + tdbDbClose(pMeta->pDB->pCtbIdx); + tdbDbClose(pMeta->pDB->pNtbIdx); + tdbDbClose(pMeta->pDB->pStbIdx); + tdbDbClose(pMeta->pDB->pNameIdx); + tdbDbClose(pMeta->pDB->pSchemaDB); + tdbDbClose(pMeta->pDB->pTbDB); + taosMemoryFree(pMeta->pDB); + } } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { - // TODO + tb_uid_t uid; + SMetaDB *pMetaDb; + void *pKey; + void *pVal; + int kLen; + int vLen; + int ret; + + pMetaDb = pMeta->pDB; + + // TODO: make this operation pre-process + if (pTbCfg->type == META_SUPER_TABLE) { + uid = pTbCfg->stbCfg.suid; + } else { + uid = metaGenerateUid(pMeta); + } + + // save to table.db + ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen); + if (ret < 0) { + return -1; + } + + // save to schema.db + ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen); + if (ret < 0) { + return -1; + } + + // update name.idx + ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, NULL, 0); + if (ret < 0) { + return -1; + } + + if (pTbCfg->type == META_SUPER_TABLE) { + ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, NULL, 0); + if (ret < 0) { + return -1; + } + } else if (pTbCfg->type == META_CHILD_TABLE) { + ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, NULL, 0); + if (ret < 0) { + return -1; + } + } else if (pTbCfg->type == META_NORMAL_TABLE) { + ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, NULL, 0); + if (ret < 0) { + return -1; + } + } + return 0; } From 2838ecbfe67bcdaba4036eeb68ea8aee939a8b4d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 10:09:09 +0000 Subject: [PATCH 08/22] refact TDB --- source/libs/tdb/CMakeLists.txt | 3 +- source/libs/tdb/src/{page => db}/tdbPage.c | 77 ++++++++++++++++++ source/libs/tdb/src/page/tdbPageL.c | 93 ---------------------- 3 files changed, 78 insertions(+), 95 deletions(-) rename source/libs/tdb/src/{page => db}/tdbPage.c (85%) delete mode 100644 source/libs/tdb/src/page/tdbPageL.c diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 0b1378dc5a..3a62b19c10 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -9,9 +9,8 @@ target_sources(tdb "src/db/tdbDb.c" "src/db/tdbEnv.c" "src/db/tdbTxn.c" + "src/db/tdbPage.c" "src/db/tdbOs.c" - "src/page/tdbPage.c" - "src/page/tdbPageL.c" ) target_include_directories( diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c similarity index 85% rename from source/libs/tdb/src/page/tdbPage.c rename to source/libs/tdb/src/db/tdbPage.c index accd174e55..cc4ce20802 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -515,4 +515,81 @@ SPageMethods pageMethods = { setPageCellOffset, // setCellOffset getPageFreeCellInfo, // getFreeCellInfo setPageFreeCellInfo // setFreeCellInfo +}; + +typedef struct __attribute__((__packed__)) { + u8 cellNum[3]; + u8 cellBody[3]; + u8 cellFree[3]; + u8 nFree[3]; +} SPageHdrL; + +typedef struct __attribute__((__packed__)) { + u8 szCell[3]; + u8 nxOffset[3]; +} SFreeCellL; + +// cellNum +static inline int getLPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } +static inline void setLPageCellNum(SPage *pPage, int cellNum) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum, cellNum); +} + +// cellBody +static inline int getLPageCellBody(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody); } +static inline void setLPageCellBody(SPage *pPage, int cellBody) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody, cellBody); +} + +// cellFree +static inline int getLPageCellFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree); } +static inline void setLPageCellFree(SPage *pPage, int cellFree) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree, cellFree); +} + +// nFree +static inline int getLPageNFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree); } +static inline void setLPageNFree(SPage *pPage, int nFree) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree, nFree); +} + +// cell offset +static inline int getLPageCellOffset(SPage *pPage, int idx) { + ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); + return TDB_GET_U24(pPage->pCellIdx + 3 * idx); +} + +static inline void setLPageCellOffset(SPage *pPage, int idx, int offset) { + TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset); +} + +// free cell info +static inline void getLPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + *szCell = TDB_GET_U24(pFreeCell->szCell); + *nxOffset = TDB_GET_U24(pFreeCell->nxOffset); +} + +static inline void setLPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + TDB_PUT_U24(pFreeCell->szCell, szCell); + TDB_PUT_U24(pFreeCell->nxOffset, nxOffset); +} + +SPageMethods pageLargeMethods = { + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell + getLPageCellNum, // getCellNum + setLPageCellNum, // setCellNum + getLPageCellBody, // getCellBody + setLPageCellBody, // setCellBody + getLPageCellFree, // getCellFree + setLPageCellFree, // setCellFree + getLPageNFree, // getFreeBytes + setLPageNFree, // setFreeBytes + getLPageCellOffset, // getCellOffset + setLPageCellOffset, // setCellOffset + getLPageFreeCellInfo, // getFreeCellInfo + setLPageFreeCellInfo // setFreeCellInfo }; \ No newline at end of file diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c deleted file mode 100644 index c5d4a6047f..0000000000 --- a/source/libs/tdb/src/page/tdbPageL.c +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -typedef struct __attribute__((__packed__)) { - u8 cellNum[3]; - u8 cellBody[3]; - u8 cellFree[3]; - u8 nFree[3]; -} SPageHdrL; - -typedef struct __attribute__((__packed__)) { - u8 szCell[3]; - u8 nxOffset[3]; -} SFreeCellL; - -// cellNum -static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } -static inline void setPageCellNum(SPage *pPage, int cellNum) { - TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum, cellNum); -} - -// cellBody -static inline int getPageCellBody(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody); } -static inline void setPageCellBody(SPage *pPage, int cellBody) { - TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody, cellBody); -} - -// cellFree -static inline int getPageCellFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree); } -static inline void setPageCellFree(SPage *pPage, int cellFree) { - TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree, cellFree); -} - -// nFree -static inline int getPageNFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree); } -static inline void setPageNFree(SPage *pPage, int nFree) { - TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree, nFree); -} - -// cell offset -static inline int getPageCellOffset(SPage *pPage, int idx) { - ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); - return TDB_GET_U24(pPage->pCellIdx + 3 * idx); -} - -static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { - TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset); -} - -// free cell info -static inline void getPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { - SFreeCellL *pFreeCell = (SFreeCellL *)pCell; - *szCell = TDB_GET_U24(pFreeCell->szCell); - *nxOffset = TDB_GET_U24(pFreeCell->nxOffset); -} - -static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { - SFreeCellL *pFreeCell = (SFreeCellL *)pCell; - TDB_PUT_U24(pFreeCell->szCell, szCell); - TDB_PUT_U24(pFreeCell->nxOffset, nxOffset); -} - -SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell - getPageCellNum, // getCellNum - setPageCellNum, // setCellNum - getPageCellBody, // getCellBody - setPageCellBody, // setCellBody - getPageCellFree, // getCellFree - setPageCellFree, // setCellFree - getPageNFree, // getFreeBytes - setPageNFree, // setFreeBytes - getPageCellOffset, // getCellOffset - setPageCellOffset, // setCellOffset - getPageFreeCellInfo, // getFreeCellInfo - setPageFreeCellInfo // setFreeCellInfo -}; \ No newline at end of file From bee3e3e20418ca44d4af8fab7a428285020dc0d2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 10:21:16 +0000 Subject: [PATCH 09/22] more TDB --- source/libs/tdb/src/db/tdbDb.c | 6 ++++++ source/libs/tdb/src/inc/tdbDb.h | 1 + 2 files changed, 7 insertions(+) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 68adb7ccfc..e2213ed858 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -129,5 +129,11 @@ int tdbDbcClose(TDBC *pDbc) { tdbOsFree(pDbc); } + return 0; +} + +int tdbDbcInsert(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen) { + // TODO + ASSERT(0); return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 4fbf65829d..f382940a83 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -34,6 +34,7 @@ int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbDbcOpen(TDB *pDb, TDBC **ppDbc); int tdbDbNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbDbcClose(TDBC *pDbc); +int tdbDbcInsert(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen); #ifdef __cplusplus } From dfaa78daf69c0f5e5f5b9ebb005f19d055da6dc8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 10:43:57 +0000 Subject: [PATCH 10/22] refact --- source/libs/tdb/src/inc/tdbTxn.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/inc/tdbTxn.h b/source/libs/tdb/src/inc/tdbTxn.h index 4300dc8324..cc11369785 100644 --- a/source/libs/tdb/src/inc/tdbTxn.h +++ b/source/libs/tdb/src/inc/tdbTxn.h @@ -20,7 +20,7 @@ extern "C" { #endif -typedef struct STxn STXN; +typedef struct STxn TXN; struct STxn { u64 txnId; From e3e78784e3b8bc2ed0d0fb732683494d3f792394 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 03:47:48 +0000 Subject: [PATCH 11/22] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 78 +++++++++++++++++++++-- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 9940ae166d..b8daf4e4d2 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -27,6 +27,72 @@ struct SMetaDB { TDB *pCtbIdx; }; +typedef struct __attribute__((__packed__)) { + tb_uid_t uid; + int32_t sver; +} SSchemaDbKey; + +typedef struct { + char *name; + tb_uid_t uid; +} SNameIdxKey; + +typedef struct { + tb_uid_t suid; + tb_uid_t uid; +} SCtbIdxKey; + +static inline int metaUidCmpr(const void *arg1, int len1, const void *arg2, int len2) { + tb_uid_t uid1, uid2; + + ASSERT(len1 == sizeof(tb_uid_t)); + ASSERT(len2 == sizeof(tb_uid_t)); + + uid1 = ((tb_uid_t *)arg1)[0]; + uid2 = ((tb_uid_t *)arg2)[1]; + + if (uid1 < uid2) { + return -1; + } + if (uid1 == uid2) { + return 0; + } else { + return 1; + } +} + +static inline int metaSchemaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { + int c; + SSchemaDbKey *pKey1 = (SSchemaDbKey *)arg1; + SSchemaDbKey *pKey2 = (SSchemaDbKey *)arg2; + + c = metaUidCmpr(arg1, sizeof(tb_uid_t), arg2, sizeof(tb_uid_t)); + if (c) return c; + + if (pKey1->sver > pKey2->sver) { + return 1; + } else if (pKey1->sver == pKey2->sver) { + return 0; + } else { + return -1; + } +} + +static inline int metaNameIdxCmpr(const void *arg1, int len1, const void *arg2, int len2) { + return strcmp((char *)arg1, (char *)arg2); +} + +static inline int metaCtbIdxCmpr(const void *arg1, int len1, const void *arg2, int len2) { + int c; + SCtbIdxKey *pKey1 = (SCtbIdxKey *)arg1; + SCtbIdxKey *pKey2 = (SCtbIdxKey *)arg2; + + c = metaUidCmpr(arg1, sizeof(tb_uid_t), arg2, sizeof(tb_uid_t)); + if (c) return c; + + return metaUidCmpr(&pKey1->uid, sizeof(tb_uid_t), &pKey2->uid, sizeof(tb_uid_t)); +} + int metaOpenDB(SMeta *pMeta) { SMetaDB *pMetaDb; int ret; @@ -48,7 +114,7 @@ int metaOpenDB(SMeta *pMeta) { } // open table DB - ret = tdbDbOpen("table.db", sizeof(tb_uid_t), TDB_VARIANT_LEN, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pTbDB)); + ret = tdbDbOpen("table.db", sizeof(tb_uid_t), TDB_VARIANT_LEN, metaUidCmpr, pMetaDb->pEnv, &(pMetaDb->pTbDB)); if (ret < 0) { // TODO ASSERT(0); @@ -56,7 +122,7 @@ int metaOpenDB(SMeta *pMeta) { } // open schema DB - ret = tdbDbOpen("schema.db", sizeof(tb_uid_t) + sizeof(int32_t), TDB_VARIANT_LEN, NULL /*TODO*/, pMetaDb->pEnv, + ret = tdbDbOpen("schema.db", sizeof(tb_uid_t) + sizeof(int32_t), TDB_VARIANT_LEN, metaSchemaKeyCmpr, pMetaDb->pEnv, &(pMetaDb->pSchemaDB)); if (ret < 0) { // TODO @@ -64,28 +130,28 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - ret = tdbDbOpen("name.idx", TDB_VARIANT_LEN, 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pNameIdx)); + ret = tdbDbOpen("name.idx", TDB_VARIANT_LEN, 0, metaNameIdxCmpr, pMetaDb->pEnv, &(pMetaDb->pNameIdx)); if (ret < 0) { // TODO ASSERT(0); return -1; } - ret = tdbDbOpen("stb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pStbIdx)); + ret = tdbDbOpen("stb.idx", sizeof(tb_uid_t), 0, metaUidCmpr, pMetaDb->pEnv, &(pMetaDb->pStbIdx)); if (ret < 0) { // TODO ASSERT(0); return -1; } - ret = tdbDbOpen("ntb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pNtbIdx)); + ret = tdbDbOpen("ntb.idx", sizeof(tb_uid_t), 0, metaUidCmpr, pMetaDb->pEnv, &(pMetaDb->pNtbIdx)); if (ret < 0) { // TODO ASSERT(0); return -1; } - ret = tdbDbOpen("ctb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pCtbIdx)); + ret = tdbDbOpen("ctb.idx", sizeof(tb_uid_t), 0, metaCtbIdxCmpr, pMetaDb->pEnv, &(pMetaDb->pCtbIdx)); if (ret < 0) { // TODO ASSERT(0); From 9b3d8b0e85496490110f5ff61e38b1e3f7c897b8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 05:34:30 +0000 Subject: [PATCH 12/22] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 91 ++++++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index b8daf4e4d2..34fd607ce1 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -42,6 +42,9 @@ typedef struct { tb_uid_t uid; } SCtbIdxKey; +static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); +static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); + static inline int metaUidCmpr(const void *arg1, int len1, const void *arg2, int len2) { tb_uid_t uid1, uid2; @@ -122,7 +125,7 @@ int metaOpenDB(SMeta *pMeta) { } // open schema DB - ret = tdbDbOpen("schema.db", sizeof(tb_uid_t) + sizeof(int32_t), TDB_VARIANT_LEN, metaSchemaKeyCmpr, pMetaDb->pEnv, + ret = tdbDbOpen("schema.db", sizeof(SSchemaDbKey), TDB_VARIANT_LEN, metaSchemaKeyCmpr, pMetaDb->pEnv, &(pMetaDb->pSchemaDB)); if (ret < 0) { // TODO @@ -151,7 +154,7 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - ret = tdbDbOpen("ctb.idx", sizeof(tb_uid_t), 0, metaCtbIdxCmpr, pMetaDb->pEnv, &(pMetaDb->pCtbIdx)); + ret = tdbDbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, metaCtbIdxCmpr, pMetaDb->pEnv, &(pMetaDb->pCtbIdx)); if (ret < 0) { // TODO ASSERT(0); @@ -181,6 +184,8 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { int kLen; int vLen; int ret; + char buf[512]; + void *pBuf; pMetaDb = pMeta->pDB; @@ -192,6 +197,11 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { } // save to table.db + pKey = &uid; + kLen = sizeof(uid); + pVal = pBuf = buf; + metaEncodeTbInfo(&pBuf, pTbCfg); + vLen = POINTER_DISTANCE(pBuf, buf); ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen); if (ret < 0) { return -1; @@ -345,3 +355,80 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { ASSERT(0); return NULL; } + +static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { + int tlen = 0; + SSchema *pSchema; + + tlen += taosEncodeFixedU32(buf, pSW->nCols); + for (int i = 0; i < pSW->nCols; i++) { + pSchema = pSW->pSchema + i; + tlen += taosEncodeFixedI8(buf, pSchema->type); + tlen += taosEncodeFixedI16(buf, pSchema->colId); + tlen += taosEncodeFixedI32(buf, pSchema->bytes); + tlen += taosEncodeString(buf, pSchema->name); + } + + return tlen; +} + +static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { + SSchema *pSchema; + + buf = taosDecodeFixedU32(buf, &pSW->nCols); + pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); + for (int i = 0; i < pSW->nCols; i++) { + pSchema = pSW->pSchema + i; + buf = taosDecodeFixedI8(buf, &pSchema->type); + buf = taosDecodeFixedI16(buf, &pSchema->colId); + buf = taosDecodeFixedI32(buf, &pSchema->bytes); + buf = taosDecodeStringTo(buf, pSchema->name); + } + + return buf; +} + +static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { + int tsize = 0; + + tsize += taosEncodeString(buf, pTbCfg->name); + tsize += taosEncodeFixedU32(buf, pTbCfg->ttl); + tsize += taosEncodeFixedU32(buf, pTbCfg->keep); + tsize += taosEncodeFixedU8(buf, pTbCfg->info); + + if (pTbCfg->type == META_SUPER_TABLE) { + SSchemaWrapper sw = {.nCols = pTbCfg->stbCfg.nTagCols, .pSchema = pTbCfg->stbCfg.pTagSchema}; + tsize += metaEncodeSchema(buf, &sw); + } else if (pTbCfg->type == META_CHILD_TABLE) { + tsize += taosEncodeFixedU64(buf, pTbCfg->ctbCfg.suid); + tsize += tdEncodeKVRow(buf, pTbCfg->ctbCfg.pTag); + } else if (pTbCfg->type == META_NORMAL_TABLE) { + // TODO + } else { + ASSERT(0); + } + + return tsize; +} + +static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { + buf = taosDecodeString(buf, &(pTbCfg->name)); + buf = taosDecodeFixedU32(buf, &(pTbCfg->ttl)); + buf = taosDecodeFixedU32(buf, &(pTbCfg->keep)); + buf = taosDecodeFixedU8(buf, &(pTbCfg->info)); + + if (pTbCfg->type == META_SUPER_TABLE) { + SSchemaWrapper sw; + buf = metaDecodeSchema(buf, &sw); + pTbCfg->stbCfg.nTagCols = sw.nCols; + pTbCfg->stbCfg.pTagSchema = sw.pSchema; + } else if (pTbCfg->type == META_CHILD_TABLE) { + buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid)); + buf = tdDecodeKVRow(buf, &(pTbCfg->ctbCfg.pTag)); + } else if (pTbCfg->type == META_NORMAL_TABLE) { + // TODO + } else { + ASSERT(0); + } + return buf; +} From 015e4fc3946289f9b1772964f7656120f88a46d4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 05:57:39 +0000 Subject: [PATCH 13/22] more TDB --- source/dnode/vnode/src/meta/metaTDBImpl.c | 78 ++++++++++++++++++----- 1 file changed, 61 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 34fd607ce1..354a9f7eaf 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -44,6 +44,8 @@ typedef struct { static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); +static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); +static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW); static inline int metaUidCmpr(const void *arg1, int len1, const void *arg2, int len2) { tb_uid_t uid1, uid2; @@ -177,15 +179,18 @@ void metaCloseDB(SMeta *pMeta) { } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { - tb_uid_t uid; - SMetaDB *pMetaDb; - void *pKey; - void *pVal; - int kLen; - int vLen; - int ret; - char buf[512]; - void *pBuf; + tb_uid_t uid; + SMetaDB *pMetaDb; + void *pKey; + void *pVal; + int kLen; + int vLen; + int ret; + char buf[512]; + void *pBuf; + SCtbIdxKey ctbIdxKey; + SSchemaDbKey schemaDbKey; + SSchemaWrapper schemaWrapper; pMetaDb = pMeta->pDB; @@ -207,30 +212,69 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { return -1; } - // save to schema.db - ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen); - if (ret < 0) { - return -1; + // save to schema.db for META_SUPER_TABLE and META_NORMAL_TABLE + if (pTbCfg->type != META_CHILD_TABLE) { + schemaDbKey.uid = uid; + schemaDbKey.sver = 0; // TODO + pKey = &schemaDbKey; + kLen = sizeof(schemaDbKey); + + if (pTbCfg->type == META_SUPER_TABLE) { + schemaWrapper.nCols = pTbCfg->stbCfg.nCols; + schemaWrapper.pSchema = pTbCfg->stbCfg.pSchema; + } else { + schemaWrapper.nCols = pTbCfg->ntbCfg.nCols; + schemaWrapper.pSchema = pTbCfg->ntbCfg.pSchema; + } + pVal = pBuf = buf; + metaEncodeSchema(&pBuf, &schemaWrapper); + vLen = POINTER_DISTANCE(pBuf, buf); + ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen); + if (ret < 0) { + return -1; + } } // update name.idx - ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, NULL, 0); + int nameLen; + memcpy(buf, pTbCfg->name, nameLen + 1); + ((tb_uid_t *)(buf + nameLen + 1))[0] = uid; + pKey = buf; + kLen = nameLen + 1 + sizeof(uid); + pVal = NULL; + vLen = 0; + ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen); if (ret < 0) { return -1; } + // update other index if (pTbCfg->type == META_SUPER_TABLE) { - ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, NULL, 0); + pKey = &uid; + kLen = sizeof(uid); + pVal = NULL; + vLen = 0; + ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen); if (ret < 0) { return -1; } } else if (pTbCfg->type == META_CHILD_TABLE) { - ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, NULL, 0); + ctbIdxKey.suid = pTbCfg->ctbCfg.suid; + ctbIdxKey.uid = uid; + pKey = &ctbIdxKey; + kLen = sizeof(ctbIdxKey); + pVal = NULL; + vLen = 0; + ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen); if (ret < 0) { return -1; } } else if (pTbCfg->type == META_NORMAL_TABLE) { - ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, NULL, 0); + pKey = &uid; + kLen = sizeof(uid); + pVal = NULL; + vLen = 0; + ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen); if (ret < 0) { return -1; } From 8a0d117acfa56efd7bec8f79a4d0c5fb3bc2d9a2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 06:38:22 +0000 Subject: [PATCH 14/22] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 46 ++++++++++++++++++++--- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 354a9f7eaf..a1280587b0 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -290,15 +290,49 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { } STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { - // TODO - ASSERT(0); - return NULL; + int ret; + SMetaDB *pMetaDb = pMeta->pDB; + void *pKey; + void *pVal; + int kLen; + int vLen; + STbCfg *pTbCfg; + + // Fetch + pKey = &uid; + kLen = sizeof(uid); + ret = tdbDbGet(pMetaDb->pTbDB, pKey, kLen, &pVal, &vLen); + if (ret < 0) { + return NULL; + } + + // Decode + pTbCfg = taosMemoryMalloc(sizeof(*pTbCfg)); + metaDecodeTbInfo(pVal, pTbCfg); + + TDB_FREE(pVal); + + return pTbCfg; } STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { - // TODO - ASSERT(0); - return NULL; + void *pKey; + void *pVal; + int kLen; + int vLen; + int ret; + + pKey = tbname; + kLen = strlen(tbname) + 1; + ret = tdbDbGet(pMeta->pDB->pNameIdx, pKey, kLen, &pVal, &vLen); + if (ret < 0) { + return NULL; + } + + *uid = *(tb_uid_t *)POINTER_SHIFT(pVal, kLen); + TDB_FREE(pVal); + + return metaGetTbInfoByUid(pMeta, *uid); } SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { From 5d1d22552a44137b5a6793f27da913bf9e3a12ae Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 07:04:17 +0000 Subject: [PATCH 15/22] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 122 +++++++++++++++++++--- 1 file changed, 109 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index a1280587b0..eff235b67c 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -301,6 +301,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { // Fetch pKey = &uid; kLen = sizeof(uid); + pVal = NULL; ret = tdbDbGet(pMetaDb->pTbDB, pKey, kLen, &pVal, &vLen); if (ret < 0) { return NULL; @@ -324,6 +325,7 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { pKey = tbname; kLen = strlen(tbname) + 1; + pVal = NULL; ret = tdbDbGet(pMeta->pDB->pNameIdx, pKey, kLen, &pVal, &vLen); if (ret < 0) { return NULL; @@ -336,34 +338,128 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { } SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { - // TODO - ASSERT(0); - return NULL; + void *pKey; + void *pVal; + int kLen; + int vLen; + int ret; + SSchemaDbKey schemaDbKey; + SSchemaWrapper *pSchemaWrapper; + void *pBuf; + + // fetch + schemaDbKey.uid = uid; + schemaDbKey.sver = sver; + pKey = &schemaDbKey; + kLen = sizeof(schemaDbKey); + pVal = NULL; + ret = tdbDbGet(pMeta->pDB->pSchemaDB, pKey, kLen, &pVal, &vLen); + if (ret < 0) { + return NULL; + } + + // decode + pBuf = pVal; + pSchemaWrapper = taosMemoryMalloc(sizeof(*pSchemaWrapper)); + metaDecodeSchema(pBuf, pSchemaWrapper); + + TDB_FREE(pVal); + + return pSchemaWrapper; } STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { - // TODO - ASSERT(0); - return NULL; + tb_uid_t quid; + SSchemaWrapper *pSW; + STSchemaBuilder sb; + SSchema *pSchema; + STSchema *pTSchema; + STbCfg *pTbCfg; + + pTbCfg = metaGetTbInfoByUid(pMeta, uid); + if (pTbCfg->type == META_CHILD_TABLE) { + quid = pTbCfg->ctbCfg.suid; + } else { + quid = uid; + } + + pSW = metaGetTableSchema(pMeta, quid, sver, true); + if (pSW == NULL) { + return NULL; + } + + tdInitTSchemaBuilder(&sb, 0); + for (int i = 0; i < pSW->nCols; i++) { + pSchema = pSW->pSchema + i; + tdAddColToSchema(&sb, pSchema->type, pSchema->colId, pSchema->bytes); + } + pTSchema = tdGetSchemaFromBuilder(&sb); + tdDestroyTSchemaBuilder(&sb); + + return pTSchema; } +struct SMTbCursor { + TDBC *pDbc; +}; + SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { - // TODO - ASSERT(0); - return NULL; + SMTbCursor *pTbCur = NULL; + SMetaDB *pDB = pMeta->pDB; + + pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); + if (pTbCur == NULL) { + return NULL; + } + + tdbDbcOpen(pDB->pTbDB, &pTbCur->pDbc); + + return pTbCur; } void metaCloseTbCursor(SMTbCursor *pTbCur) { - // TODO - ASSERT(0); + if (pTbCur) { + if (pTbCur->pDbc) { + tdbDbcClose(pTbCur->pDbc); + } + taosMemoryFree(pTbCur); + } } char *metaTbCursorNext(SMTbCursor *pTbCur) { - // TODO - ASSERT(0); + void *pKey = NULL; + void *pVal = NULL; + int kLen; + int vLen; + int ret; + void *pBuf; + STbCfg tbCfg; + + for (;;) { + ret = tdbDbNext(pTbCur->pDbc, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + pBuf = pVal; + metaDecodeTbInfo(pBuf, &tbCfg); + if (tbCfg.type == META_SUPER_TABLE) { + taosMemoryFree(tbCfg.name); + taosMemoryFree(tbCfg.stbCfg.pTagSchema); + continue; + ; + } else if (tbCfg.type == META_CHILD_TABLE) { + kvRowFree(tbCfg.ctbCfg.pTag); + } + + return tbCfg.name; + } + return NULL; } +struct SMCtbCursor { + TDBC *pCur; + tb_uid_t suid; +}; + SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { // TODO ASSERT(0); From e9585e74f7fa8cc25267fba52da2bd8ad907253e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 07:15:07 +0000 Subject: [PATCH 16/22] more TDB --- source/dnode/vnode/src/meta/metaTDBImpl.c | 52 +++++++++++++++++++---- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index eff235b67c..acc0319009 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -458,23 +458,59 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { struct SMCtbCursor { TDBC *pCur; tb_uid_t suid; + void *pKey; + void *pVal; + int kLen; + int vLen; }; SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { - // TODO - ASSERT(0); - return NULL; + SMCtbCursor *pCtbCur = NULL; + SMetaDB *pDB = pMeta->pDB; + int ret; + + pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); + if (pCtbCur == NULL) { + return NULL; + } + + pCtbCur->suid = uid; + ret = tdbDbcOpen(pDB->pCtbIdx, &pCtbCur->pCur); + if (ret < 0) { + taosMemoryFree(pCtbCur); + return NULL; + } + + // TODO: move the cursor to the suid there + + return pCtbCur; } void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { - // TODO - ASSERT(0); + if (pCtbCur) { + if (pCtbCur->pCur) { + tdbDbcClose(pCtbCur->pCur); + + TDB_FREE(pCtbCur->pKey); + TDB_FREE(pCtbCur->pVal); + } + + taosMemoryFree(pCtbCur); + } } tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { - // TODO - ASSERT(0); - return 0; + int ret; + SCtbIdxKey *pCtbIdxKey; + + ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); + if (ret < 0) { + return 0; + } + + pCtbIdxKey = pCtbCur->pVal; + + return pCtbIdxKey->uid; } int metaGetTbNum(SMeta *pMeta) { From d22d069bd95e41a986746e2977bd174fffdacb4e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 07:57:06 +0000 Subject: [PATCH 17/22] add an interface for index db query --- source/libs/tdb/src/db/tdbBtree.c | 34 ++++++++++++++++++++++++++++++ source/libs/tdb/src/db/tdbDb.c | 4 ++++ source/libs/tdb/src/inc/tdbBtree.h | 1 + source/libs/tdb/src/inc/tdbDb.h | 1 + 4 files changed, 40 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4ea4d6a8f1..e0c27b6788 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -220,6 +220,40 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen return 0; } +int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { + SBTC btc; + SCell *pCell; + int cret; + void *pTKey; + void *pTVal; + SCellDecoder cd; + + tdbBtcOpen(&btc, pBt); + + tdbBtCursorMoveTo(&btc, pKey, kLen, &cret); + if (cret) { + return cret; + } + + pCell = tdbPageGetCell(btc.pPage, btc.idx); + tdbBtreeDecodeCell(btc.pPage, pCell, &cd); + + pTKey = TDB_REALLOC(*ppKey, cd.kLen); + pTVal = TDB_REALLOC(*ppVal, cd.vLen); + + if (pTKey == NULL || pTVal == NULL) { + TDB_FREE(pTKey); + TDB_FREE(pTVal); + } + + *ppKey = pTKey; + *ppVal = pTVal; + *pkLen = cd.kLen; + *vLen = cd.vLen; + + return 0; +} + static int tdbBtCursorMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { int ret; SBTree *pBt; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index e2213ed858..e41e41f72a 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -96,6 +96,10 @@ int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } +int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { + return tdbBtreePGet(pDb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen); +} + int tdbDbcOpen(TDB *pDb, TDBC **ppDbc) { int ret; TDBC *pDbc = NULL; diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index e46e8bb78b..0e8ca0c803 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -43,6 +43,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SB int tdbBtreeClose(SBTree *pBt); int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); // SBTC int tdbBtcOpen(SBTC *pCur, SBTree *pBt); diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index f382940a83..e60371c734 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -29,6 +29,7 @@ int tdbDbClose(TDB *pDb); int tdbDbDrop(TDB *pDb); int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); // TDBC int tdbDbcOpen(TDB *pDb, TDBC **ppDbc); From 4552f9ba2f5b1dfb4fd0a36a4cb94c89c3b2c821 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 07:57:38 +0000 Subject: [PATCH 18/22] fix a problem --- source/dnode/vnode/src/meta/metaTDBImpl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index acc0319009..634907b0f5 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -163,6 +163,7 @@ int metaOpenDB(SMeta *pMeta) { return -1; } + pMeta->pDB = pMetaDb; return 0; } @@ -236,7 +237,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { } // update name.idx - int nameLen; + int nameLen = strlen(pTbCfg->name); memcpy(buf, pTbCfg->name, nameLen + 1); ((tb_uid_t *)(buf + nameLen + 1))[0] = uid; pKey = buf; From 22d43247a99a3c920a09753409b15d780c039bbf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 08:10:02 +0000 Subject: [PATCH 19/22] fix another bug --- source/dnode/vnode/src/meta/metaTDBImpl.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 634907b0f5..867e597b9b 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -320,6 +320,8 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { void *pKey; void *pVal; + void *ppKey; + int pkLen; int kLen; int vLen; int ret; @@ -327,12 +329,16 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { pKey = tbname; kLen = strlen(tbname) + 1; pVal = NULL; - ret = tdbDbGet(pMeta->pDB->pNameIdx, pKey, kLen, &pVal, &vLen); + ppKey = NULL; + ret = tdbDbPGet(pMeta->pDB->pNameIdx, pKey, kLen, &ppKey, &pkLen, &pVal, &vLen); if (ret < 0) { return NULL; } - *uid = *(tb_uid_t *)POINTER_SHIFT(pVal, kLen); + ASSERT(pkLen == kLen + sizeof(uid)); + + *uid = *(tb_uid_t *)POINTER_SHIFT(ppKey, kLen); + TDB_FREE(ppKey); TDB_FREE(pVal); return metaGetTbInfoByUid(pMeta, *uid); From 7b932e1b25ddc3ab581523226180689bf04be2dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 08:15:55 +0000 Subject: [PATCH 20/22] fix pget problem --- source/libs/tdb/src/db/tdbBtree.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e0c27b6788..b133e1fcbc 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -251,6 +251,9 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL *pkLen = cd.kLen; *vLen = cd.vLen; + memcpy(*ppKey, cd.pKey, cd.kLen); + memcpy(*ppVal, cd.pVal, cd.vLen); + return 0; } From f0e6b6ede6d67fe63ef5d9363532c0f79a58fbbf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 08:38:42 +0000 Subject: [PATCH 21/22] fix a stupic bug --- source/dnode/vnode/src/meta/metaTDBImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 867e597b9b..ee928ef039 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -54,7 +54,7 @@ static inline int metaUidCmpr(const void *arg1, int len1, const void *arg2, int ASSERT(len2 == sizeof(tb_uid_t)); uid1 = ((tb_uid_t *)arg1)[0]; - uid2 = ((tb_uid_t *)arg2)[1]; + uid2 = ((tb_uid_t *)arg2)[0]; if (uid1 < uid2) { return -1; From abfb80acac4a729b63cb435edf48f7c6a6c4cdb4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Mar 2022 03:21:57 +0000 Subject: [PATCH 22/22] still use BDB --- source/dnode/vnode/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 6b21f935c4..56a8698d77 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,5 +1,5 @@ set(META_DB_IMPL_LIST "BDB" "TDB") -set(META_DB_IMPL "TDB" CACHE STRING "Use BDB as the default META implementation") +set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST)