From 8ff248a308bb4135e62537bf418bfaeaeb1493b6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 06:35:00 +0000 Subject: [PATCH 01/25] meta with tdb compile --- source/dnode/vnode/CMakeLists.txt | 11 +- source/dnode/vnode/src/meta/metaTDBImpl.c | 134 ++++++++++++---------- 2 files changed, 79 insertions(+), 66 deletions(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index bdc8a71b04..6b21f935c4 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,5 +1,5 @@ set(META_DB_IMPL_LIST "BDB" "TDB") -set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") +set(META_DB_IMPL "TDB" CACHE STRING "Use BDB as the default META implementation") set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) @@ -46,14 +46,11 @@ target_link_libraries( PUBLIC stream PUBLIC qworker PUBLIC sync + # TODO: get rid of BDB + PUBLIC bdb + PUBLIC tdb ) -if(${META_DB_IMPL} STREQUAL "BDB") - target_link_libraries(vnode PUBLIC bdb) -elseif(${META_DB_IMPL} STREQUAL "TDB") - target_link_libraries(vnode PUBLIC tdb) -endif() - if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index f4b450b4a8..344fcbe97c 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -15,75 +15,27 @@ #include "metaDef.h" -#include "tdb.h" +#include "tdbInt.h" struct SMetaDB { TENV *pEnv; - TDB * pTbDB; - TDB * pSchemaDB; - TDB * pNameIdx; - TDB * pStbIdx; - TDB * pNtbIdx; - TDB * pCtbIdx; - // tag index hash table - // suid+colid --> TDB * - struct { - } tagIdxHt; + TDB *pTbDB; + TDB *pSchemaDB; + TDB *pNameIdx; + TDB *pStbIdx; + TDB *pNtbIdx; + TDB *pCtbIdx; }; -#define A(op, flag) \ - do { \ - if ((ret = op) != 0) goto flag; \ - } while (0) - int metaOpenDB(SMeta *pMeta) { - SMetaDB *pDb; - TENV * pEnv; - TDB * pTbDB; - TDB * pSchemaDB; - TDB * pNameIdx; - TDB * pStbIdx; - TDB * pNtbIdx; - TDB * pCtbIdx; - int ret; - - pDb = (SMetaDB *)taosMemoryCalloc(1, sizeof(*pDb)); - if (pDb == NULL) { - return -1; - } - - // Create and open the ENV - A((tdbEnvCreate(&pEnv)), _err); -#if 0 - // Set options of the environment - A(tdbEnvSetPageSize(pEnv, 8192), _err); - A(tdbEnvSetCacheSize(pEnv, 16 * 1024 * 1024), _err); -#endif - A((tdbEnvOpen(&pEnv)), _err); - - // Create and open each DB - A(tdbCreate(&pTbDB), _err); - A(tdbOpen(&pTbDB, "table.db", NULL, pEnv), _err); - - A(tdbCreate(&pSchemaDB), _err); - A(tdbOpen(&pSchemaDB, "schema.db", NULL, pEnv), _err); - - A(tdbCreate(&pNameIdx), _err); - A(tdbOpen(&pNameIdx, "name.db", NULL, pEnv), _err); - // tdbAssociate(); - - pDb->pEnv = pEnv; - pDb->pTbDB = pTbDB; - pDb->pSchemaDB = pSchemaDB; - pMeta->pDB = pDb; + // TODO + ASSERT(0); return 0; - -_err: - return -1; } void metaCloseDB(SMeta *pMeta) { // TODO + ASSERT(0); } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { @@ -93,53 +45,117 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { // TODO + ASSERT(0); return 0; } STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { // TODO + ASSERT(0); return NULL; } STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { // TODO + ASSERT(0); return NULL; } SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { // TODO + ASSERT(0); return NULL; } STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { // TODO + ASSERT(0); return NULL; } SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { // TODO + ASSERT(0); return NULL; } void metaCloseTbCursor(SMTbCursor *pTbCur) { // TODO + ASSERT(0); } char *metaTbCursorNext(SMTbCursor *pTbCur) { // TODO + ASSERT(0); return NULL; } SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { // TODO + ASSERT(0); return NULL; } void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { // TODO + ASSERT(0); } tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { // TODO + ASSERT(0); return 0; -} \ No newline at end of file +} + +int metaGetTbNum(SMeta *pMeta) { + // TODO + ASSERT(0); + return 0; +} + +STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { + // TODO + ASSERT(0); + return NULL; +} + +int metaRemoveSmaFromDb(SMeta *pMeta, int64_t indexUid) { + // TODO + ASSERT(0); + return 0; +} + +int metaSaveSmaToDB(SMeta *pMeta, STSma *pSmaCfg) { + // TODO + ASSERT(0); + return 0; +} + +STSma *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid) { + // TODO + ASSERT(0); + return NULL; +} + +const char *metaSmaCursorNext(SMSmaCursor *pCur) { + // TODO + ASSERT(0); + return NULL; +} + +void metaCloseSmaCurosr(SMSmaCursor *pCur) { + // TODO + ASSERT(0); +} + +SArray *metaGetSmaTbUids(SMeta *pMeta, bool isDup) { + // TODO + ASSERT(0); + return NULL; +} + +SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { + // TODO + ASSERT(0); + return NULL; +} From 2003240959127460142c2cab0103d4e4afef82e9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 07:12:03 +0000 Subject: [PATCH 02/25] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 66 ++++++++++++++++++++++- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 344fcbe97c..5f35ac6efb 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -28,8 +28,70 @@ struct SMetaDB { }; int metaOpenDB(SMeta *pMeta) { - // TODO - ASSERT(0); + SMetaDB *pMetaDb; + int ret; + + // allocate DB handle + pMetaDb = taosMemoryCalloc(1, sizeof(*pMetaDb)); + if (pMetaDb == NULL) { + // TODO + ASSERT(0); + return -1; + } + + // open the ENV + ret = tdbEnvOpen(pMeta->path, 4096, 256, &(pMetaDb->pEnv)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + // open table DB + ret = tdbDbOpen("table.db", sizeof(tb_uid_t), TDB_VARIANT_LEN, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pTbDB)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + // open schema DB + ret = tdbDbOpen("schema.db", sizeof(tb_uid_t) + sizeof(int32_t), TDB_VARIANT_LEN, NULL /*TODO*/, pMetaDb->pEnv, + &(pMetaDb->pSchemaDB)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + ret = tdbDbOpen("name.idx", TDB_VARIANT_LEN, 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pNameIdx)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + ret = tdbDbOpen("stb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pStbIdx)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + ret = tdbDbOpen("ntb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pNtbIdx)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + + ret = tdbDbOpen("ctb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pCtbIdx)); + if (ret < 0) { + // TODO + ASSERT(0); + return -1; + } + return 0; } From 27a567543b903bae3369dfe263fd1d7e2086e9f5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 08:44:04 +0000 Subject: [PATCH 03/25] fix mutex problem --- source/libs/tdb/src/db/tdbPCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 981dd63593..07c267a15c 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -122,7 +122,7 @@ static void tdbPCacheClearLock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mute static void tdbPCacheLock(SPCache *pCache) { tdbMutexLock(&(pCache->mutex)); } -static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexDestroy(&(pCache->mutex)); } +static void tdbPCacheUnlock(SPCache *pCache) { tdbMutexUnlock(&(pCache->mutex)); } static bool tdbPCacheLocked(SPCache *pCache) { assert(0); From 387936e4378f0e735705d4a589c0b6220a3b299e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 08:58:09 +0000 Subject: [PATCH 04/25] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 14 +++----------- source/libs/tdb/src/inc/tdbPage.h | 1 + source/libs/tdb/src/page/tdbPage.c | 12 ++++++++++++ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index faced8e839..b568de848e 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -30,7 +30,6 @@ struct SBTree { int valLen; SPager *pPager; FKeyComparator kcmpr; - u8 fanout; int pageSize; int maxLocal; int minLocal; @@ -100,21 +99,14 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S pBt->pPager = pPager; // pBt->kcmpr pBt->kcmpr = kcmpr ? kcmpr : tdbDefaultKeyCmprFn; - // pBt->fanout - if (keyLen == TDB_VARIANT_LEN) { - pBt->fanout = TDB_DEFAULT_FANOUT; - } else { - ASSERT(0); - // TODO: pBt->fanout = 0; - } // pBt->pageSize pBt->pageSize = tdbPagerGetPageSize(pPager); // pBt->maxLocal - pBt->maxLocal = (pBt->pageSize - 14) / pBt->fanout; + pBt->maxLocal = tdbPageCapacity(pBt->pageSize, sizeof(SIntHdr)) / 4; // pBt->minLocal: Should not be allowed smaller than 15, which is [nPayload][nKey][nData] - pBt->minLocal = (pBt->pageSize - 14) / pBt->fanout / 2; + pBt->minLocal = pBt->maxLocal / 2; // pBt->maxLeaf - pBt->maxLeaf = pBt->pageSize - 14; + pBt->maxLeaf = tdbPageCapacity(pBt->pageSize, sizeof(SLeafHdr)); // pBt->minLeaf pBt->minLeaf = pBt->minLocal; diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 49aa9f4398..563fb53e98 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -111,6 +111,7 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); int tdbPageDropCell(SPage *pPage, int idx); void tdbPageCopy(SPage *pFromPage, SPage *pToPage); +int tdbPageCapacity(int pageSize, int amHdrSize); static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { SCell *pCell; diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/page/tdbPage.c index 3301202a33..accd174e55 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/page/tdbPage.c @@ -242,6 +242,18 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { pToPage->nOverflow = pFromPage->nOverflow; } +int tdbPageCapacity(int pageSize, int amHdrSize) { + int szPageHdr; + + if (pageSize < 65536) { + szPageHdr = pageMethods.szPageHdr; + } else { + szPageHdr = pageLargeMethods.szPageHdr; + } + + return pageSize - szPageHdr - amHdrSize; +} + static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { SCell *pFreeCell; u8 *pOffset; From 71e04b29ec5e06ba5ea434080f4f712523308a3d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 09:14:50 +0000 Subject: [PATCH 05/25] more TDB --- source/libs/tdb/src/db/tdbBtree.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index b568de848e..4ea4d6a8f1 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -84,6 +84,8 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S SBTree *pBt; int ret; + ASSERT(keyLen != 0); + *ppBt = NULL; pBt = (SBTree *)tdbOsCalloc(1, sizeof(*pBt)); @@ -921,7 +923,7 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo } // 2. Encode payload part - if (leaf) { + if (leaf && vLen > 0) { ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, pVal, vLen, &nPayload); } else { ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, NULL, 0, &nPayload); @@ -960,6 +962,7 @@ static int tdbBtreeDecodePayload(SPage *pPage, const u8 *pPayload, SCellDecoder return 0; } +// TODO: here has problem static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder) { u8 flags; u8 leaf; From 0b336e693961edf7118a5c3ca1a62bb743e27025 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 09:19:19 +0000 Subject: [PATCH 06/25] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 5f35ac6efb..61f62320c4 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -171,7 +171,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { int metaGetTbNum(SMeta *pMeta) { // TODO - ASSERT(0); + // ASSERT(0); return 0; } From 0d3a30049c685b68036370cf92c3db5cd1d99af3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 09:50:00 +0000 Subject: [PATCH 07/25] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 64 +++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 61f62320c4..9940ae166d 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -96,12 +96,70 @@ int metaOpenDB(SMeta *pMeta) { } void metaCloseDB(SMeta *pMeta) { - // TODO - ASSERT(0); + if (pMeta->pDB) { + tdbDbClose(pMeta->pDB->pCtbIdx); + tdbDbClose(pMeta->pDB->pNtbIdx); + tdbDbClose(pMeta->pDB->pStbIdx); + tdbDbClose(pMeta->pDB->pNameIdx); + tdbDbClose(pMeta->pDB->pSchemaDB); + tdbDbClose(pMeta->pDB->pTbDB); + taosMemoryFree(pMeta->pDB); + } } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { - // TODO + tb_uid_t uid; + SMetaDB *pMetaDb; + void *pKey; + void *pVal; + int kLen; + int vLen; + int ret; + + pMetaDb = pMeta->pDB; + + // TODO: make this operation pre-process + if (pTbCfg->type == META_SUPER_TABLE) { + uid = pTbCfg->stbCfg.suid; + } else { + uid = metaGenerateUid(pMeta); + } + + // save to table.db + ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen); + if (ret < 0) { + return -1; + } + + // save to schema.db + ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen); + if (ret < 0) { + return -1; + } + + // update name.idx + ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, NULL, 0); + if (ret < 0) { + return -1; + } + + if (pTbCfg->type == META_SUPER_TABLE) { + ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, NULL, 0); + if (ret < 0) { + return -1; + } + } else if (pTbCfg->type == META_CHILD_TABLE) { + ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, NULL, 0); + if (ret < 0) { + return -1; + } + } else if (pTbCfg->type == META_NORMAL_TABLE) { + ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, NULL, 0); + if (ret < 0) { + return -1; + } + } + return 0; } From 2838ecbfe67bcdaba4036eeb68ea8aee939a8b4d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 10:09:09 +0000 Subject: [PATCH 08/25] refact TDB --- source/libs/tdb/CMakeLists.txt | 3 +- source/libs/tdb/src/{page => db}/tdbPage.c | 77 ++++++++++++++++++ source/libs/tdb/src/page/tdbPageL.c | 93 ---------------------- 3 files changed, 78 insertions(+), 95 deletions(-) rename source/libs/tdb/src/{page => db}/tdbPage.c (85%) delete mode 100644 source/libs/tdb/src/page/tdbPageL.c diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 0b1378dc5a..3a62b19c10 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -9,9 +9,8 @@ target_sources(tdb "src/db/tdbDb.c" "src/db/tdbEnv.c" "src/db/tdbTxn.c" + "src/db/tdbPage.c" "src/db/tdbOs.c" - "src/page/tdbPage.c" - "src/page/tdbPageL.c" ) target_include_directories( diff --git a/source/libs/tdb/src/page/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c similarity index 85% rename from source/libs/tdb/src/page/tdbPage.c rename to source/libs/tdb/src/db/tdbPage.c index accd174e55..cc4ce20802 100644 --- a/source/libs/tdb/src/page/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -515,4 +515,81 @@ SPageMethods pageMethods = { setPageCellOffset, // setCellOffset getPageFreeCellInfo, // getFreeCellInfo setPageFreeCellInfo // setFreeCellInfo +}; + +typedef struct __attribute__((__packed__)) { + u8 cellNum[3]; + u8 cellBody[3]; + u8 cellFree[3]; + u8 nFree[3]; +} SPageHdrL; + +typedef struct __attribute__((__packed__)) { + u8 szCell[3]; + u8 nxOffset[3]; +} SFreeCellL; + +// cellNum +static inline int getLPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } +static inline void setLPageCellNum(SPage *pPage, int cellNum) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum, cellNum); +} + +// cellBody +static inline int getLPageCellBody(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody); } +static inline void setLPageCellBody(SPage *pPage, int cellBody) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody, cellBody); +} + +// cellFree +static inline int getLPageCellFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree); } +static inline void setLPageCellFree(SPage *pPage, int cellFree) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree, cellFree); +} + +// nFree +static inline int getLPageNFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree); } +static inline void setLPageNFree(SPage *pPage, int nFree) { + TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree, nFree); +} + +// cell offset +static inline int getLPageCellOffset(SPage *pPage, int idx) { + ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); + return TDB_GET_U24(pPage->pCellIdx + 3 * idx); +} + +static inline void setLPageCellOffset(SPage *pPage, int idx, int offset) { + TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset); +} + +// free cell info +static inline void getLPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + *szCell = TDB_GET_U24(pFreeCell->szCell); + *nxOffset = TDB_GET_U24(pFreeCell->nxOffset); +} + +static inline void setLPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { + SFreeCellL *pFreeCell = (SFreeCellL *)pCell; + TDB_PUT_U24(pFreeCell->szCell, szCell); + TDB_PUT_U24(pFreeCell->nxOffset, nxOffset); +} + +SPageMethods pageLargeMethods = { + 3, // szOffset + sizeof(SPageHdrL), // szPageHdr + sizeof(SFreeCellL), // szFreeCell + getLPageCellNum, // getCellNum + setLPageCellNum, // setCellNum + getLPageCellBody, // getCellBody + setLPageCellBody, // setCellBody + getLPageCellFree, // getCellFree + setLPageCellFree, // setCellFree + getLPageNFree, // getFreeBytes + setLPageNFree, // setFreeBytes + getLPageCellOffset, // getCellOffset + setLPageCellOffset, // setCellOffset + getLPageFreeCellInfo, // getFreeCellInfo + setLPageFreeCellInfo // setFreeCellInfo }; \ No newline at end of file diff --git a/source/libs/tdb/src/page/tdbPageL.c b/source/libs/tdb/src/page/tdbPageL.c deleted file mode 100644 index c5d4a6047f..0000000000 --- a/source/libs/tdb/src/page/tdbPageL.c +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbInt.h" - -typedef struct __attribute__((__packed__)) { - u8 cellNum[3]; - u8 cellBody[3]; - u8 cellFree[3]; - u8 nFree[3]; -} SPageHdrL; - -typedef struct __attribute__((__packed__)) { - u8 szCell[3]; - u8 nxOffset[3]; -} SFreeCellL; - -// cellNum -static inline int getPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } -static inline void setPageCellNum(SPage *pPage, int cellNum) { - TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum, cellNum); -} - -// cellBody -static inline int getPageCellBody(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody); } -static inline void setPageCellBody(SPage *pPage, int cellBody) { - TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellBody, cellBody); -} - -// cellFree -static inline int getPageCellFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree); } -static inline void setPageCellFree(SPage *pPage, int cellFree) { - TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellFree, cellFree); -} - -// nFree -static inline int getPageNFree(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree); } -static inline void setPageNFree(SPage *pPage, int nFree) { - TDB_PUT_U24(((SPageHdrL *)(pPage->pPageHdr))[0].nFree, nFree); -} - -// cell offset -static inline int getPageCellOffset(SPage *pPage, int idx) { - ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); - return TDB_GET_U24(pPage->pCellIdx + 3 * idx); -} - -static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { - TDB_PUT_U24(pPage->pCellIdx + 3 * idx, offset); -} - -// free cell info -static inline void getPageFreeCellInfo(SCell *pCell, int *szCell, int *nxOffset) { - SFreeCellL *pFreeCell = (SFreeCellL *)pCell; - *szCell = TDB_GET_U24(pFreeCell->szCell); - *nxOffset = TDB_GET_U24(pFreeCell->nxOffset); -} - -static inline void setPageFreeCellInfo(SCell *pCell, int szCell, int nxOffset) { - SFreeCellL *pFreeCell = (SFreeCellL *)pCell; - TDB_PUT_U24(pFreeCell->szCell, szCell); - TDB_PUT_U24(pFreeCell->nxOffset, nxOffset); -} - -SPageMethods pageLargeMethods = { - 3, // szOffset - sizeof(SPageHdrL), // szPageHdr - sizeof(SFreeCellL), // szFreeCell - getPageCellNum, // getCellNum - setPageCellNum, // setCellNum - getPageCellBody, // getCellBody - setPageCellBody, // setCellBody - getPageCellFree, // getCellFree - setPageCellFree, // setCellFree - getPageNFree, // getFreeBytes - setPageNFree, // setFreeBytes - getPageCellOffset, // getCellOffset - setPageCellOffset, // setCellOffset - getPageFreeCellInfo, // getFreeCellInfo - setPageFreeCellInfo // setFreeCellInfo -}; \ No newline at end of file From bee3e3e20418ca44d4af8fab7a428285020dc0d2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 10:21:16 +0000 Subject: [PATCH 09/25] more TDB --- source/libs/tdb/src/db/tdbDb.c | 6 ++++++ source/libs/tdb/src/inc/tdbDb.h | 1 + 2 files changed, 7 insertions(+) diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 68adb7ccfc..e2213ed858 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -129,5 +129,11 @@ int tdbDbcClose(TDBC *pDbc) { tdbOsFree(pDbc); } + return 0; +} + +int tdbDbcInsert(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen) { + // TODO + ASSERT(0); return 0; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index 4fbf65829d..f382940a83 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -34,6 +34,7 @@ int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbDbcOpen(TDB *pDb, TDBC **ppDbc); int tdbDbNext(TDBC *pDbc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbDbcClose(TDBC *pDbc); +int tdbDbcInsert(TDBC *pDbc, const void *pKey, int keyLen, const void *pVal, int valLen); #ifdef __cplusplus } From dfaa78daf69c0f5e5f5b9ebb005f19d055da6dc8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Mar 2022 10:43:57 +0000 Subject: [PATCH 10/25] refact --- source/libs/tdb/src/inc/tdbTxn.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tdb/src/inc/tdbTxn.h b/source/libs/tdb/src/inc/tdbTxn.h index 4300dc8324..cc11369785 100644 --- a/source/libs/tdb/src/inc/tdbTxn.h +++ b/source/libs/tdb/src/inc/tdbTxn.h @@ -20,7 +20,7 @@ extern "C" { #endif -typedef struct STxn STXN; +typedef struct STxn TXN; struct STxn { u64 txnId; From e3e78784e3b8bc2ed0d0fb732683494d3f792394 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 03:47:48 +0000 Subject: [PATCH 11/25] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 78 +++++++++++++++++++++-- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 9940ae166d..b8daf4e4d2 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -27,6 +27,72 @@ struct SMetaDB { TDB *pCtbIdx; }; +typedef struct __attribute__((__packed__)) { + tb_uid_t uid; + int32_t sver; +} SSchemaDbKey; + +typedef struct { + char *name; + tb_uid_t uid; +} SNameIdxKey; + +typedef struct { + tb_uid_t suid; + tb_uid_t uid; +} SCtbIdxKey; + +static inline int metaUidCmpr(const void *arg1, int len1, const void *arg2, int len2) { + tb_uid_t uid1, uid2; + + ASSERT(len1 == sizeof(tb_uid_t)); + ASSERT(len2 == sizeof(tb_uid_t)); + + uid1 = ((tb_uid_t *)arg1)[0]; + uid2 = ((tb_uid_t *)arg2)[1]; + + if (uid1 < uid2) { + return -1; + } + if (uid1 == uid2) { + return 0; + } else { + return 1; + } +} + +static inline int metaSchemaKeyCmpr(const void *arg1, int len1, const void *arg2, int len2) { + int c; + SSchemaDbKey *pKey1 = (SSchemaDbKey *)arg1; + SSchemaDbKey *pKey2 = (SSchemaDbKey *)arg2; + + c = metaUidCmpr(arg1, sizeof(tb_uid_t), arg2, sizeof(tb_uid_t)); + if (c) return c; + + if (pKey1->sver > pKey2->sver) { + return 1; + } else if (pKey1->sver == pKey2->sver) { + return 0; + } else { + return -1; + } +} + +static inline int metaNameIdxCmpr(const void *arg1, int len1, const void *arg2, int len2) { + return strcmp((char *)arg1, (char *)arg2); +} + +static inline int metaCtbIdxCmpr(const void *arg1, int len1, const void *arg2, int len2) { + int c; + SCtbIdxKey *pKey1 = (SCtbIdxKey *)arg1; + SCtbIdxKey *pKey2 = (SCtbIdxKey *)arg2; + + c = metaUidCmpr(arg1, sizeof(tb_uid_t), arg2, sizeof(tb_uid_t)); + if (c) return c; + + return metaUidCmpr(&pKey1->uid, sizeof(tb_uid_t), &pKey2->uid, sizeof(tb_uid_t)); +} + int metaOpenDB(SMeta *pMeta) { SMetaDB *pMetaDb; int ret; @@ -48,7 +114,7 @@ int metaOpenDB(SMeta *pMeta) { } // open table DB - ret = tdbDbOpen("table.db", sizeof(tb_uid_t), TDB_VARIANT_LEN, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pTbDB)); + ret = tdbDbOpen("table.db", sizeof(tb_uid_t), TDB_VARIANT_LEN, metaUidCmpr, pMetaDb->pEnv, &(pMetaDb->pTbDB)); if (ret < 0) { // TODO ASSERT(0); @@ -56,7 +122,7 @@ int metaOpenDB(SMeta *pMeta) { } // open schema DB - ret = tdbDbOpen("schema.db", sizeof(tb_uid_t) + sizeof(int32_t), TDB_VARIANT_LEN, NULL /*TODO*/, pMetaDb->pEnv, + ret = tdbDbOpen("schema.db", sizeof(tb_uid_t) + sizeof(int32_t), TDB_VARIANT_LEN, metaSchemaKeyCmpr, pMetaDb->pEnv, &(pMetaDb->pSchemaDB)); if (ret < 0) { // TODO @@ -64,28 +130,28 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - ret = tdbDbOpen("name.idx", TDB_VARIANT_LEN, 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pNameIdx)); + ret = tdbDbOpen("name.idx", TDB_VARIANT_LEN, 0, metaNameIdxCmpr, pMetaDb->pEnv, &(pMetaDb->pNameIdx)); if (ret < 0) { // TODO ASSERT(0); return -1; } - ret = tdbDbOpen("stb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pStbIdx)); + ret = tdbDbOpen("stb.idx", sizeof(tb_uid_t), 0, metaUidCmpr, pMetaDb->pEnv, &(pMetaDb->pStbIdx)); if (ret < 0) { // TODO ASSERT(0); return -1; } - ret = tdbDbOpen("ntb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pNtbIdx)); + ret = tdbDbOpen("ntb.idx", sizeof(tb_uid_t), 0, metaUidCmpr, pMetaDb->pEnv, &(pMetaDb->pNtbIdx)); if (ret < 0) { // TODO ASSERT(0); return -1; } - ret = tdbDbOpen("ctb.idx", sizeof(tb_uid_t), 0, NULL /*TODO*/, pMetaDb->pEnv, &(pMetaDb->pCtbIdx)); + ret = tdbDbOpen("ctb.idx", sizeof(tb_uid_t), 0, metaCtbIdxCmpr, pMetaDb->pEnv, &(pMetaDb->pCtbIdx)); if (ret < 0) { // TODO ASSERT(0); From 9b3d8b0e85496490110f5ff61e38b1e3f7c897b8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 05:34:30 +0000 Subject: [PATCH 12/25] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 91 ++++++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index b8daf4e4d2..34fd607ce1 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -42,6 +42,9 @@ typedef struct { tb_uid_t uid; } SCtbIdxKey; +static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); +static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); + static inline int metaUidCmpr(const void *arg1, int len1, const void *arg2, int len2) { tb_uid_t uid1, uid2; @@ -122,7 +125,7 @@ int metaOpenDB(SMeta *pMeta) { } // open schema DB - ret = tdbDbOpen("schema.db", sizeof(tb_uid_t) + sizeof(int32_t), TDB_VARIANT_LEN, metaSchemaKeyCmpr, pMetaDb->pEnv, + ret = tdbDbOpen("schema.db", sizeof(SSchemaDbKey), TDB_VARIANT_LEN, metaSchemaKeyCmpr, pMetaDb->pEnv, &(pMetaDb->pSchemaDB)); if (ret < 0) { // TODO @@ -151,7 +154,7 @@ int metaOpenDB(SMeta *pMeta) { return -1; } - ret = tdbDbOpen("ctb.idx", sizeof(tb_uid_t), 0, metaCtbIdxCmpr, pMetaDb->pEnv, &(pMetaDb->pCtbIdx)); + ret = tdbDbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, metaCtbIdxCmpr, pMetaDb->pEnv, &(pMetaDb->pCtbIdx)); if (ret < 0) { // TODO ASSERT(0); @@ -181,6 +184,8 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { int kLen; int vLen; int ret; + char buf[512]; + void *pBuf; pMetaDb = pMeta->pDB; @@ -192,6 +197,11 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { } // save to table.db + pKey = &uid; + kLen = sizeof(uid); + pVal = pBuf = buf; + metaEncodeTbInfo(&pBuf, pTbCfg); + vLen = POINTER_DISTANCE(pBuf, buf); ret = tdbDbInsert(pMetaDb->pTbDB, pKey, kLen, pVal, vLen); if (ret < 0) { return -1; @@ -345,3 +355,80 @@ SMSmaCursor *metaOpenSmaCursor(SMeta *pMeta, tb_uid_t uid) { ASSERT(0); return NULL; } + +static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW) { + int tlen = 0; + SSchema *pSchema; + + tlen += taosEncodeFixedU32(buf, pSW->nCols); + for (int i = 0; i < pSW->nCols; i++) { + pSchema = pSW->pSchema + i; + tlen += taosEncodeFixedI8(buf, pSchema->type); + tlen += taosEncodeFixedI16(buf, pSchema->colId); + tlen += taosEncodeFixedI32(buf, pSchema->bytes); + tlen += taosEncodeString(buf, pSchema->name); + } + + return tlen; +} + +static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW) { + SSchema *pSchema; + + buf = taosDecodeFixedU32(buf, &pSW->nCols); + pSW->pSchema = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * pSW->nCols); + for (int i = 0; i < pSW->nCols; i++) { + pSchema = pSW->pSchema + i; + buf = taosDecodeFixedI8(buf, &pSchema->type); + buf = taosDecodeFixedI16(buf, &pSchema->colId); + buf = taosDecodeFixedI32(buf, &pSchema->bytes); + buf = taosDecodeStringTo(buf, pSchema->name); + } + + return buf; +} + +static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg) { + int tsize = 0; + + tsize += taosEncodeString(buf, pTbCfg->name); + tsize += taosEncodeFixedU32(buf, pTbCfg->ttl); + tsize += taosEncodeFixedU32(buf, pTbCfg->keep); + tsize += taosEncodeFixedU8(buf, pTbCfg->info); + + if (pTbCfg->type == META_SUPER_TABLE) { + SSchemaWrapper sw = {.nCols = pTbCfg->stbCfg.nTagCols, .pSchema = pTbCfg->stbCfg.pTagSchema}; + tsize += metaEncodeSchema(buf, &sw); + } else if (pTbCfg->type == META_CHILD_TABLE) { + tsize += taosEncodeFixedU64(buf, pTbCfg->ctbCfg.suid); + tsize += tdEncodeKVRow(buf, pTbCfg->ctbCfg.pTag); + } else if (pTbCfg->type == META_NORMAL_TABLE) { + // TODO + } else { + ASSERT(0); + } + + return tsize; +} + +static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg) { + buf = taosDecodeString(buf, &(pTbCfg->name)); + buf = taosDecodeFixedU32(buf, &(pTbCfg->ttl)); + buf = taosDecodeFixedU32(buf, &(pTbCfg->keep)); + buf = taosDecodeFixedU8(buf, &(pTbCfg->info)); + + if (pTbCfg->type == META_SUPER_TABLE) { + SSchemaWrapper sw; + buf = metaDecodeSchema(buf, &sw); + pTbCfg->stbCfg.nTagCols = sw.nCols; + pTbCfg->stbCfg.pTagSchema = sw.pSchema; + } else if (pTbCfg->type == META_CHILD_TABLE) { + buf = taosDecodeFixedU64(buf, &(pTbCfg->ctbCfg.suid)); + buf = tdDecodeKVRow(buf, &(pTbCfg->ctbCfg.pTag)); + } else if (pTbCfg->type == META_NORMAL_TABLE) { + // TODO + } else { + ASSERT(0); + } + return buf; +} From 015e4fc3946289f9b1772964f7656120f88a46d4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 05:57:39 +0000 Subject: [PATCH 13/25] more TDB --- source/dnode/vnode/src/meta/metaTDBImpl.c | 78 ++++++++++++++++++----- 1 file changed, 61 insertions(+), 17 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 34fd607ce1..354a9f7eaf 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -44,6 +44,8 @@ typedef struct { static int metaEncodeTbInfo(void **buf, STbCfg *pTbCfg); static void *metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); +static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); +static void *metaDecodeSchema(void *buf, SSchemaWrapper *pSW); static inline int metaUidCmpr(const void *arg1, int len1, const void *arg2, int len2) { tb_uid_t uid1, uid2; @@ -177,15 +179,18 @@ void metaCloseDB(SMeta *pMeta) { } int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { - tb_uid_t uid; - SMetaDB *pMetaDb; - void *pKey; - void *pVal; - int kLen; - int vLen; - int ret; - char buf[512]; - void *pBuf; + tb_uid_t uid; + SMetaDB *pMetaDb; + void *pKey; + void *pVal; + int kLen; + int vLen; + int ret; + char buf[512]; + void *pBuf; + SCtbIdxKey ctbIdxKey; + SSchemaDbKey schemaDbKey; + SSchemaWrapper schemaWrapper; pMetaDb = pMeta->pDB; @@ -207,30 +212,69 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { return -1; } - // save to schema.db - ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen); - if (ret < 0) { - return -1; + // save to schema.db for META_SUPER_TABLE and META_NORMAL_TABLE + if (pTbCfg->type != META_CHILD_TABLE) { + schemaDbKey.uid = uid; + schemaDbKey.sver = 0; // TODO + pKey = &schemaDbKey; + kLen = sizeof(schemaDbKey); + + if (pTbCfg->type == META_SUPER_TABLE) { + schemaWrapper.nCols = pTbCfg->stbCfg.nCols; + schemaWrapper.pSchema = pTbCfg->stbCfg.pSchema; + } else { + schemaWrapper.nCols = pTbCfg->ntbCfg.nCols; + schemaWrapper.pSchema = pTbCfg->ntbCfg.pSchema; + } + pVal = pBuf = buf; + metaEncodeSchema(&pBuf, &schemaWrapper); + vLen = POINTER_DISTANCE(pBuf, buf); + ret = tdbDbInsert(pMetaDb->pSchemaDB, pKey, kLen, pVal, vLen); + if (ret < 0) { + return -1; + } } // update name.idx - ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, NULL, 0); + int nameLen; + memcpy(buf, pTbCfg->name, nameLen + 1); + ((tb_uid_t *)(buf + nameLen + 1))[0] = uid; + pKey = buf; + kLen = nameLen + 1 + sizeof(uid); + pVal = NULL; + vLen = 0; + ret = tdbDbInsert(pMetaDb->pNameIdx, pKey, kLen, pVal, vLen); if (ret < 0) { return -1; } + // update other index if (pTbCfg->type == META_SUPER_TABLE) { - ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, NULL, 0); + pKey = &uid; + kLen = sizeof(uid); + pVal = NULL; + vLen = 0; + ret = tdbDbInsert(pMetaDb->pStbIdx, pKey, kLen, pVal, vLen); if (ret < 0) { return -1; } } else if (pTbCfg->type == META_CHILD_TABLE) { - ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, NULL, 0); + ctbIdxKey.suid = pTbCfg->ctbCfg.suid; + ctbIdxKey.uid = uid; + pKey = &ctbIdxKey; + kLen = sizeof(ctbIdxKey); + pVal = NULL; + vLen = 0; + ret = tdbDbInsert(pMetaDb->pCtbIdx, pKey, kLen, pVal, vLen); if (ret < 0) { return -1; } } else if (pTbCfg->type == META_NORMAL_TABLE) { - ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, NULL, 0); + pKey = &uid; + kLen = sizeof(uid); + pVal = NULL; + vLen = 0; + ret = tdbDbInsert(pMetaDb->pNtbIdx, pKey, kLen, pVal, vLen); if (ret < 0) { return -1; } From 8a0d117acfa56efd7bec8f79a4d0c5fb3bc2d9a2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 06:38:22 +0000 Subject: [PATCH 14/25] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 46 ++++++++++++++++++++--- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 354a9f7eaf..a1280587b0 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -290,15 +290,49 @@ int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { } STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { - // TODO - ASSERT(0); - return NULL; + int ret; + SMetaDB *pMetaDb = pMeta->pDB; + void *pKey; + void *pVal; + int kLen; + int vLen; + STbCfg *pTbCfg; + + // Fetch + pKey = &uid; + kLen = sizeof(uid); + ret = tdbDbGet(pMetaDb->pTbDB, pKey, kLen, &pVal, &vLen); + if (ret < 0) { + return NULL; + } + + // Decode + pTbCfg = taosMemoryMalloc(sizeof(*pTbCfg)); + metaDecodeTbInfo(pVal, pTbCfg); + + TDB_FREE(pVal); + + return pTbCfg; } STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { - // TODO - ASSERT(0); - return NULL; + void *pKey; + void *pVal; + int kLen; + int vLen; + int ret; + + pKey = tbname; + kLen = strlen(tbname) + 1; + ret = tdbDbGet(pMeta->pDB->pNameIdx, pKey, kLen, &pVal, &vLen); + if (ret < 0) { + return NULL; + } + + *uid = *(tb_uid_t *)POINTER_SHIFT(pVal, kLen); + TDB_FREE(pVal); + + return metaGetTbInfoByUid(pMeta, *uid); } SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { From 5d1d22552a44137b5a6793f27da913bf9e3a12ae Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 07:04:17 +0000 Subject: [PATCH 15/25] more meta --- source/dnode/vnode/src/meta/metaTDBImpl.c | 122 +++++++++++++++++++--- 1 file changed, 109 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index a1280587b0..eff235b67c 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -301,6 +301,7 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { // Fetch pKey = &uid; kLen = sizeof(uid); + pVal = NULL; ret = tdbDbGet(pMetaDb->pTbDB, pKey, kLen, &pVal, &vLen); if (ret < 0) { return NULL; @@ -324,6 +325,7 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { pKey = tbname; kLen = strlen(tbname) + 1; + pVal = NULL; ret = tdbDbGet(pMeta->pDB->pNameIdx, pKey, kLen, &pVal, &vLen); if (ret < 0) { return NULL; @@ -336,34 +338,128 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { } SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, bool isinline) { - // TODO - ASSERT(0); - return NULL; + void *pKey; + void *pVal; + int kLen; + int vLen; + int ret; + SSchemaDbKey schemaDbKey; + SSchemaWrapper *pSchemaWrapper; + void *pBuf; + + // fetch + schemaDbKey.uid = uid; + schemaDbKey.sver = sver; + pKey = &schemaDbKey; + kLen = sizeof(schemaDbKey); + pVal = NULL; + ret = tdbDbGet(pMeta->pDB->pSchemaDB, pKey, kLen, &pVal, &vLen); + if (ret < 0) { + return NULL; + } + + // decode + pBuf = pVal; + pSchemaWrapper = taosMemoryMalloc(sizeof(*pSchemaWrapper)); + metaDecodeSchema(pBuf, pSchemaWrapper); + + TDB_FREE(pVal); + + return pSchemaWrapper; } STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { - // TODO - ASSERT(0); - return NULL; + tb_uid_t quid; + SSchemaWrapper *pSW; + STSchemaBuilder sb; + SSchema *pSchema; + STSchema *pTSchema; + STbCfg *pTbCfg; + + pTbCfg = metaGetTbInfoByUid(pMeta, uid); + if (pTbCfg->type == META_CHILD_TABLE) { + quid = pTbCfg->ctbCfg.suid; + } else { + quid = uid; + } + + pSW = metaGetTableSchema(pMeta, quid, sver, true); + if (pSW == NULL) { + return NULL; + } + + tdInitTSchemaBuilder(&sb, 0); + for (int i = 0; i < pSW->nCols; i++) { + pSchema = pSW->pSchema + i; + tdAddColToSchema(&sb, pSchema->type, pSchema->colId, pSchema->bytes); + } + pTSchema = tdGetSchemaFromBuilder(&sb); + tdDestroyTSchemaBuilder(&sb); + + return pTSchema; } +struct SMTbCursor { + TDBC *pDbc; +}; + SMTbCursor *metaOpenTbCursor(SMeta *pMeta) { - // TODO - ASSERT(0); - return NULL; + SMTbCursor *pTbCur = NULL; + SMetaDB *pDB = pMeta->pDB; + + pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); + if (pTbCur == NULL) { + return NULL; + } + + tdbDbcOpen(pDB->pTbDB, &pTbCur->pDbc); + + return pTbCur; } void metaCloseTbCursor(SMTbCursor *pTbCur) { - // TODO - ASSERT(0); + if (pTbCur) { + if (pTbCur->pDbc) { + tdbDbcClose(pTbCur->pDbc); + } + taosMemoryFree(pTbCur); + } } char *metaTbCursorNext(SMTbCursor *pTbCur) { - // TODO - ASSERT(0); + void *pKey = NULL; + void *pVal = NULL; + int kLen; + int vLen; + int ret; + void *pBuf; + STbCfg tbCfg; + + for (;;) { + ret = tdbDbNext(pTbCur->pDbc, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + pBuf = pVal; + metaDecodeTbInfo(pBuf, &tbCfg); + if (tbCfg.type == META_SUPER_TABLE) { + taosMemoryFree(tbCfg.name); + taosMemoryFree(tbCfg.stbCfg.pTagSchema); + continue; + ; + } else if (tbCfg.type == META_CHILD_TABLE) { + kvRowFree(tbCfg.ctbCfg.pTag); + } + + return tbCfg.name; + } + return NULL; } +struct SMCtbCursor { + TDBC *pCur; + tb_uid_t suid; +}; + SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { // TODO ASSERT(0); From e9585e74f7fa8cc25267fba52da2bd8ad907253e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 07:15:07 +0000 Subject: [PATCH 16/25] more TDB --- source/dnode/vnode/src/meta/metaTDBImpl.c | 52 +++++++++++++++++++---- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index eff235b67c..acc0319009 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -458,23 +458,59 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { struct SMCtbCursor { TDBC *pCur; tb_uid_t suid; + void *pKey; + void *pVal; + int kLen; + int vLen; }; SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { - // TODO - ASSERT(0); - return NULL; + SMCtbCursor *pCtbCur = NULL; + SMetaDB *pDB = pMeta->pDB; + int ret; + + pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); + if (pCtbCur == NULL) { + return NULL; + } + + pCtbCur->suid = uid; + ret = tdbDbcOpen(pDB->pCtbIdx, &pCtbCur->pCur); + if (ret < 0) { + taosMemoryFree(pCtbCur); + return NULL; + } + + // TODO: move the cursor to the suid there + + return pCtbCur; } void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) { - // TODO - ASSERT(0); + if (pCtbCur) { + if (pCtbCur->pCur) { + tdbDbcClose(pCtbCur->pCur); + + TDB_FREE(pCtbCur->pKey); + TDB_FREE(pCtbCur->pVal); + } + + taosMemoryFree(pCtbCur); + } } tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { - // TODO - ASSERT(0); - return 0; + int ret; + SCtbIdxKey *pCtbIdxKey; + + ret = tdbDbNext(pCtbCur->pCur, &pCtbCur->pKey, &pCtbCur->kLen, &pCtbCur->pVal, &pCtbCur->vLen); + if (ret < 0) { + return 0; + } + + pCtbIdxKey = pCtbCur->pVal; + + return pCtbIdxKey->uid; } int metaGetTbNum(SMeta *pMeta) { From d22d069bd95e41a986746e2977bd174fffdacb4e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 07:57:06 +0000 Subject: [PATCH 17/25] add an interface for index db query --- source/libs/tdb/src/db/tdbBtree.c | 34 ++++++++++++++++++++++++++++++ source/libs/tdb/src/db/tdbDb.c | 4 ++++ source/libs/tdb/src/inc/tdbBtree.h | 1 + source/libs/tdb/src/inc/tdbDb.h | 1 + 4 files changed, 40 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 4ea4d6a8f1..e0c27b6788 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -220,6 +220,40 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen return 0; } +int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { + SBTC btc; + SCell *pCell; + int cret; + void *pTKey; + void *pTVal; + SCellDecoder cd; + + tdbBtcOpen(&btc, pBt); + + tdbBtCursorMoveTo(&btc, pKey, kLen, &cret); + if (cret) { + return cret; + } + + pCell = tdbPageGetCell(btc.pPage, btc.idx); + tdbBtreeDecodeCell(btc.pPage, pCell, &cd); + + pTKey = TDB_REALLOC(*ppKey, cd.kLen); + pTVal = TDB_REALLOC(*ppVal, cd.vLen); + + if (pTKey == NULL || pTVal == NULL) { + TDB_FREE(pTKey); + TDB_FREE(pTVal); + } + + *ppKey = pTKey; + *ppVal = pTVal; + *pkLen = cd.kLen; + *vLen = cd.vLen; + + return 0; +} + static int tdbBtCursorMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { int ret; SBTree *pBt; diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index e2213ed858..e41e41f72a 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -96,6 +96,10 @@ int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen) { return tdbBtreeGet(pDb->pBt, pKey, kLen, ppVal, vLen); } +int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen) { + return tdbBtreePGet(pDb->pBt, pKey, kLen, ppKey, pkLen, ppVal, vLen); +} + int tdbDbcOpen(TDB *pDb, TDBC **ppDbc) { int ret; TDBC *pDbc = NULL; diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index e46e8bb78b..0e8ca0c803 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -43,6 +43,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, FKeyComparator kcmpr, SB int tdbBtreeClose(SBTree *pBt); int tdbBtCursorInsert(SBTC *pCur, const void *pKey, int kLen, const void *pVal, int vLen); int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); // SBTC int tdbBtcOpen(SBTC *pCur, SBTree *pBt); diff --git a/source/libs/tdb/src/inc/tdbDb.h b/source/libs/tdb/src/inc/tdbDb.h index f382940a83..e60371c734 100644 --- a/source/libs/tdb/src/inc/tdbDb.h +++ b/source/libs/tdb/src/inc/tdbDb.h @@ -29,6 +29,7 @@ int tdbDbClose(TDB *pDb); int tdbDbDrop(TDB *pDb); int tdbDbInsert(TDB *pDb, const void *pKey, int keyLen, const void *pVal, int valLen); int tdbDbGet(TDB *pDb, const void *pKey, int kLen, void **ppVal, int *vLen); +int tdbDbPGet(TDB *pDb, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); // TDBC int tdbDbcOpen(TDB *pDb, TDBC **ppDbc); From 4552f9ba2f5b1dfb4fd0a36a4cb94c89c3b2c821 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 07:57:38 +0000 Subject: [PATCH 18/25] fix a problem --- source/dnode/vnode/src/meta/metaTDBImpl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index acc0319009..634907b0f5 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -163,6 +163,7 @@ int metaOpenDB(SMeta *pMeta) { return -1; } + pMeta->pDB = pMetaDb; return 0; } @@ -236,7 +237,7 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { } // update name.idx - int nameLen; + int nameLen = strlen(pTbCfg->name); memcpy(buf, pTbCfg->name, nameLen + 1); ((tb_uid_t *)(buf + nameLen + 1))[0] = uid; pKey = buf; From 22d43247a99a3c920a09753409b15d780c039bbf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 08:10:02 +0000 Subject: [PATCH 19/25] fix another bug --- source/dnode/vnode/src/meta/metaTDBImpl.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 634907b0f5..867e597b9b 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -320,6 +320,8 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { void *pKey; void *pVal; + void *ppKey; + int pkLen; int kLen; int vLen; int ret; @@ -327,12 +329,16 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { pKey = tbname; kLen = strlen(tbname) + 1; pVal = NULL; - ret = tdbDbGet(pMeta->pDB->pNameIdx, pKey, kLen, &pVal, &vLen); + ppKey = NULL; + ret = tdbDbPGet(pMeta->pDB->pNameIdx, pKey, kLen, &ppKey, &pkLen, &pVal, &vLen); if (ret < 0) { return NULL; } - *uid = *(tb_uid_t *)POINTER_SHIFT(pVal, kLen); + ASSERT(pkLen == kLen + sizeof(uid)); + + *uid = *(tb_uid_t *)POINTER_SHIFT(ppKey, kLen); + TDB_FREE(ppKey); TDB_FREE(pVal); return metaGetTbInfoByUid(pMeta, *uid); From 7b932e1b25ddc3ab581523226180689bf04be2dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 08:15:55 +0000 Subject: [PATCH 20/25] fix pget problem --- source/libs/tdb/src/db/tdbBtree.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index e0c27b6788..b133e1fcbc 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -251,6 +251,9 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL *pkLen = cd.kLen; *vLen = cd.vLen; + memcpy(*ppKey, cd.pKey, cd.kLen); + memcpy(*ppVal, cd.pVal, cd.vLen); + return 0; } From f0e6b6ede6d67fe63ef5d9363532c0f79a58fbbf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 29 Mar 2022 08:38:42 +0000 Subject: [PATCH 21/25] fix a stupic bug --- source/dnode/vnode/src/meta/metaTDBImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 867e597b9b..ee928ef039 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -54,7 +54,7 @@ static inline int metaUidCmpr(const void *arg1, int len1, const void *arg2, int ASSERT(len2 == sizeof(tb_uid_t)); uid1 = ((tb_uid_t *)arg1)[0]; - uid2 = ((tb_uid_t *)arg2)[1]; + uid2 = ((tb_uid_t *)arg2)[0]; if (uid1 < uid2) { return -1; From 9f93b1ebc6a7cf4faf868d108c6c7d865543962a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 29 Mar 2022 08:14:46 -0400 Subject: [PATCH 22/25] TD-14281 deal memory leak --- include/libs/nodes/plannodes.h | 2 +- source/client/src/clientEnv.c | 3 + source/libs/nodes/src/nodesCodeFuncs.c | 7 + source/libs/nodes/src/nodesUtilFuncs.c | 370 +++++++++++++++++++-- source/libs/parser/src/parTranslater.c | 3 + source/libs/parser/src/parser.c | 2 + source/libs/planner/src/planPhysiCreater.c | 15 +- source/libs/planner/src/planScaleOut.c | 2 + source/libs/planner/test/plannerTest.cpp | 5 + 9 files changed, 371 insertions(+), 38 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 68e081cd15..a41f10068a 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -197,7 +197,6 @@ typedef struct STableScanPhysiNode { SScanPhysiNode scan; uint8_t scanFlag; // denotes reversed scan of data or not STimeWindow scanRange; - SNode* pScanConditions; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; @@ -252,6 +251,7 @@ typedef struct SIntervalPhysiNode { int64_t sliding; int8_t intervalUnit; int8_t slidingUnit; + uint8_t precision; SFillNode* pFill; } SIntervalPhysiNode; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 8e67703ce5..53cc77af1a 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -200,6 +200,9 @@ static void doDestroyRequest(void *p) { taosArrayDestroy(pRequest->body.showInfo.pArray); } + taosArrayDestroy(pRequest->tableList); + taosArrayDestroy(pRequest->dbList); + deregisterRequest(pRequest); taosMemoryFreeClear(pRequest); } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 01d49cb364..32040a2e3f 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1057,6 +1057,7 @@ static const char* jkIntervalPhysiPlanIntervalUnit = "intervalUnit"; static const char* jkIntervalPhysiPlanSlidingUnit = "slidingUnit"; static const char* jkIntervalPhysiPlanFill = "Fill"; static const char* jkIntervalPhysiPlanTsPk = "TsPk"; +static const char* jkIntervalPhysiPlanPrecision = "Precision"; static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; @@ -1083,6 +1084,9 @@ static int32_t physiIntervalNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkIntervalPhysiPlanTsPk, nodeToJson, pNode->pTspk); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkIntervalPhysiPlanPrecision, pNode->precision); + } return code; } @@ -1112,6 +1116,9 @@ static int32_t jsonToPhysiIntervalNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkIntervalPhysiPlanTsPk, (SNode**)&pNode->pTspk); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkIntervalPhysiPlanPrecision, &pNode->precision); + } return code; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index bd8e9a7518..18874d7c5b 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -204,49 +204,159 @@ SNodeptr nodesMakeNode(ENodeType type) { return NULL; } -static EDealRes destroyNode(SNode** pNode, void* pContext) { - switch (nodeType(*pNode)) { +static void destroyVgDataBlockArray(SArray* pArray) { + size_t size = taosArrayGetSize(pArray); + for (size_t i = 0; i < size; ++i) { + SVgDataBlocks* pVg = taosArrayGetP(pArray, i); + taosMemoryFreeClear(pVg->pData); + taosMemoryFreeClear(pVg); + } + taosArrayDestroy(pArray); +} + +static void destroyLogicNode(SLogicNode* pNode) { + nodesDestroyList(pNode->pChildren); + nodesDestroyNode(pNode->pConditions); + nodesDestroyList(pNode->pTargets); +} + +static void destroyPhysiNode(SPhysiNode* pNode) { + nodesDestroyList(pNode->pChildren); + nodesDestroyNode(pNode->pConditions); + nodesDestroyNode(pNode->pOutputDataBlockDesc); +} + +static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) { + destroyPhysiNode((SPhysiNode*)pNode); + nodesDestroyList(pNode->pExprs); + nodesDestroyList(pNode->pFuncs); +} + +static void destroyScanPhysiNode(SScanPhysiNode* pNode) { + destroyPhysiNode((SPhysiNode*)pNode); + nodesDestroyList(pNode->pScanCols); +} + +static void destroyDataSinkNode(SDataSinkNode* pNode) { + nodesDestroyNode(pNode->pInputDataBlockDesc); +} + +void nodesDestroyNode(SNodeptr pNode) { + if (NULL == pNode) { + return; + } + + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: // pProjectRef is weak reference, no need to release + break; case QUERY_NODE_VALUE: { - SValueNode* pValue = (SValueNode*)*pNode; - + SValueNode* pValue = (SValueNode*)pNode; taosMemoryFreeClear(pValue->literal); if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { taosMemoryFreeClear(pValue->datum.p); } - + break; + } + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOp = (SOperatorNode*)pNode; + nodesDestroyNode(pOp->pLeft); + nodesDestroyNode(pOp->pRight); break; } case QUERY_NODE_LOGIC_CONDITION: - nodesClearList(((SLogicConditionNode*)(*pNode))->pParameterList); + nodesDestroyList(((SLogicConditionNode*)pNode)->pParameterList); break; case QUERY_NODE_FUNCTION: - nodesClearList(((SFunctionNode*)(*pNode))->pParameterList); + nodesDestroyList(((SFunctionNode*)pNode)->pParameterList); break; case QUERY_NODE_REAL_TABLE: { - SRealTableNode* pReal = (SRealTableNode*)*pNode; + SRealTableNode* pReal = (SRealTableNode*)pNode; taosMemoryFreeClear(pReal->pMeta); taosMemoryFreeClear(pReal->pVgroupList); break; } case QUERY_NODE_TEMP_TABLE: - nodesDestroyNode(((STempTableNode*)(*pNode))->pSubquery); + nodesDestroyNode(((STempTableNode*)pNode)->pSubquery); break; + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoin = (SJoinTableNode*)pNode; + nodesDestroyNode(pJoin->pLeft); + nodesDestroyNode(pJoin->pRight); + nodesDestroyNode(pJoin->pOnCond); + break; + } case QUERY_NODE_GROUPING_SET: - nodesClearList(((SGroupingSetNode*)(*pNode))->pParameterList); + nodesDestroyList(((SGroupingSetNode*)pNode)->pParameterList); break; + case QUERY_NODE_ORDER_BY_EXPR: + nodesDestroyNode(((SOrderByExprNode*)pNode)->pExpr); + break; + case QUERY_NODE_LIMIT: // no pointer field + break; + case QUERY_NODE_STATE_WINDOW: + nodesDestroyNode(((SStateWindowNode*)pNode)->pCol); + break; + case QUERY_NODE_SESSION_WINDOW: { + SSessionWindowNode* pSession = (SSessionWindowNode*)pNode; + nodesDestroyNode(pSession->pCol); + nodesDestroyNode(pSession->pGap); + break; + } + case QUERY_NODE_INTERVAL_WINDOW: { + SIntervalWindowNode* pJoin = (SIntervalWindowNode*)pNode; + nodesDestroyNode(pJoin->pCol); + nodesDestroyNode(pJoin->pInterval); + nodesDestroyNode(pJoin->pOffset); + nodesDestroyNode(pJoin->pSliding); + nodesDestroyNode(pJoin->pFill); + break; + } case QUERY_NODE_NODE_LIST: - nodesClearList(((SNodeListNode*)(*pNode))->pNodeList); + nodesDestroyList(((SNodeListNode*)pNode)->pNodeList); break; + case QUERY_NODE_FILL: + nodesDestroyNode(((SFillNode*)pNode)->pValues); + break; + case QUERY_NODE_RAW_EXPR: + nodesDestroyNode(((SRawExprNode*)pNode)->pNode); + break; + case QUERY_NODE_TARGET: + nodesDestroyNode(((STargetNode*)pNode)->pExpr); + break; + case QUERY_NODE_DATABLOCK_DESC: + nodesDestroyList(((SDataBlockDescNode*)pNode)->pSlots); + break; + case QUERY_NODE_SLOT_DESC: // no pointer field + case QUERY_NODE_COLUMN_DEF: // no pointer field + case QUERY_NODE_DOWNSTREAM_SOURCE: // no pointer field + break; + case QUERY_NODE_DATABASE_OPTIONS: + nodesDestroyList(((SDatabaseOptions*)pNode)->pRetentions); + break; + case QUERY_NODE_TABLE_OPTIONS: { + STableOptions* pStmt = (STableOptions*)pNode; + nodesDestroyList(pStmt->pSma); + nodesDestroyList(pStmt->pFuncs); + break; + } case QUERY_NODE_INDEX_OPTIONS: { - SIndexOptions* pStmt = (SIndexOptions*)*pNode; + SIndexOptions* pStmt = (SIndexOptions*)pNode; nodesDestroyList(pStmt->pFuncs); nodesDestroyNode(pStmt->pInterval); nodesDestroyNode(pStmt->pOffset); nodesDestroyNode(pStmt->pSliding); break; } + case QUERY_NODE_SET_OPERATOR: { + SSetOperator* pStmt = (SSetOperator*)pNode; + nodesDestroyNode(pStmt->pLeft); + nodesDestroyNode(pStmt->pRight); + nodesDestroyList(pStmt->pOrderByList); + nodesDestroyNode(pStmt->pLimit); + break; + } case QUERY_NODE_SELECT_STMT: { - SSelectStmt* pStmt = (SSelectStmt*)*pNode; + SSelectStmt* pStmt = (SSelectStmt*)pNode; nodesDestroyList(pStmt->pProjectionList); nodesDestroyNode(pStmt->pFromTable); nodesDestroyNode(pStmt->pWhere); @@ -259,50 +369,238 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { nodesDestroyNode(pStmt->pSlimit); break; } - case QUERY_NODE_VNODE_MODIF_STMT: { - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)*pNode; - size_t size = taosArrayGetSize(pStmt->pDataBlocks); - for (size_t i = 0; i < size; ++i) { - SVgDataBlocks* pVg = taosArrayGetP(pStmt->pDataBlocks, i); - taosMemoryFreeClear(pVg->pData); - taosMemoryFreeClear(pVg); - } - taosArrayDestroy(pStmt->pDataBlocks); + case QUERY_NODE_VNODE_MODIF_STMT: + destroyVgDataBlockArray(((SVnodeModifOpStmt*)pNode)->pDataBlocks); + break; + case QUERY_NODE_CREATE_DATABASE_STMT: + nodesDestroyNode(((SCreateDatabaseStmt*)pNode)->pOptions); + break; + case QUERY_NODE_DROP_DATABASE_STMT: // no pointer field + break; + case QUERY_NODE_ALTER_DATABASE_STMT: + nodesDestroyNode(((SAlterDatabaseStmt*)pNode)->pOptions); break; - } case QUERY_NODE_CREATE_TABLE_STMT: { - SCreateTableStmt* pStmt = (SCreateTableStmt*)*pNode; + SCreateTableStmt* pStmt = (SCreateTableStmt*)pNode; nodesDestroyList(pStmt->pCols); nodesDestroyList(pStmt->pTags); + nodesDestroyNode(pStmt->pOptions); break; } case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: { - SCreateSubTableClause* pStmt = (SCreateSubTableClause*)*pNode; + SCreateSubTableClause* pStmt = (SCreateSubTableClause*)pNode; nodesDestroyList(pStmt->pSpecificTags); nodesDestroyList(pStmt->pValsOfTags); break; } case QUERY_NODE_CREATE_MULTI_TABLE_STMT: - nodesDestroyList(((SCreateMultiTableStmt*)(*pNode))->pSubTables); + nodesDestroyList(((SCreateMultiTableStmt*)pNode)->pSubTables); + break; + case QUERY_NODE_DROP_TABLE_CLAUSE: // no pointer field + break; + case QUERY_NODE_DROP_TABLE_STMT: + nodesDestroyNode(((SDropTableStmt*)pNode)->pTables); + break; + case QUERY_NODE_DROP_SUPER_TABLE_STMT: // no pointer field + break; + case QUERY_NODE_ALTER_TABLE_STMT: { + SAlterTableStmt* pStmt = (SAlterTableStmt*)pNode; + nodesDestroyNode(pStmt->pOptions); + nodesDestroyNode(pStmt->pVal); + break; + } + case QUERY_NODE_CREATE_USER_STMT: // no pointer field + case QUERY_NODE_ALTER_USER_STMT: // no pointer field + case QUERY_NODE_DROP_USER_STMT: // no pointer field + case QUERY_NODE_USE_DATABASE_STMT: // no pointer field + case QUERY_NODE_CREATE_DNODE_STMT: // no pointer field + case QUERY_NODE_DROP_DNODE_STMT: // no pointer field + case QUERY_NODE_ALTER_DNODE_STMT: // no pointer field break; case QUERY_NODE_CREATE_INDEX_STMT: { - SCreateIndexStmt* pStmt = (SCreateIndexStmt*)*pNode; + SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pNode; nodesDestroyNode(pStmt->pOptions); nodesDestroyList(pStmt->pCols); break; } + case QUERY_NODE_DROP_INDEX_STMT: // no pointer field + case QUERY_NODE_CREATE_QNODE_STMT: // no pointer field + case QUERY_NODE_DROP_QNODE_STMT: // no pointer field + break; + case QUERY_NODE_CREATE_TOPIC_STMT: + nodesDestroyNode(((SCreateTopicStmt*)pNode)->pQuery); + break; + case QUERY_NODE_DROP_TOPIC_STMT: // no pointer field + case QUERY_NODE_ALTER_LOCAL_STMT: // no pointer field + break; + case QUERY_NODE_SHOW_DATABASES_STMT: + case QUERY_NODE_SHOW_TABLES_STMT: + case QUERY_NODE_SHOW_STABLES_STMT: + case QUERY_NODE_SHOW_USERS_STMT: + case QUERY_NODE_SHOW_DNODES_STMT: + case QUERY_NODE_SHOW_VGROUPS_STMT: + case QUERY_NODE_SHOW_MNODES_STMT: + case QUERY_NODE_SHOW_MODULES_STMT: + case QUERY_NODE_SHOW_QNODES_STMT: + case QUERY_NODE_SHOW_FUNCTIONS_STMT: + case QUERY_NODE_SHOW_INDEXES_STMT: + case QUERY_NODE_SHOW_STREAMS_STMT: { + SShowStmt* pStmt = (SShowStmt*)pNode; + nodesDestroyNode(pStmt->pDbName); + nodesDestroyNode(pStmt->pTbNamePattern); + break; + } + case QUERY_NODE_LOGIC_PLAN_SCAN: { + SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pScanCols); + taosMemoryFreeClear(pLogicNode->pMeta); + taosMemoryFreeClear(pLogicNode->pVgroupList); + break; + } + case QUERY_NODE_LOGIC_PLAN_JOIN: { + SJoinLogicNode* pLogicNode = (SJoinLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyNode(pLogicNode->pOnConditions); + break; + } + case QUERY_NODE_LOGIC_PLAN_AGG: { + SAggLogicNode* pLogicNode = (SAggLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pAggFuncs); + nodesDestroyList(pLogicNode->pGroupKeys); + break; + } + case QUERY_NODE_LOGIC_PLAN_PROJECT: { + SProjectLogicNode* pLogicNode = (SProjectLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pProjections); + break; + } + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIF: { + SVnodeModifLogicNode* pLogicNode = (SVnodeModifLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + destroyVgDataBlockArray(pLogicNode->pDataBlocks); + taosMemoryFreeClear(pLogicNode->pVgDataBlocks->pData); + taosMemoryFreeClear(pLogicNode->pVgDataBlocks); + break; + } + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + destroyLogicNode((SLogicNode*)pNode); + break; + case QUERY_NODE_LOGIC_PLAN_WINDOW: { + SWindowLogicNode* pLogicNode = (SWindowLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pFuncs); + nodesDestroyNode(pLogicNode->pFill); + nodesDestroyNode(pLogicNode->pTspk); + break; + } + case QUERY_NODE_LOGIC_PLAN_SORT: { + SSortLogicNode* pLogicNode = (SSortLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pSortKeys); + break; + } + case QUERY_NODE_LOGIC_PLAN_PARTITION: { + SPartitionLogicNode* pLogicNode = (SPartitionLogicNode*)pNode; + destroyLogicNode((SLogicNode*)pLogicNode); + nodesDestroyList(pLogicNode->pPartitionKeys); + break; + } + case QUERY_NODE_LOGIC_SUBPLAN: { + SLogicSubplan* pSubplan = (SLogicSubplan*)pNode; + nodesDestroyList(pSubplan->pChildren); + nodesDestroyNode(pSubplan->pNode); + taosMemoryFreeClear(pSubplan->pVgroupList); + break; + } + case QUERY_NODE_LOGIC_PLAN: + nodesDestroyList(((SQueryLogicPlan*)pNode)->pTopSubplans); + break; + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + destroyScanPhysiNode((SScanPhysiNode*)pNode); + break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + destroyScanPhysiNode((SScanPhysiNode*)pNode); + break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + destroyScanPhysiNode((SScanPhysiNode*)pNode); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + destroyScanPhysiNode((SScanPhysiNode*)pNode); + break; + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + destroyScanPhysiNode((SScanPhysiNode*)pNode); + break; + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { + SProjectPhysiNode* pPhyNode = (SProjectPhysiNode*)pNode; + destroyPhysiNode((SPhysiNode*)pPhyNode); + nodesDestroyList(pPhyNode->pProjections); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_JOIN: { + SJoinPhysiNode* pPhyNode = (SJoinPhysiNode*)pNode; + destroyPhysiNode((SPhysiNode*)pPhyNode); + nodesDestroyNode(pPhyNode->pOnConditions); + nodesDestroyList(pPhyNode->pTargets); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_AGG: { + SAggPhysiNode* pPhyNode = (SAggPhysiNode*)pNode; + destroyPhysiNode((SPhysiNode*)pPhyNode); + nodesDestroyList(pPhyNode->pExprs); + nodesDestroyList(pPhyNode->pAggFuncs); + nodesDestroyList(pPhyNode->pGroupKeys); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: { + SExchangePhysiNode* pPhyNode = (SExchangePhysiNode*)pNode; + destroyPhysiNode((SPhysiNode*)pPhyNode); + nodesDestroyList(pPhyNode->pSrcEndPoints); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SORT: { + SSortPhysiNode* pPhyNode = (SSortPhysiNode*)pNode; + destroyPhysiNode((SPhysiNode*)pPhyNode); + nodesDestroyNode(pPhyNode->pExprs); + nodesDestroyNode(pPhyNode->pSortKeys); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: { + SIntervalPhysiNode* pPhyNode = (SIntervalPhysiNode*)pNode; + destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + nodesDestroyNode(pPhyNode->pFill); + nodesDestroyNode(pPhyNode->pTspk); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW: + destroyWinodwPhysiNode((SWinodwPhysiNode*)pNode); + break; + case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: + destroyDataSinkNode((SDataSinkNode*)pNode); + break; + case QUERY_NODE_PHYSICAL_PLAN_INSERT: { + SDataInserterNode* pSink = (SDataInserterNode*)pNode; + destroyDataSinkNode((SDataSinkNode*)pSink); + taosMemoryFreeClear(pSink->pData); + break; + } + case QUERY_NODE_PHYSICAL_SUBPLAN: { + SSubplan* pSubplan = (SSubplan*)pNode; + nodesDestroyList(pSubplan->pChildren); + nodesDestroyNode(pSubplan->pNode); + nodesDestroyNode(pSubplan->pDataSink); + break; + } + case QUERY_NODE_PHYSICAL_PLAN: + nodesDestroyList(((SQueryPlan*)pNode)->pSubplans); + break; default: break; } - taosMemoryFreeClear(*pNode); - return DEAL_RES_CONTINUE; -} - -void nodesDestroyNode(SNodeptr pNode) { - if (NULL == pNode) { - return; - } - nodesRewriteNodePostOrder((SNode**)&pNode, destroyNode, NULL); + taosMemoryFreeClear(pNode); + return; } SNodeList* nodesMakeList() { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0f26bec9b2..e3fa2fc631 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -228,6 +228,9 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p pCol->colType = isTag ? COLUMN_TYPE_TAG : COLUMN_TYPE_COLUMN; pCol->node.resType.type = pColSchema->type; pCol->node.resType.bytes = pColSchema->bytes; + if (TSDB_DATA_TYPE_TIMESTAMP == pCol->node.resType.type) { + pCol->node.resType.precision = pTable->pMeta->tableInfo.precision; + } } static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SColumnNode* pCol) { diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 92bd6f111b..d9bff4b9ef 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -58,5 +58,7 @@ void qDestroyQuery(SQuery* pQueryNode) { taosMemoryFreeClear(pQueryNode->pCmdMsg->pMsg); taosMemoryFreeClear(pQueryNode->pCmdMsg); } + taosArrayDestroy(pQueryNode->pDbList); + taosArrayDestroy(pQueryNode->pTableList); taosMemoryFreeClear(pQueryNode); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index db0201bc81..004597bccb 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -780,6 +780,7 @@ static int32_t createIntervalPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChil pInterval->sliding = pWindowLogicNode->sliding; pInterval->intervalUnit = pWindowLogicNode->intervalUnit; pInterval->slidingUnit = pWindowLogicNode->slidingUnit; + pInterval->precision = ((SColumnNode*)pWindowLogicNode->pTspk)->node.resType.precision; pInterval->pFill = nodesCloneNode(pWindowLogicNode->pFill); if (NULL != pWindowLogicNode->pFill && NULL == pInterval->pFill) { @@ -1080,6 +1081,16 @@ static int32_t doCreatePhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogi return code; } +static void destoryLocationHash(void* p) { + SHashObj* pHash = *(SHashObj**)p; + // todo + taosHashCleanup(pHash); +} + +static void destoryPhysiPlanContext(SPhysiPlanContext* pCxt) { + taosArrayDestroyEx(pCxt->pLocationHelper, destoryLocationHash); +} + int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) { SPhysiPlanContext cxt = { .pPlanCxt = pCxt, @@ -1091,5 +1102,7 @@ int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryP if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } - return doCreatePhysiPlan(&cxt, pLogicPlan, pPlan); + int32_t code = doCreatePhysiPlan(&cxt, pLogicPlan, pPlan); + destoryPhysiPlanContext(&cxt); + return code; } diff --git a/source/libs/planner/src/planScaleOut.c b/source/libs/planner/src/planScaleOut.c index ca6c7a2577..2b5fd12e22 100644 --- a/source/libs/planner/src/planScaleOut.c +++ b/source/libs/planner/src/planScaleOut.c @@ -167,6 +167,8 @@ static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32 if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pCurrentGroup); + } else { + nodesClearList(pCurrentGroup); } return code; diff --git a/source/libs/planner/test/plannerTest.cpp b/source/libs/planner/test/plannerTest.cpp index ea6498c0c8..7b706242e5 100644 --- a/source/libs/planner/test/plannerTest.cpp +++ b/source/libs/planner/test/plannerTest.cpp @@ -248,6 +248,11 @@ TEST_F(PlannerTest, showTables) { bind("show tables"); ASSERT_TRUE(run()); + + setDatabase("root", "information_schema"); + + bind("show tables"); + ASSERT_TRUE(run()); } TEST_F(PlannerTest, showStables) { From abfb80acac4a729b63cb435edf48f7c6a6c4cdb4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 30 Mar 2022 03:21:57 +0000 Subject: [PATCH 23/25] still use BDB --- source/dnode/vnode/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 6b21f935c4..56a8698d77 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,5 +1,5 @@ set(META_DB_IMPL_LIST "BDB" "TDB") -set(META_DB_IMPL "TDB" CACHE STRING "Use BDB as the default META implementation") +set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) From 2a422c3abb05d0200c42bcc6b934dbaa5c2ed477 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 30 Mar 2022 11:31:59 +0800 Subject: [PATCH 24/25] Feature/sangshuduo/td 13603 python connector (#11114) * add a few stmt_ interface declartion * add taos_load_table_info * add taos_set_config(() interface * add more interfaces --- include/client/taos.h | 20 ++++++++++++++++++++ source/client/src/clientMain.c | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/include/client/taos.h b/include/client/taos.h index 0260316618..111cd8ad3b 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -124,8 +124,25 @@ typedef struct TAOS_MULTI_BIND { int num; } TAOS_MULTI_BIND; +typedef enum { + SET_CONF_RET_SUCC = 0, + SET_CONF_RET_ERR_PART = -1, + SET_CONF_RET_ERR_INNER = -2, + SET_CONF_RET_ERR_JSON_INVALID = -3, + SET_CONF_RET_ERR_JSON_PARSE = -4, + SET_CONF_RET_ERR_ONLY_ONCE = -5, + SET_CONF_RET_ERR_TOO_LONG = -6 +} SET_CONF_RET_CODE; + +#define RET_MSG_LENGTH 1024 +typedef struct setConfRet { + SET_CONF_RET_CODE retCode; + char retMsg[RET_MSG_LENGTH]; +} setConfRet; + DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); +DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); @@ -168,10 +185,13 @@ DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); DLL_EXPORT void taos_stop_query(TAOS_RES *res); DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); +DLL_EXPORT bool taos_is_update_query(TAOS_RES *res); DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); +DLL_EXPORT void taos_reset_current_db(TAOS *taos); DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); +DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_client_info(); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 40c601f63f..42d4b5b2e5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -71,6 +71,12 @@ void taos_cleanup(void) { tscInfo("all local resources released"); } +setConfRet taos_set_config(const char *config) { + // TODO + setConfRet ret = {SET_CONF_RET_SUCC, {0}}; + return ret; +} + TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db); if (user == NULL) { @@ -257,6 +263,11 @@ int *taos_fetch_lengths(TAOS_RES *res) { return ((SRequestObj *)res)->body.resInfo.length; } +TAOS_ROW *taos_result_block(TAOS_RES *res) { + // TODO + return NULL; +} + // todo intergrate with tDataTypes const char *taos_data_type(int type) { switch (type) { @@ -353,6 +364,11 @@ bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { return colDataIsNull_f(pCol->nullbitmap, row); } +bool taos_is_update_query(TAOS_RES *res) { + // TODO + return true; +} + int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { if (res == NULL) { return 0; @@ -376,6 +392,11 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { int taos_validate_sql(TAOS *taos, const char *sql) { return true; } +void taos_reset_current_db(TAOS *taos) { + // TODO + return; +} + const char *taos_get_server_info(TAOS *taos) { if (taos == NULL) { return NULL; From 7593fc6ae628a191e9a1ae215d66fb1e43d8e386 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 29 Mar 2022 23:47:12 -0400 Subject: [PATCH 25/25] bugfix --- source/libs/nodes/src/nodesUtilFuncs.c | 25 ++++++++++++++++++---- source/libs/planner/src/planLogicCreater.c | 2 +- source/libs/planner/src/planPhysiCreater.c | 6 +++++- source/libs/planner/src/planSpliter.c | 1 + 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 18874d7c5b..fdfef6434b 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -481,8 +481,7 @@ void nodesDestroyNode(SNodeptr pNode) { SVnodeModifLogicNode* pLogicNode = (SVnodeModifLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); destroyVgDataBlockArray(pLogicNode->pDataBlocks); - taosMemoryFreeClear(pLogicNode->pVgDataBlocks->pData); - taosMemoryFreeClear(pLogicNode->pVgDataBlocks); + // pVgDataBlocks is weak reference break; } case QUERY_NODE_LOGIC_PLAN_EXCHANGE: @@ -512,6 +511,7 @@ void nodesDestroyNode(SNodeptr pNode) { SLogicSubplan* pSubplan = (SLogicSubplan*)pNode; nodesDestroyList(pSubplan->pChildren); nodesDestroyNode(pSubplan->pNode); + nodesClearList(pSubplan->pParents); taosMemoryFreeClear(pSubplan->pVgroupList); break; } @@ -591,11 +591,28 @@ void nodesDestroyNode(SNodeptr pNode) { nodesDestroyList(pSubplan->pChildren); nodesDestroyNode(pSubplan->pNode); nodesDestroyNode(pSubplan->pDataSink); + nodesClearList(pSubplan->pParents); break; } - case QUERY_NODE_PHYSICAL_PLAN: - nodesDestroyList(((SQueryPlan*)pNode)->pSubplans); + case QUERY_NODE_PHYSICAL_PLAN: { + SQueryPlan* pPlan = (SQueryPlan*)pNode; + if (NULL != pPlan->pSubplans) { + // only need to destroy the top-level subplans, because they will recurse to all the subplans below + bool first = true; + SNode* pElement = NULL; + FOREACH(pElement, pPlan->pSubplans) { + if (first) { + first = false; + nodesDestroyNode(pElement); + } else { + nodesClearList(((SNodeListNode*)pElement)->pNodeList); + taosMemoryFreeClear(pElement); + } + } + nodesClearList(pPlan->pSubplans); + } break; + } default: break; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index bd4b6481f1..c0d7ff238e 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -692,7 +692,7 @@ static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpS if (NULL == pModif) { return TSDB_CODE_OUT_OF_MEMORY; } - pModif->pDataBlocks = pStmt->pDataBlocks; + TSWAP(pModif->pDataBlocks, pStmt->pDataBlocks, SArray*); pModif->msgType = getMsgType(pStmt->sqlNodeType); *pLogicNode = (SLogicNode*)pModif; return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 004597bccb..d5e35fa594 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1083,7 +1083,11 @@ static int32_t doCreatePhysiPlan(SPhysiPlanContext* pCxt, SQueryLogicPlan* pLogi static void destoryLocationHash(void* p) { SHashObj* pHash = *(SHashObj**)p; - // todo + SSlotIndex* pIndex = taosHashIterate(pHash, NULL); + while (NULL != pIndex) { + taosArrayDestroy(pIndex->pSlotIdsInfo); + pIndex = taosHashIterate(pHash, pIndex); + } taosHashCleanup(pHash); } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index df546c32be..3ce225abab 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -140,6 +140,7 @@ static int32_t stsSplit(SSplitContext* pCxt) { code = stsCreateExchangeNode(pCxt, pInfo->pSubplan, pInfo->pScan); } ++(pCxt->groupId); + taosMemoryFreeClear(pCxt->pInfo); return code; }