From 0db6b12a47e16d16d1768cd9de5e24e1d0d5869a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 01:40:33 +0000 Subject: [PATCH 01/12] more tkv --- .../tkv/src/inc/tkvBtree.h} | 19 ++++++++++++++++++- source/libs/tkv/src/inc/tkvDB.h | 16 +++++++++++++++- .../tkv/src/inc/tkvHash.h} | 13 ++++++++----- 3 files changed, 41 insertions(+), 7 deletions(-) rename source/{dnode/vnode/src/vnd/vnodeRead.c => libs/tkv/src/inc/tkvBtree.h} (72%) rename source/{dnode/vnode/src/inc/vnodeRead.h => libs/tkv/src/inc/tkvHash.h} (81%) diff --git a/source/dnode/vnode/src/vnd/vnodeRead.c b/source/libs/tkv/src/inc/tkvBtree.h similarity index 72% rename from source/dnode/vnode/src/vnd/vnodeRead.c rename to source/libs/tkv/src/inc/tkvBtree.h index 5e9f89ccd5..6360e62c79 100644 --- a/source/dnode/vnode/src/vnd/vnodeRead.c +++ b/source/libs/tkv/src/inc/tkvBtree.h @@ -13,4 +13,21 @@ * along with this program. If not, see . */ -#include "vnodeDef.h" \ No newline at end of file +#ifndef _TD_TKV_BTREE_H_ +#define _TD_TKV_BTREE_H_ + +#include "tkvDef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + pgid_t root; // root page number +} STkvBtree; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tkv/src/inc/tkvDB.h index 1a45702540..fc8219db3f 100644 --- a/source/libs/tkv/src/inc/tkvDB.h +++ b/source/libs/tkv/src/inc/tkvDB.h @@ -16,12 +16,26 @@ #ifndef _TD_TKV_DB_H_ #define _TD_TKV_DB_H_ +#include "tkvBtree.h" + #ifdef __cplusplus extern "C" { #endif +typedef enum { + TDB_BTREE = 0, + TDB_HASH, + TDB_HEAP, +} tdb_db_t; + struct TDB { - // TODO + pgsize_t pageSize; + + tdb_db_t type; // DB type + + union { + STkvBtree btree; + } dbimpl; }; #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/vnodeRead.h b/source/libs/tkv/src/inc/tkvHash.h similarity index 81% rename from source/dnode/vnode/src/inc/vnodeRead.h rename to source/libs/tkv/src/inc/tkvHash.h index 5ce84b2ebf..06343b7ac9 100644 --- a/source/dnode/vnode/src/inc/vnodeRead.h +++ b/source/libs/tkv/src/inc/tkvHash.h @@ -13,18 +13,21 @@ * along with this program. If not, see . */ -#ifndef _TD_VNODE_READ_H_ -#define _TD_VNODE_READ_H_ +#ifndef _TD_TKV_HAHS_H_ +#define _TD_TKV_HAHS_H_ + +#include "tkvDef.h" #ifdef __cplusplus extern "C" { #endif -#include "vnodeInt.h" -void vnodeProcessReadMsg(SVnode *pVnode, SVnodeMsg *pMsg); +typedef struct STkvHash { + // TODO +} STkvHash; #ifdef __cplusplus } #endif -#endif /*_TD_VNODE_READ_H_*/ +#endif /*_TD_TKV_HAHS_H_*/ \ No newline at end of file From c31134216c9da4c07898b8bda25e9ff14419b1d4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 01:50:42 +0000 Subject: [PATCH 02/12] more --- source/libs/tkv/src/inc/tkvDB.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tkv/src/inc/tkvDB.h index fc8219db3f..6fe9dabf05 100644 --- a/source/libs/tkv/src/inc/tkvDB.h +++ b/source/libs/tkv/src/inc/tkvDB.h @@ -17,6 +17,7 @@ #define _TD_TKV_DB_H_ #include "tkvBtree.h" +#include "tkvHash.h" #ifdef __cplusplus extern "C" { @@ -30,11 +31,10 @@ typedef enum { struct TDB { pgsize_t pageSize; - - tdb_db_t type; // DB type - + tdb_db_t type; union { STkvBtree btree; + STkvhash hash; } dbimpl; }; From c8c09c8a8cbee993c3c944a62f47fa96d64067aa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 02:12:24 +0000 Subject: [PATCH 03/12] refact --- source/libs/CMakeLists.txt | 2 +- source/libs/{tkv => tdb}/CMakeLists.txt | 8 ++++---- source/libs/{tkv/inc/tkv.h => tdb/inc/tdb.h} | 8 ++++---- .../src/inc/tkvBtree.h => tdb/src/inc/tdbBtree.h} | 8 ++++---- .../inc/tkvBufPool.h => tdb/src/inc/tdbBufPool.h} | 12 ++++++------ .../{tkv/src/inc/tkvDB.h => tdb/src/inc/tdbDB.h} | 10 +++++----- .../{tkv/src/inc/tkvDef.h => tdb/src/inc/tdbDef.h} | 6 +++--- .../inc/tkvDiskMgr.h => tdb/src/inc/tdbDiskMgr.h} | 2 +- .../{tkv/src/inc/tkvEnv.h => tdb/src/inc/tdbEnv.h} | 6 +++--- .../src/inc/tkvHash.h => tdb/src/inc/tdbHash.h} | 6 +++--- .../src/inc/tkvPage.h => tdb/src/inc/tdbPage.h} | 14 ++++---------- .../{tkv/src/tkvBufPool.c => tdb/src/tdbBufPool.c} | 2 +- .../{tkv/src/tDiskMgr.c => tdb/src/tdbDiskMgr.c} | 2 +- source/libs/{tkv => tdb}/test/tDiskMgrTest.cpp | 0 source/libs/{tkv => tdb}/test/tkvTests.cpp | 0 15 files changed, 40 insertions(+), 46 deletions(-) rename source/libs/{tkv => tdb}/CMakeLists.txt (79%) rename source/libs/{tkv/inc/tkv.h => tdb/inc/tdb.h} (91%) rename source/libs/{tkv/src/inc/tkvBtree.h => tdb/src/inc/tdbBtree.h} (89%) rename source/libs/{tkv/src/inc/tkvBufPool.h => tdb/src/inc/tdbBufPool.h} (82%) rename source/libs/{tkv/src/inc/tkvDB.h => tdb/src/inc/tdbDB.h} (88%) rename source/libs/{tkv/src/inc/tkvDef.h => tdb/src/inc/tdbDef.h} (93%) rename source/libs/{tkv/src/inc/tkvDiskMgr.h => tdb/src/inc/tdbDiskMgr.h} (98%) rename source/libs/{tkv/src/inc/tkvEnv.h => tdb/src/inc/tdbEnv.h} (91%) rename source/libs/{tkv/src/inc/tkvHash.h => tdb/src/inc/tdbHash.h} (93%) rename source/libs/{tkv/src/inc/tkvPage.h => tdb/src/inc/tdbPage.h} (77%) rename source/libs/{tkv/src/tkvBufPool.c => tdb/src/tdbBufPool.c} (98%) rename source/libs/{tkv/src/tDiskMgr.c => tdb/src/tdbDiskMgr.c} (98%) rename source/libs/{tkv => tdb}/test/tDiskMgrTest.cpp (100%) rename source/libs/{tkv => tdb}/test/tkvTests.cpp (100%) diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 1dc16c74f7..1d23f333b2 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(transport) add_subdirectory(sync) -add_subdirectory(tkv) +add_subdirectory(tdb) add_subdirectory(index) add_subdirectory(wal) add_subdirectory(parser) diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt similarity index 79% rename from source/libs/tkv/CMakeLists.txt rename to source/libs/tdb/CMakeLists.txt index fec3f37cd5..1832ea0b5d 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,17 +1,17 @@ -aux_source_directory(src TKV_SRC) -add_library(tkv STATIC ${TKV_SRC}) +aux_source_directory(src TDB_SRC) +add_library(tdb STATIC ${TDB_SRC}) # target_include_directories( # tkv # PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" # PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" # ) target_include_directories( - tkv + tdb PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src/inc" ) target_link_libraries( - tkv + tdb PUBLIC os PUBLIC util ) \ No newline at end of file diff --git a/source/libs/tkv/inc/tkv.h b/source/libs/tdb/inc/tdb.h similarity index 91% rename from source/libs/tkv/inc/tkv.h rename to source/libs/tdb/inc/tdb.h index 00534d2827..2f47f545b1 100644 --- a/source/libs/tkv/inc/tkv.h +++ b/source/libs/tdb/inc/tdb.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_H_ -#define _TD_TKV_H_ +#ifndef _TD_TDB_H_ +#define _TD_TDB_H_ #include "os.h" @@ -28,7 +28,7 @@ typedef struct TDB_ENV TDB_ENV; // SKey typedef struct { - void * bdata; + void* bdata; uint32_t size; } TDB_KEY, TDB_VALUE; @@ -36,4 +36,4 @@ typedef struct { } #endif -#endif /*_TD_TKV_H_*/ \ No newline at end of file +#endif /*_TD_TDB_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvBtree.h b/source/libs/tdb/src/inc/tdbBtree.h similarity index 89% rename from source/libs/tkv/src/inc/tkvBtree.h rename to source/libs/tdb/src/inc/tdbBtree.h index 6360e62c79..c68f94bb48 100644 --- a/source/libs/tkv/src/inc/tkvBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_BTREE_H_ -#define _TD_TKV_BTREE_H_ +#ifndef _TD_TDB_BTREE_H_ +#define _TD_TDB_BTREE_H_ #include "tkvDef.h" @@ -24,10 +24,10 @@ extern "C" { typedef struct { pgid_t root; // root page number -} STkvBtree; +} TDB_BTREE; #ifdef __cplusplus } #endif -#endif /*_TD_TKV_BTREE_H_*/ \ No newline at end of file +#endif /*_TD_TDB_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvBufPool.h b/source/libs/tdb/src/inc/tdbBufPool.h similarity index 82% rename from source/libs/tkv/src/inc/tkvBufPool.h rename to source/libs/tdb/src/inc/tdbBufPool.h index ec8d177a9a..7fda5bf2e6 100644 --- a/source/libs/tkv/src/inc/tkvBufPool.h +++ b/source/libs/tdb/src/inc/tdbBufPool.h @@ -13,10 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_BUF_POOL_H_ -#define _TD_TKV_BUF_POOL_H_ +#ifndef _TD_TDB_BUF_POOL_H_ +#define _TD_TDB_BUF_POOL_H_ -#include "tkvPage.h" +#include "tdbPage.h" #ifdef __cplusplus extern "C" { @@ -26,9 +26,9 @@ typedef struct STkvBufPool STkvBufPool; int tbpOpen(STkvBufPool **ppTkvBufPool); int tbpClose(STkvBufPool *pTkvBufPool); -STkvPage *tbpNewPage(STkvBufPool *pTkvBufPool); +STdbPage *tbpNewPage(STkvBufPool *pTkvBufPool); int tbpDelPage(STkvBufPool *pTkvBufPool); -STkvPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); +STdbPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); int tbpUnpinPage(STkvBufPool *pTkvBufPool, pgid_t pgid); void tbpFlushPages(STkvBufPool *pTkvBufPool); @@ -36,4 +36,4 @@ void tbpFlushPages(STkvBufPool *pTkvBufPool); } #endif -#endif /*_TD_TKV_BUF_POOL_H_*/ \ No newline at end of file +#endif /*_TD_TDB_BUF_POOL_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tdb/src/inc/tdbDB.h similarity index 88% rename from source/libs/tkv/src/inc/tkvDB.h rename to source/libs/tdb/src/inc/tdbDB.h index 6fe9dabf05..40ddb1eb31 100644 --- a/source/libs/tkv/src/inc/tkvDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_DB_H_ -#define _TD_TKV_DB_H_ +#ifndef _TD_TDB_DB_H_ +#define _TD_TDB_DB_H_ -#include "tkvBtree.h" -#include "tkvHash.h" +#include "tdbBtree.h" +#include "tdbHash.h" #ifdef __cplusplus extern "C" { @@ -42,4 +42,4 @@ struct TDB { } #endif -#endif /*_TD_TKV_DB_H_*/ \ No newline at end of file +#endif /*_TD_TDB_DB_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDef.h b/source/libs/tdb/src/inc/tdbDef.h similarity index 93% rename from source/libs/tkv/src/inc/tkvDef.h rename to source/libs/tdb/src/inc/tdbDef.h index cd418019be..a04b8cc402 100644 --- a/source/libs/tkv/src/inc/tkvDef.h +++ b/source/libs/tdb/src/inc/tdbDef.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_DEF_H_ -#define _TD_TKV_DEF_H_ +#ifndef _TD_TDB_DEF_H_ +#define _TD_TDB_DEF_H_ #include "os.h" @@ -39,4 +39,4 @@ typedef int32_t pgsize_t; } #endif -#endif /*_TD_TKV_DEF_H_*/ \ No newline at end of file +#endif /*_TD_TDB_DEF_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDiskMgr.h b/source/libs/tdb/src/inc/tdbDiskMgr.h similarity index 98% rename from source/libs/tkv/src/inc/tkvDiskMgr.h rename to source/libs/tdb/src/inc/tdbDiskMgr.h index 2ebe98ace2..b83a147437 100644 --- a/source/libs/tkv/src/inc/tkvDiskMgr.h +++ b/source/libs/tdb/src/inc/tdbDiskMgr.h @@ -22,7 +22,7 @@ extern "C" { #include "os.h" -#include "tkvDef.h" +#include "tdbDef.h" typedef struct STkvDiskMgr STkvDiskMgr; diff --git a/source/libs/tkv/src/inc/tkvEnv.h b/source/libs/tdb/src/inc/tdbEnv.h similarity index 91% rename from source/libs/tkv/src/inc/tkvEnv.h rename to source/libs/tdb/src/inc/tdbEnv.h index eba442e5a5..f3e4ef5888 100644 --- a/source/libs/tkv/src/inc/tkvEnv.h +++ b/source/libs/tdb/src/inc/tdbEnv.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_ENV_H_ -#define _TD_TKV_ENV_H_ +#ifndef _TD_TDB_ENV_H_ +#define _TD_TDB_ENV_H_ #ifdef __cplusplus extern "C" { @@ -28,4 +28,4 @@ struct TDB_ENV { } #endif -#endif /*_TD_TKV_ENV_H_*/ \ No newline at end of file +#endif /*_TD_TDB_ENV_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvHash.h b/source/libs/tdb/src/inc/tdbHash.h similarity index 93% rename from source/libs/tkv/src/inc/tkvHash.h rename to source/libs/tdb/src/inc/tdbHash.h index 06343b7ac9..fca19035f1 100644 --- a/source/libs/tkv/src/inc/tkvHash.h +++ b/source/libs/tdb/src/inc/tdbHash.h @@ -16,15 +16,15 @@ #ifndef _TD_TKV_HAHS_H_ #define _TD_TKV_HAHS_H_ -#include "tkvDef.h" +#include "tdbDef.h" #ifdef __cplusplus extern "C" { #endif -typedef struct STkvHash { +typedef struct { // TODO -} STkvHash; +} TDB_HASH; #ifdef __cplusplus } diff --git a/source/libs/tkv/src/inc/tkvPage.h b/source/libs/tdb/src/inc/tdbPage.h similarity index 77% rename from source/libs/tkv/src/inc/tkvPage.h rename to source/libs/tdb/src/inc/tdbPage.h index d596d215cd..e7245b6c39 100644 --- a/source/libs/tkv/src/inc/tkvPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -17,30 +17,24 @@ #define _TD_TKV_PAGE_H_ #include "os.h" -#include "tkvDef.h" +#include "tdbDef.h" #ifdef __cplusplus extern "C" { #endif -typedef struct STkvPage { +typedef struct { pgid_t pgid; int32_t pinCount; bool idDirty; char* pData; -} STkvPage; +} STdbPage; typedef struct { uint16_t dbver; uint16_t pgsize; uint32_t cksm; -} STkvPgHdr; - -// typedef struct { -// SPgHdr chdr; -// uint16_t used; // number of used slots -// uint16_t loffset; // the offset of the starting location of the last slot used -// } SSlottedPgHdr; +} STdbPgHdr; #ifdef __cplusplus } diff --git a/source/libs/tkv/src/tkvBufPool.c b/source/libs/tdb/src/tdbBufPool.c similarity index 98% rename from source/libs/tkv/src/tkvBufPool.c rename to source/libs/tdb/src/tdbBufPool.c index 86bfa0ba3e..6ae3499064 100644 --- a/source/libs/tkv/src/tkvBufPool.c +++ b/source/libs/tdb/src/tdbBufPool.c @@ -26,7 +26,7 @@ struct SFrameIdWrapper { }; struct STkvBufPool { - STkvPage* pages; + STdbPage* pages; STkvDiskMgr* pDiskMgr; SHashObj* pgTb; // page_id_t --> frame_id_t TD_SLIST(SFrameIdWrapper) freeList; diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tdb/src/tdbDiskMgr.c similarity index 98% rename from source/libs/tkv/src/tDiskMgr.c rename to source/libs/tdb/src/tdbDiskMgr.c index fa8f6062d8..71ab5f2589 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tdb/src/tdbDiskMgr.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tkvDiskMgr.h" +#include "tdbDiskMgr.h" struct STkvDiskMgr { char * fname; diff --git a/source/libs/tkv/test/tDiskMgrTest.cpp b/source/libs/tdb/test/tDiskMgrTest.cpp similarity index 100% rename from source/libs/tkv/test/tDiskMgrTest.cpp rename to source/libs/tdb/test/tDiskMgrTest.cpp diff --git a/source/libs/tkv/test/tkvTests.cpp b/source/libs/tdb/test/tkvTests.cpp similarity index 100% rename from source/libs/tkv/test/tkvTests.cpp rename to source/libs/tdb/test/tkvTests.cpp From 3aae62f5dc55153df9081b01e54b9dfef3670294 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 02:39:12 +0000 Subject: [PATCH 04/12] more --- source/libs/tdb/src/inc/tdbBufPool.h | 16 ++++++++-------- source/libs/tdb/src/tdbBufPool.c | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/libs/tdb/src/inc/tdbBufPool.h b/source/libs/tdb/src/inc/tdbBufPool.h index 7fda5bf2e6..5200d22faa 100644 --- a/source/libs/tdb/src/inc/tdbBufPool.h +++ b/source/libs/tdb/src/inc/tdbBufPool.h @@ -22,15 +22,15 @@ extern "C" { #endif -typedef struct STkvBufPool STkvBufPool; +typedef struct STdbBufPool STdbBufPool; -int tbpOpen(STkvBufPool **ppTkvBufPool); -int tbpClose(STkvBufPool *pTkvBufPool); -STdbPage *tbpNewPage(STkvBufPool *pTkvBufPool); -int tbpDelPage(STkvBufPool *pTkvBufPool); -STdbPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); -int tbpUnpinPage(STkvBufPool *pTkvBufPool, pgid_t pgid); -void tbpFlushPages(STkvBufPool *pTkvBufPool); +int tbpOpen(STdbBufPool **ppTkvBufPool); +int tbpClose(STdbBufPool *pTkvBufPool); +STdbPage *tbpNewPage(STdbBufPool *pTkvBufPool); +int tbpDelPage(STdbBufPool *pTkvBufPool); +STdbPage *tbpFetchPage(STdbBufPool *pTkvBufPool, pgid_t pgid); +int tbpUnpinPage(STdbBufPool *pTkvBufPool, pgid_t pgid); +void tbpFlushPages(STdbBufPool *pTkvBufPool); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/tdbBufPool.c b/source/libs/tdb/src/tdbBufPool.c index 6ae3499064..3eed100566 100644 --- a/source/libs/tdb/src/tdbBufPool.c +++ b/source/libs/tdb/src/tdbBufPool.c @@ -25,7 +25,7 @@ struct SFrameIdWrapper { frame_id_t id; }; -struct STkvBufPool { +struct STdbBufPool { STdbPage* pages; STkvDiskMgr* pDiskMgr; SHashObj* pgTb; // page_id_t --> frame_id_t From f1df5eaabfe8df2f9e527f39a9d14c537f46409f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 05:37:12 +0000 Subject: [PATCH 05/12] more refact --- source/dnode/vnode/src/inc/vnodeDef.h | 13 ++- source/dnode/vnode/src/inc/vnodeMAF.h | 32 ------ source/dnode/vnode/src/inc/vnodeRequest.h | 33 ------ source/dnode/vnode/src/inc/vnodeSync.h | 33 ------ source/dnode/vnode/src/meta/CMakeLists.txt | 46 -------- source/dnode/vnode/src/tq/CMakeLists.txt | 20 ---- source/dnode/vnode/src/tsdb/CMakeLists.txt | 34 ------ source/dnode/vnode/src/vnd/CMakeLists.txt | 24 ----- source/dnode/vnode/src/vnd/vnodeMgr.c | 2 +- source/dnode/vnode/src/vnd/vnodeRequest.c | 119 --------------------- source/dnode/vnode/src/vnd/vnodeSync.c | 14 --- 11 files changed, 11 insertions(+), 359 deletions(-) delete mode 100644 source/dnode/vnode/src/inc/vnodeMAF.h delete mode 100644 source/dnode/vnode/src/inc/vnodeRequest.h delete mode 100644 source/dnode/vnode/src/inc/vnodeSync.h delete mode 100644 source/dnode/vnode/src/meta/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/tq/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/tsdb/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/vnd/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/vnd/vnodeRequest.c delete mode 100644 source/dnode/vnode/src/vnd/vnodeSync.c diff --git a/source/dnode/vnode/src/inc/vnodeDef.h b/source/dnode/vnode/src/inc/vnodeDef.h index 1c534a8aeb..7054d5bc11 100644 --- a/source/dnode/vnode/src/inc/vnodeDef.h +++ b/source/dnode/vnode/src/inc/vnodeDef.h @@ -33,9 +33,7 @@ #include "vnodeFS.h" #include "vnodeMemAllocator.h" #include "vnodeQuery.h" -#include "vnodeRequest.h" #include "vnodeStateMgr.h" -#include "vnodeSync.h" #ifdef __cplusplus extern "C" { @@ -73,7 +71,6 @@ struct SVnode { STsdb* pTsdb; STQ* pTq; SWal* pWal; - SVnodeSync* pSync; SVnodeFS* pFs; tsem_t canCommit; SQHandle* pQuery; @@ -84,6 +81,16 @@ int vnodeScheduleTask(SVnodeTask* task); int32_t vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq); +// For Log +extern int32_t vDebugFlag; + +#define vFatal(...) do { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) +#define vError(...) do { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) +#define vWarn(...) do { if (vDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) +#define vInfo(...) do { if (vDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) +#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeMAF.h b/source/dnode/vnode/src/inc/vnodeMAF.h deleted file mode 100644 index 7aa405103c..0000000000 --- a/source/dnode/vnode/src/inc/vnodeMAF.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_MAF_H_ -#define _TD_VNODE_MAF_H_ - -#include "vnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -int vnodeOpenMAF(SVnode *pVnode); -void vnodeCloseMAF(SVnode *pVnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_MAF_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeRequest.h b/source/dnode/vnode/src/inc/vnodeRequest.h deleted file mode 100644 index 52f4281eea..0000000000 --- a/source/dnode/vnode/src/inc/vnodeRequest.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_REQUEST_H_ -#define _TD_VNODE_REQUEST_H_ - -#include "vnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// SVDropTbReq -// int vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq); -// void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_REQUEST_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeSync.h b/source/dnode/vnode/src/inc/vnodeSync.h deleted file mode 100644 index e82979551d..0000000000 --- a/source/dnode/vnode/src/inc/vnodeSync.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_SYNC_H_ -#define _TD_VNODE_SYNC_H_ - -// #include "sync.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - /* data */ -} SVnodeSync; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_SYNC_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/CMakeLists.txt b/source/dnode/vnode/src/meta/CMakeLists.txt deleted file mode 100644 index 7041811617..0000000000 --- a/source/dnode/vnode/src/meta/CMakeLists.txt +++ /dev/null @@ -1,46 +0,0 @@ -set(META_DB_IMPL_LIST "BDB" "SQLITE") -set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") -set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) - -if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) - message(STATUS "META DB Impl: ${META_DB_IMPL}==============") -else() - message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============") -endif() - -aux_source_directory(src META_SRC) -if(${META_DB_IMPL} STREQUAL "BDB") - list(REMOVE_ITEM META_SRC "src/metaSQLiteImpl.c") -elseif(${META_DB_IMPL} STREQUAL "SQLITE") - list(REMOVE_ITEM META_SRC "src/metaBDBImpl.c") -endif() - -add_library(meta STATIC ${META_SRC}) -target_include_directories( - meta - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/meta" - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) -target_link_libraries( - meta - PUBLIC common - PUBLIC index -) - -if(${META_DB_IMPL} STREQUAL "BDB") - target_link_libraries( - meta - PUBLIC bdb - ) -elseif(${META_DB_IMPL} STREQUAL "SQLITE") - target_link_libraries( - meta - PUBLIC sqlite - ) -endif() - - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/tq/CMakeLists.txt b/source/dnode/vnode/src/tq/CMakeLists.txt deleted file mode 100644 index 7cb7499d64..0000000000 --- a/source/dnode/vnode/src/tq/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -aux_source_directory(src TQ_SRC) -add_library(tq ${TQ_SRC}) -target_include_directories( - tq - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tq" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -target_link_libraries( - tq - PUBLIC wal - PUBLIC os - PUBLIC util - PUBLIC common - PUBLIC transport -) - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/tsdb/CMakeLists.txt b/source/dnode/vnode/src/tsdb/CMakeLists.txt deleted file mode 100644 index e38ba1c466..0000000000 --- a/source/dnode/vnode/src/tsdb/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -aux_source_directory(src TSDB_SRC) -if(0) - add_library(tsdb ${TSDB_SRC}) -else(0) - add_library(tsdb STATIC "") - target_sources(tsdb - PRIVATE - "src/tsdbCommit.c" - "src/tsdbMain.c" - "src/tsdbMemTable.c" - "src/tsdbOptions.c" - "src/tsdbWrite.c" - "src/tsdbReadImpl.c" - "src/tsdbFile.c" - "src/tsdbFS.c" - "src/tsdbRead.c" - ) -endif(0) - -target_include_directories( - tsdb - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tsdb" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -target_link_libraries( - tsdb - PUBLIC os - PUBLIC util - PUBLIC common - PUBLIC tkv - PUBLIC tfs - PUBLIC meta -) \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/CMakeLists.txt b/source/dnode/vnode/src/vnd/CMakeLists.txt deleted file mode 100644 index 944a4276db..0000000000 --- a/source/dnode/vnode/src/vnd/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -aux_source_directory(src VNODE_SRC) -add_library(vnode STATIC ${VNODE_SRC}) -target_include_directories( - vnode - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode" - private "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) -target_link_libraries( - vnode - PUBLIC os - PUBLIC transport - PUBLIC meta - PUBLIC tq - PUBLIC tsdb - PUBLIC wal - PUBLIC sync - PUBLIC cjson - PUBLIC qworker -) - -# test -if(${BUILD_TEST}) -# add_subdirectory(test) -endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index fdb96e52e2..ce9e487076 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -41,7 +41,7 @@ int vnodeInit(const SVnodeOpt *pOption) { for (uint16_t i = 0; i < pOption->nthreads; i++) { pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); - pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); + // pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); } } else { // TODO: if no commit thread is set, then another mechanism should be diff --git a/source/dnode/vnode/src/vnd/vnodeRequest.c b/source/dnode/vnode/src/vnd/vnodeRequest.c deleted file mode 100644 index 5367c9e091..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeRequest.c +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" - -#if 0 - -static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq); -static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq); - -int vnodeBuildReq(void **buf, const SVnodeReq *pReq, tmsg_t type) { - int tsize = 0; - - tsize += taosEncodeFixedU64(buf, pReq->ver); - switch (type) { - case TDMT_VND_CREATE_STB: - tsize += vnodeBuildCreateTableReq(buf, &(pReq->ctReq)); - break; - case TDMT_VND_SUBMIT: - /* code */ - break; - default: - break; - } - /* TODO */ - return tsize; -} - -void *vnodeParseReq(void *buf, SVnodeReq *pReq, tmsg_t type) { - buf = taosDecodeFixedU64(buf, &(pReq->ver)); - - switch (type) { - case TDMT_VND_CREATE_STB: - buf = vnodeParseCreateTableReq(buf, &(pReq->ctReq)); - break; - - default: - break; - } - - // TODO - return buf; -} - -static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq) { - int tsize = 0; - - tsize += taosEncodeString(buf, pReq->name); - tsize += taosEncodeFixedU32(buf, pReq->ttl); - tsize += taosEncodeFixedU32(buf, pReq->keep); - tsize += taosEncodeFixedU8(buf, pReq->type); - - switch (pReq->type) { - case META_SUPER_TABLE: - tsize += taosEncodeFixedU64(buf, pReq->stbCfg.suid); - tsize += tdEncodeSchema(buf, pReq->stbCfg.pSchema); - tsize += tdEncodeSchema(buf, pReq->stbCfg.pTagSchema); - break; - case META_CHILD_TABLE: - tsize += taosEncodeFixedU64(buf, pReq->ctbCfg.suid); - tsize += tdEncodeKVRow(buf, pReq->ctbCfg.pTag); - break; - case META_NORMAL_TABLE: - tsize += tdEncodeSchema(buf, pReq->ntbCfg.pSchema); - break; - default: - break; - } - - return tsize; -} - -static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq) { - buf = taosDecodeString(buf, &(pReq->name)); - buf = taosDecodeFixedU32(buf, &(pReq->ttl)); - buf = taosDecodeFixedU32(buf, &(pReq->keep)); - buf = taosDecodeFixedU8(buf, &(pReq->type)); - - switch (pReq->type) { - case META_SUPER_TABLE: - buf = taosDecodeFixedU64(buf, &(pReq->stbCfg.suid)); - buf = tdDecodeSchema(buf, &(pReq->stbCfg.pSchema)); - buf = tdDecodeSchema(buf, &(pReq->stbCfg.pTagSchema)); - break; - case META_CHILD_TABLE: - buf = taosDecodeFixedU64(buf, &(pReq->ctbCfg.suid)); - buf = tdDecodeKVRow(buf, &(pReq->ctbCfg.pTag)); - break; - case META_NORMAL_TABLE: - buf = tdDecodeSchema(buf, &(pReq->ntbCfg.pSchema)); - break; - default: - break; - } - - return buf; -} - -int vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq) { - // TODO - return 0; -} - -void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq) { - // TODO -} -#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file From 093b982fa01b4d13349ed2315118f115de6b2550 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 05:45:03 +0000 Subject: [PATCH 06/12] more refact --- source/dnode/vnode/src/inc/vnodeDef.h | 2 -- source/dnode/vnode/src/inc/vnodeFS.h | 47 ------------------------- source/dnode/vnode/src/inc/vnodeQuery.h | 3 +- source/dnode/vnode/src/vnd/vnodeFS.c | 16 --------- source/libs/tdb/src/tdbBufPool.c | 6 ++-- 5 files changed, 4 insertions(+), 70 deletions(-) delete mode 100644 source/dnode/vnode/src/inc/vnodeFS.h delete mode 100644 source/dnode/vnode/src/vnd/vnodeFS.c diff --git a/source/dnode/vnode/src/inc/vnodeDef.h b/source/dnode/vnode/src/inc/vnodeDef.h index 7054d5bc11..85563890fa 100644 --- a/source/dnode/vnode/src/inc/vnodeDef.h +++ b/source/dnode/vnode/src/inc/vnodeDef.h @@ -30,7 +30,6 @@ #include "vnodeBufferPool.h" #include "vnodeCfg.h" #include "vnodeCommit.h" -#include "vnodeFS.h" #include "vnodeMemAllocator.h" #include "vnodeQuery.h" #include "vnodeStateMgr.h" @@ -71,7 +70,6 @@ struct SVnode { STsdb* pTsdb; STQ* pTq; SWal* pWal; - SVnodeFS* pFs; tsem_t canCommit; SQHandle* pQuery; SDnode* pDnode; diff --git a/source/dnode/vnode/src/inc/vnodeFS.h b/source/dnode/vnode/src/inc/vnodeFS.h deleted file mode 100644 index dbec985695..0000000000 --- a/source/dnode/vnode/src/inc/vnodeFS.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_FS_H_ -#define _TD_VNODE_FS_H_ - -#include "vnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { -} SDir; - -typedef struct { -} SFile; - -typedef struct SFS { - void *pImpl; - int (*startEdit)(struct SFS *); - int (*endEdit)(struct SFS *); -} SFS; - -typedef struct { -} SVnodeFS; - -int vnodeOpenFS(SVnode *pVnode); -void vnodeCloseFS(SVnode *pVnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_FS_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeQuery.h b/source/dnode/vnode/src/inc/vnodeQuery.h index d43f5b1cf1..9d40c34f00 100644 --- a/source/dnode/vnode/src/inc/vnodeQuery.h +++ b/source/dnode/vnode/src/inc/vnodeQuery.h @@ -19,12 +19,11 @@ #ifdef __cplusplus extern "C" { #endif -#include "vnodeInt.h" #include "qworker.h" +#include "vnode.h" typedef struct SQWorkerMgmt SQHandle; - int vnodeQueryOpen(SVnode *pVnode); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/vnd/vnodeFS.c b/source/dnode/vnode/src/vnd/vnodeFS.c deleted file mode 100644 index 5e9f89ccd5..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeFS.c +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" \ No newline at end of file diff --git a/source/libs/tdb/src/tdbBufPool.c b/source/libs/tdb/src/tdbBufPool.c index 3eed100566..bc3c386b0f 100644 --- a/source/libs/tdb/src/tdbBufPool.c +++ b/source/libs/tdb/src/tdbBufPool.c @@ -16,9 +16,9 @@ #include "thash.h" #include "tlist.h" -#include "tkvBufPool.h" -#include "tkvDiskMgr.h" -#include "tkvPage.h" +#include "tdbBufPool.h" +#include "tdbDiskMgr.h" +#include "tdbPage.h" struct SFrameIdWrapper { TD_SLIST_NODE(SFrameIdWrapper); From c9118a52e6776db03c854afadc9d4c77f0c9fa82 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 06:13:10 +0000 Subject: [PATCH 07/12] more tkv and refact --- source/dnode/mgmt/impl/src/dndEnv.c | 2 +- source/dnode/vnode/inc/vnode.h | 18 ---------------- source/libs/tdb/inc/tdb.h | 15 +++++++++++-- source/libs/tdb/src/inc/tdbDB.h | 11 +++------- source/libs/tdb/src/inc/tdbEnv.h | 31 --------------------------- source/libs/tdb/test/tDiskMgrTest.cpp | 10 --------- source/libs/tdb/test/tdbTest.cpp | 5 +++++ source/libs/tdb/test/tkvTests.cpp | 0 8 files changed, 22 insertions(+), 70 deletions(-) delete mode 100644 source/libs/tdb/src/inc/tdbEnv.h delete mode 100644 source/libs/tdb/test/tDiskMgrTest.cpp create mode 100644 source/libs/tdb/test/tdbTest.cpp delete mode 100644 source/libs/tdb/test/tkvTests.cpp diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 1bf1ea2b92..23fc643abe 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -293,7 +293,7 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { if (vnodeInit(&vnodeOpt) != 0) { dError("failed to init vnode since %s", terrstr()); dndCleanup(); - return NULL; + return -1; } memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg)); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6113d0a536..7ff02309ec 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -37,36 +37,18 @@ typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); typedef struct SVnodeCfg { int32_t vgId; SDnode *pDnode; - - /** vnode buffer pool options */ struct { - /** write buffer size */ uint64_t wsize; uint64_t ssize; uint64_t lsize; - /** use heap allocator or arena allocator */ bool isHeapAllocator; }; - - /** time to live of tables in this vnode */ uint32_t ttl; - - /** data to keep in this vnode */ uint32_t keep; - - /** if TS data is eventually consistency */ bool isWeak; - - /** TSDB config */ STsdbCfg tsdbCfg; - - /** META config */ SMetaCfg metaCfg; - - /** TQ config */ STqCfg tqCfg; - - /** WAL config */ SWalCfg walCfg; } SVnodeCfg; diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 2f47f545b1..40d79de821 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,9 +22,15 @@ extern "C" { #endif +typedef enum { + TDB_BTREE = 0, + TDB_HASH, + TDB_HEAP, +} tdb_db_t; + // Forward declaration -typedef struct TDB TDB; -typedef struct TDB_ENV TDB_ENV; +typedef struct TDB TDB; +typedef struct TDB_CURSOR TDB_CURSOR; // SKey typedef struct { @@ -32,6 +38,11 @@ typedef struct { uint32_t size; } TDB_KEY, TDB_VALUE; +// TDB Operations +int tdbCreateDB(TDB** dbpp); +int tdbOpenDB(TDB* dbp, tdb_db_t type, uint32_t flags); +int tdbCloseDB(TDB* dbp, uint32_t flags); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index 40ddb1eb31..479ef77711 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -23,19 +23,14 @@ extern "C" { #endif -typedef enum { - TDB_BTREE = 0, - TDB_HASH, - TDB_HEAP, -} tdb_db_t; struct TDB { pgsize_t pageSize; tdb_db_t type; union { - STkvBtree btree; - STkvhash hash; - } dbimpl; + TDB_BTREE btree; + TDB_HASH hash; + } dbam; // Different access methods }; #ifdef __cplusplus diff --git a/source/libs/tdb/src/inc/tdbEnv.h b/source/libs/tdb/src/inc/tdbEnv.h deleted file mode 100644 index f3e4ef5888..0000000000 --- a/source/libs/tdb/src/inc/tdbEnv.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_ENV_H_ -#define _TD_TDB_ENV_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -struct TDB_ENV { - char *homeDir; -}; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_ENV_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/test/tDiskMgrTest.cpp b/source/libs/tdb/test/tDiskMgrTest.cpp deleted file mode 100644 index a735fdb33d..0000000000 --- a/source/libs/tdb/test/tDiskMgrTest.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "gtest/gtest.h" - -#include "iostream" - -#include "tDiskMgr.h" - -TEST(tDiskMgrTest, simple_test) { - // TODO - std::cout << "This is in tDiskMgrTest::simple_test" << std::endl; -} \ No newline at end of file diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp new file mode 100644 index 0000000000..ee91c7669b --- /dev/null +++ b/source/libs/tdb/test/tdbTest.cpp @@ -0,0 +1,5 @@ +#include "gtest/gtest.h" + +TEST(tdb_api_test, tdb_create_open_close_db_test) { + +} \ No newline at end of file diff --git a/source/libs/tdb/test/tkvTests.cpp b/source/libs/tdb/test/tkvTests.cpp deleted file mode 100644 index e69de29bb2..0000000000 From ac87dcf9661c1b2c2793add1219a15dbec2b8ffa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 08:04:43 +0000 Subject: [PATCH 08/12] more --- source/libs/tdb/CMakeLists.txt | 6 +++++- source/libs/tdb/test/CMakeLists.txt | 0 source/libs/tdb/test/tdbTest.cpp | 11 ++++++++++- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 source/libs/tdb/test/CMakeLists.txt diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 1832ea0b5d..68aa40c072 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -14,4 +14,8 @@ target_link_libraries( tdb PUBLIC os PUBLIC util -) \ No newline at end of file +) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index ee91c7669b..3ff43fdd69 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,5 +1,14 @@ #include "gtest/gtest.h" -TEST(tdb_api_test, tdb_create_open_close_db_test) { +#include "tdb.h" +TEST(tdb_api_test, tdb_create_open_close_db_test) { + int ret; + TDB *dbp; + + tdbCreateDB(&dbp); + + tdbOpenDB(dbp, TDB_BTREE, 0); + + tdbCloseDB(dbp, 0); } \ No newline at end of file From a20724f46eda6f4dcb9bfeedf5e9c8c93a2370fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 08:15:28 +0000 Subject: [PATCH 09/12] more --- source/dnode/vnode/src/vnd/vnodeWrite.c | 1 + source/libs/tdb/CMakeLists.txt | 2 +- source/libs/tdb/test/CMakeLists.txt | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 185487757f..01619b5c77 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -83,6 +83,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error } + vTrace("vgId:%d process create table %s", pVnode->vgId, pCreateTbReq->name); if (pCreateTbReq->type == TD_SUPER_TABLE) { free(pCreateTbReq->stbCfg.pSchema); free(pCreateTbReq->stbCfg.pTagSchema); diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 68aa40c072..647771fd2d 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -17,5 +17,5 @@ target_link_libraries( ) if(${BUILD_TEST}) - add_subdirectory(test) + # add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index e69de29bb2..2d77c1f4e9 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -0,0 +1,3 @@ +# tdbTest +add_executable(tdbTest "tdbTest.cpp") +target_link_libraries(tdbTest tdb gtest gtest_main) \ No newline at end of file From 6a0b6671cdb576ac40b2c49217d2b2893357a83a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 13 Jan 2022 06:24:22 -0500 Subject: [PATCH 10/12] TD-12678 single table aggregation physical plan code --- include/libs/planner/planner.h | 19 +++ include/libs/planner/plannerOp.h | 2 +- source/libs/planner/src/physicalPlan.c | 25 +++- source/libs/planner/src/physicalPlanJson.c | 153 ++++++++++++++++----- source/libs/planner/test/phyPlanTests.cpp | 32 ++++- 5 files changed, 192 insertions(+), 39 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index d98c1ca595..4f1db37f79 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -119,6 +119,25 @@ typedef struct SExchangePhyNode { SArray *pSrcEndPoints; // SEpAddr, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhyNode; +typedef enum EAggAlgo { + AGG_ALGO_PLAIN = 1, // simple agg across all input rows + AGG_ALGO_SORTED, // grouped agg, input must be sorted + AGG_ALGO_HASHED // grouped agg, use internal hashtable +} EAggAlgo; + +typedef enum EAggSplit { + AGG_SPLIT_PRE = 1, // first level agg, maybe don't need calculate the final result + AGG_SPLIT_FINAL // second level agg, must calculate the final result +} EAggSplit; + +typedef struct SAggPhyNode { + SPhyNode node; + EAggAlgo aggAlgo; // algorithm used by agg operator + EAggSplit aggSplit; // distributed splitting mode + SArray *pExprs; // SExprInfo list, these are expression list of group_by_clause and parameter expression of aggregate function + SArray *pGroupByList; // SColIndex list, but these must be column node +} SAggPhyNode; + typedef struct SSubplanId { uint64_t queryId; uint64_t templateId; diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 41d6e028cf..5cc896f1c2 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -30,7 +30,7 @@ OP_ENUM_MACRO(TagScan) OP_ENUM_MACRO(SystemTableScan) OP_ENUM_MACRO(Aggregate) OP_ENUM_MACRO(Project) -OP_ENUM_MACRO(Groupby) +// OP_ENUM_MACRO(Groupby) OP_ENUM_MACRO(Limit) OP_ENUM_MACRO(SLimit) OP_ENUM_MACRO(TimeWindow) diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 71e69c67d2..65c97110ce 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -189,7 +189,6 @@ static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableI return (SPhyNode*)node; } - static bool isSystemTable(SQueryTableInfo* pTable) { // todo return false; @@ -295,13 +294,31 @@ static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTabl static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; - if (needMultiNodeScan(pTable)) { return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable)); } return createSingleTableScanNode(pPlanNode, pTable, pCxt->pCurrentSubplan); } +static SPhyNode* createSingleTableAgg(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SAggPhyNode* node = (SAggPhyNode*)initPhyNode(pPlanNode, OP_Aggregate, sizeof(SAggPhyNode)); + SGroupbyExpr* pGroupBy = (SGroupbyExpr*)pPlanNode->pExtInfo; + node->aggAlgo = AGG_ALGO_PLAIN; + node->aggSplit = AGG_SPLIT_FINAL; + if (NULL != pGroupBy) { + node->aggAlgo = AGG_ALGO_HASHED; + node->pGroupByList = validPointer(taosArrayDup(pGroupBy->columnInfo)); + } + return (SPhyNode*)node; +} + +static SPhyNode* createAggNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + // if (needMultiNodeAgg(pPlanNode)) { + + // } + return createSingleTableAgg(pCxt, pPlanNode); +} + static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SPhyNode* node = NULL; switch (pPlanNode->info.type) { @@ -311,6 +328,10 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; + case QNODE_AGGREGATE: + case QNODE_GROUPBY: + node = createAggNode(pCxt, pPlanNode); + break; case QNODE_MODIFY: // Insert is not an operator in a physical plan. break; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 7e36e0e124..123364134d 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -20,6 +20,19 @@ typedef bool (*FToJson)(const void* obj, cJSON* json); typedef bool (*FFromJson)(const cJSON* json, void* obj); +static char* getString(const cJSON* json, const char* name) { + char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); + return strdup(p); +} + +static void copyString(const cJSON* json, const char* name, char* dst) { + strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name))); +} + +static int64_t getNumber(const cJSON* json, const char* name) { + return cJSON_GetNumberValue(cJSON_GetObjectItem(json, name)); +} + static bool addObject(cJSON* json, const char* name, FToJson func, const void* obj) { if (NULL == obj) { return true; @@ -62,6 +75,39 @@ static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson f return func(jObj, *obj); } +static const char* jkPnodeType = "Type"; +static int32_t getPnodeTypeSize(cJSON* json) { + switch (getNumber(json, jkPnodeType)) { + case OP_TableScan: + case OP_DataBlocksOptScan: + case OP_TableSeqScan: + return sizeof(STableScanPhyNode); + case OP_TagScan: + return sizeof(STagScanPhyNode); + case OP_SystemTableScan: + return sizeof(SSystemTableScanPhyNode); + case OP_Aggregate: + return sizeof(SAggPhyNode); + case OP_Exchange: + return sizeof(SExchangePhyNode); + default: + break; + }; + return -1; +} + +static bool fromPnode(const cJSON* json, const char* name, FFromJson func, void** obj) { + cJSON* jObj = cJSON_GetObjectItem(json, name); + if (NULL == jObj) { + return true; + } + *obj = calloc(1, getPnodeTypeSize(jObj)); + if (NULL == *obj) { + return false; + } + return func(jObj, *obj); +} + static bool addTarray(cJSON* json, const char* name, FToJson func, const SArray* array, bool isPoint) { size_t size = (NULL == array) ? 0 : taosArrayGetSize(array); if (size > 0) { @@ -154,26 +200,9 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson return fromItem(jArray, func, *array, itemSize, *size); } -static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) { +static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) { const cJSON* jArray = getArray(json, name, size); - if (*array == NULL) { - *array = calloc(*size, itemSize); - } - - return fromItem(jArray, func, *array, itemSize, *size); -} - -static char* getString(const cJSON* json, const char* name) { - char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); - return strdup(p); -} - -static void copyString(const cJSON* json, const char* name, char* dst) { - strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name))); -} - -static int64_t getNumber(const cJSON* json, const char* name) { - return cJSON_GetNumberValue(cJSON_GetObjectItem(json, name)); + return fromItem(jArray, func, array, itemSize, *size); } static const char* jkSchemaType = "Type"; @@ -221,7 +250,7 @@ static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) { schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize); schema->precision = getNumber(json, jkDataBlockSchemaPrecision); - return fromRawArray(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**) &(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols); + return fromRawArrayWithAlloc(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**)&(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols); } static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr"; @@ -534,19 +563,15 @@ static const char* jkScanNodeTableCount = "Count"; static bool scanNodeToJson(const void* obj, cJSON* json) { const SScanPhyNode* scan = (const SScanPhyNode*)obj; bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, scan->uid); - if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableType, scan->tableType); } - if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableOrder, scan->order); } - if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableCount, scan->count); } - return res; } @@ -559,6 +584,66 @@ static bool scanNodeFromJson(const cJSON* json, void* obj) { return true; } +static const char* jkColIndexColId = "ColId"; +static const char* jkColIndexColIndex = "ColIndex"; +static const char* jkColIndexFlag = "Flag"; +static const char* jkColIndexName = "Name"; + +static bool colIndexToJson(const void* obj, cJSON* json) { + const SColIndex* col = (const SColIndex*)obj; + bool res = cJSON_AddNumberToObject(json, jkColIndexColId, col->colId); + if (res) { + res = cJSON_AddNumberToObject(json, jkColIndexColIndex, col->colIndex); + } + if (res) { + res = cJSON_AddNumberToObject(json, jkColIndexFlag, col->flag); + } + if (res) { + res = cJSON_AddStringToObject(json, jkColIndexName, col->name); + } + return res; +} + +static bool colIndexFromJson(const cJSON* json, void* obj) { + SColIndex* col = (SColIndex*)obj; + col->colId = getNumber(json, jkColIndexColId); + col->colIndex = getNumber(json, jkColIndexColIndex); + col->flag = getNumber(json, jkColIndexFlag); + copyString(json, jkColIndexName, col->name); + return true; +} + +static const char* jkAggNodeAggAlgo = "AggAlgo"; +static const char* jkAggNodeAggSplit = "AggSplit"; +static const char* jkAggNodeExprs = "Exprs"; +static const char* jkAggNodeGroupByList = "GroupByList"; + +static bool aggNodeToJson(const void* obj, cJSON* json) { + const SAggPhyNode* agg = (const SAggPhyNode*)obj; + bool res = cJSON_AddNumberToObject(json, jkAggNodeAggAlgo, agg->aggAlgo); + if (res) { + res = cJSON_AddNumberToObject(json, jkAggNodeAggSplit, agg->aggSplit); + } + if (res) { + res = addArray(json, jkAggNodeExprs, exprInfoToJson, agg->pExprs); + } + if (res) { + res = addArray(json, jkAggNodeGroupByList, colIndexToJson, agg->pGroupByList); + } + return res; +} + +static bool aggNodeFromJson(const cJSON* json, void* obj) { + SAggPhyNode* agg = (SAggPhyNode*)obj; + agg->aggAlgo = getNumber(json, jkAggNodeAggAlgo); + agg->aggSplit = getNumber(json, jkAggNodeAggSplit); + bool res = fromArray(json, jkAggNodeExprs, exprInfoFromJson, &agg->pExprs, sizeof(SExprInfo)); + if (res) { + res = fromArray(json, jkAggNodeGroupByList, colIndexFromJson, &agg->pGroupByList, sizeof(SExprInfo)); + } + return res; +} + static const char* jkTableScanNodeFlag = "Flag"; static const char* jkTableScanNodeWindow = "Window"; static const char* jkTableScanNodeTagsConditions = "TagsConditions"; @@ -667,10 +752,10 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) { case OP_SystemTableScan: return scanNodeToJson(obj, json); case OP_Aggregate: - break; // todo + return aggNodeToJson(obj, json); case OP_Project: return true; - case OP_Groupby: + // case OP_Groupby: case OP_Limit: case OP_SLimit: case OP_TimeWindow: @@ -708,7 +793,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { break; // todo case OP_Project: return true; - case OP_Groupby: + // case OP_Groupby: case OP_Limit: case OP_SLimit: case OP_TimeWindow: @@ -735,12 +820,15 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { static const char* jkPnodeName = "Name"; static const char* jkPnodeTargets = "Targets"; static const char* jkPnodeConditions = "Conditions"; -static const char* jkPnodeSchema = "InputSchema"; +static const char* jkPnodeSchema = "TargetSchema"; static const char* jkPnodeChildren = "Children"; // The 'pParent' field do not need to be serialized. static bool phyNodeToJson(const void* obj, cJSON* jNode) { const SPhyNode* phyNode = (const SPhyNode*)obj; - bool res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name); + bool res = cJSON_AddNumberToObject(jNode, jkPnodeType, phyNode->info.type); + if (res) { + res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name); + } if (res) { res = addArray(jNode, jkPnodeTargets, exprInfoToJson, phyNode->pTargets); } @@ -762,8 +850,8 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) { static bool phyNodeFromJson(const cJSON* json, void* obj) { SPhyNode* node = (SPhyNode*) obj; - node->info.name = getString(json, jkPnodeName); - node->info.type = opNameToOpType(node->info.name); + node->info.type = getNumber(json, jkPnodeType); + node->info.name = opTypeToOpName(node->info.type); bool res = fromArray(json, jkPnodeTargets, exprInfoFromJson, &node->pTargets, sizeof(SExprInfo)); if (res) { @@ -910,8 +998,7 @@ static SSubplan* subplanFromJson(const cJSON* json) { } bool res = fromObject(json, jkSubplanId, subplanIdFromJson, &subplan->id, true); if (res) { - size_t size = MAX(sizeof(SPhyNode), sizeof(SScanPhyNode)); - res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, size, false); + res = fromPnode(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode); } if (res) { res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), false); diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 29f6e48dc7..6d9e08e829 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -30,6 +30,21 @@ void* myCalloc(size_t nmemb, size_t size) { class PhyPlanTest : public Test { protected: + void pushAgg(int32_t aggOp) { + unique_ptr agg((SQueryPlanNode*)myCalloc(1, sizeof(SQueryPlanNode))); + agg->info.type = aggOp; + agg->pExpr = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + unique_ptr expr((SExprInfo*)myCalloc(1, sizeof(SExprInfo))); + expr->base.resSchema.type = TSDB_DATA_TYPE_INT; + expr->base.resSchema.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; + expr->pExpr = (tExprNode*)myCalloc(1, sizeof(tExprNode)); + expr->pExpr->nodeType = TEXPR_FUNCTION_NODE; + strcpy(expr->pExpr->_function.functionName, "Count"); + SExprInfo* item = expr.release(); + taosArrayPush(agg->pExpr, &item); + pushNode(agg.release()); + } + void pushScan(const string& db, const string& table, int32_t scanOp) { shared_ptr meta = mockCatalogService->getTableMeta(db, table); EXPECT_TRUE(meta); @@ -95,10 +110,11 @@ protected: private: void pushNode(SQueryPlanNode* node) { if (logicPlan_) { - // todo - } else { - logicPlan_.reset(node); + node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + SQueryPlanNode* child = logicPlan_.release(); + taosArrayPush(node->pChildren, &child); } + logicPlan_.reset(node); } void copySchemaMeta(STableMeta** dst, const STableMeta* src) { @@ -174,6 +190,16 @@ TEST_F(PhyPlanTest, superTableScanTest) { // todo check } +// select count(*) from table +TEST_F(PhyPlanTest, simpleAggTest) { + pushScan("test", "t1", QNODE_TABLESCAN); + pushAgg(QNODE_AGGREGATE); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + explain(); + SQueryDag* dag = result(); + // todo check +} + // insert into t values(...) TEST_F(PhyPlanTest, insertTest) { ASSERT_EQ(run("test", "insert into t1 values (now, 1, \"beijing\")"), TSDB_CODE_SUCCESS); From 552c151f494468827c03548342c46f6a934392d5 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 14 Jan 2022 05:05:54 -0500 Subject: [PATCH 11/12] TD-12678 dsGetDataBlock allowed to call again after the query is completed to return statistics --- source/libs/executor/src/dataDispatcher.c | 6 ++++++ source/libs/planner/src/planner.c | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index a69084f3db..a6af9ff388 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -182,6 +182,12 @@ static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryE static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; + if (NULL == pDispatcher->nextOutput.pData) { + assert(pDispatcher->queryEnd); + pOutput->useconds = pDispatcher->useconds; + pOutput->precision = pDispatcher->schema.precision; + return TSDB_CODE_SUCCESS; + } SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 21f57d95d5..bf815b26b2 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { -// char* str = NULL; -// queryPlanToString(pLogicPlan, &str); -// printf("%s\n", str); + char* str = NULL; + queryPlanToString(pLogicPlan, &str); + printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); From a3f70fafa44fff572b249f737375880604cd7518 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 14 Jan 2022 21:36:15 -0500 Subject: [PATCH 12/12] TD-12678 SOutputData adjust --- include/libs/executor/dataSinkMgt.h | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- source/libs/qworker/src/qworker.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index f13ba5f87e..5cef3b2253 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -48,7 +48,7 @@ typedef struct SOutputData { int8_t compressed; char* pData; bool queryEnd; - bool needSchedule; + int32_t scheduleJobNo; int32_t bufStatus; int64_t useconds; int8_t precision; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index a6af9ff388..8280f9d0af 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -196,7 +196,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->bufStatus = updateStatus(pDispatcher); pthread_mutex_lock(&pDispatcher->mutex); pOutput->queryEnd = pDispatcher->queryEnd; - pOutput->needSchedule = false; + pOutput->scheduleJobNo = 0; pOutput->useconds = pDispatcher->useconds; pOutput->precision = pDispatcher->schema.precision; pthread_mutex_unlock(&pDispatcher->mutex); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 2a395fcfe1..04a327e7b9 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -976,7 +976,7 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 rsp->completed = 1; } - if (output.needSchedule) { + if (output.scheduleJobNo) { //TODO }