From cb5c12f1c60a507769589ef4a969329782f0961f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 16:55:31 +0800 Subject: [PATCH 01/14] refact test --- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 51 ++++++------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index f0bca3aa2e..49f3ff0160 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -14,7 +14,7 @@ #include "vnode.h" -static STSchema *createBasicSchema() { +static STSchema *vtCreateBasicSchema() { STSchemaBuilder sb; STSchema * pSchema = NULL; @@ -32,7 +32,7 @@ static STSchema *createBasicSchema() { return pSchema; } -static STSchema *createBasicTagSchema() { +static STSchema *vtCreateBasicTagSchema() { STSchemaBuilder sb; STSchema * pSchema = NULL; @@ -50,7 +50,7 @@ static STSchema *createBasicTagSchema() { return pSchema; } -static SKVRow createBasicTag() { +static SKVRow vtCreateBasicTag() { SKVRowBuilder rb; SKVRow pTag; @@ -71,27 +71,7 @@ static SKVRow createBasicTag() { return pTag; } -#if 0 -TEST(vnodeApiTest, test_create_table_encode_and_decode_function) { - tb_uid_t suid = 1638166374163; - STSchema *pSchema = createBasicSchema(); - STSchema *pTagSchema = createBasicTagSchema(); - char tbname[128] = "st"; - char * buffer = new char[1024]; - void * pBuf = (void *)buffer; - SVnodeReq vCreateSTbReq = VNODE_INIT_CREATE_STB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); - - vnodeBuildReq(&pBuf, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - - SVnodeReq decoded_req; - - vnodeParseReq(buffer, &decoded_req, TSDB_MSG_TYPE_CREATE_TABLE); - - int k = 10; -} -#endif - -TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { +TEST(vnodeApiTest, vnode_simple_create_table_test) { vnodeDestroy("vnode1"); GTEST_ASSERT_GE(vnodeInit(2), 0); @@ -103,8 +83,8 @@ TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { tb_uid_t suid = 1638166374163; { // Create a super table - STSchema *pSchema = createBasicSchema(); - STSchema *pTagSchema = createBasicTagSchema(); + STSchema *pSchema = vtCreateBasicSchema(); + STSchema *pTagSchema = vtCreateBasicTagSchema(); char tbname[128] = "st"; SArray * pMsgs = (SArray *)taosArrayInit(1, sizeof(SRpcMsg *)); @@ -138,7 +118,7 @@ TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { for (int i = 0; i < ntables / batch; i++) { SArray *pMsgs = (SArray *)taosArrayInit(batch, sizeof(SRpcMsg *)); for (int j = 0; j < batch; j++) { - SKVRow pTag = createBasicTag(); + SKVRow pTag = vtCreateBasicTag(); char tbname[128]; sprintf(tbname, "tb%d", i * batch + j); SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); @@ -176,13 +156,14 @@ TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { vnodeClear(); } -TEST(vnodeApiTest, DISABLED_vnode_process_create_table) { - STSchema * pSchema = NULL; - STSchema * pTagSchema = NULL; - char stname[15]; - SVCreateTableReq pReq = META_INIT_STB_CFG(stname, UINT32_MAX, UINT32_MAX, 0, pSchema, pTagSchema); +TEST(vnodeApiTest, vnode_simple_insert_test) { + const char *vname = "vnode2"; + vnodeDestroy(vname); - int k = 10; + GTEST_ASSERT_GE(vnodeInit(2), 0); - META_CLEAR_TB_CFG(pReq); -} + SVnode *pVnode = vnodeOpen(vname, NULL); + + vnodeClose(pVnode); + vnodeClear(); +} \ No newline at end of file From 2ad2efdfa37c5dc90465b3af2aff7bb9c7875d4e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 17:52:10 +0800 Subject: [PATCH 02/14] more --- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 170 ++++++++++-------- 1 file changed, 98 insertions(+), 72 deletions(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 49f3ff0160..bbb1ef5a8b 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -71,89 +71,115 @@ static SKVRow vtCreateBasicTag() { return pTag; } -TEST(vnodeApiTest, vnode_simple_create_table_test) { - vnodeDestroy("vnode1"); +static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { + SRpcMsg * pMsg; + STSchema *pSchema; + STSchema *pTagSchema; + int zs; + void * pBuf; + pSchema = vtCreateBasicSchema(); + pTagSchema = vtCreateBasicTagSchema(); + + SVnodeReq vCreateSTbReq = VNODE_INIT_CREATE_STB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); + + zs = vnodeBuildReq(NULL, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); + pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + pMsg->contLen = zs; + pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg)); + + pBuf = pMsg->pCont; + vnodeBuildReq(&pBuf, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + META_CLEAR_TB_CFG(&vCreateSTbReq); + + tdFreeSchema(pSchema); + tdFreeSchema(pTagSchema); + + *ppMsg = pMsg; +} + +static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { + SRpcMsg *pMsg; + int tz; + SKVRow pTag; + + pTag = vtCreateBasicTag(); + SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); + + tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); + pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + pMsg->contLen = tz; + pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); + void *pBuf = pMsg->pCont; + + vnodeBuildReq(&pBuf, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + META_CLEAR_TB_CFG(&vCreateCTbReq); + free(pTag); + + *ppMsg = pMsg; +} + +static void vtClearMsgBatch(SArray *pMsgArr) { + SRpcMsg *pMsg; + for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { + pMsg = *(SRpcMsg **)taosArrayGet(pMsgArr, i); + free(pMsg); + } + + taosArrayClear(pMsgArr); +} + +TEST(vnodeApiTest, vnode_simple_create_table_test) { + tb_uid_t suid = 1638166374163; + SRpcMsg *pMsg; + SArray * pMsgArr = NULL; + SVnode * pVnode; + int rcode; + int ntables = 1000000; + int batch = 10; + char tbname[128]; + + pMsgArr = (SArray *)taosArrayInit(batch, sizeof(pMsg)); + + vnodeDestroy("vnode1"); GTEST_ASSERT_GE(vnodeInit(2), 0); - // Create and open a vnode - SVnode *pVnode = vnodeOpen("vnode1", NULL); + // CREATE AND OPEN A VNODE + pVnode = vnodeOpen("vnode1", NULL); ASSERT_NE(pVnode, nullptr); - tb_uid_t suid = 1638166374163; - { - // Create a super table - STSchema *pSchema = vtCreateBasicSchema(); - STSchema *pTagSchema = vtCreateBasicTagSchema(); - char tbname[128] = "st"; + // CREATE A SUPER TABLE + sprintf(tbname, "st"); + vtBuildCreateStbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); - SArray * pMsgs = (SArray *)taosArrayInit(1, sizeof(SRpcMsg *)); - SVnodeReq vCreateSTbReq = VNODE_INIT_CREATE_STB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); - - int zs = vnodeBuildReq(NULL, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); - pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; - pMsg->contLen = zs; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg)); - - void *pBuf = pMsg->pCont; - - vnodeBuildReq(&pBuf, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - META_CLEAR_TB_CFG(&vCreateSTbReq); - - taosArrayPush(pMsgs, &(pMsg)); - - vnodeProcessWMsgs(pVnode, pMsgs); - - free(pMsg); - taosArrayDestroy(pMsgs); - tdFreeSchema(pSchema); - tdFreeSchema(pTagSchema); - } - - { - // Create some child tables - int ntables = 1000000; - int batch = 10; - for (int i = 0; i < ntables / batch; i++) { - SArray *pMsgs = (SArray *)taosArrayInit(batch, sizeof(SRpcMsg *)); - for (int j = 0; j < batch; j++) { - SKVRow pTag = vtCreateBasicTag(); - char tbname[128]; - sprintf(tbname, "tb%d", i * batch + j); - SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); - - int tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); - pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; - pMsg->contLen = tz; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); - void *pBuf = pMsg->pCont; - - vnodeBuildReq(&pBuf, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - META_CLEAR_TB_CFG(&vCreateCTbReq); - free(pTag); - - taosArrayPush(pMsgs, &(pMsg)); - } - - vnodeProcessWMsgs(pVnode, pMsgs); - - for (int j = 0; j < batch; j++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayPop(pMsgs); - free(pMsg); - } - - taosArrayDestroy(pMsgs); - - // std::cout << "the " << i << "th batch is created" << std::endl; + // CREATE A LOT OF CHILD TABLES + for (int i = 0; i < ntables / batch; i++) { + // Build request batch + for (int j = 0; j < batch; j++) { + sprintf(tbname, "ct%d", i * batch + j + 1); + vtBuildCreateCtbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); } + + // Process request batch + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + ASSERT_EQ(rcode, 0); + + // Clear request batch + vtClearMsgBatch(pMsgArr); } - // Close the vnode + // CLOSE THE VNODE vnodeClose(pVnode); - vnodeClear(); + + taosArrayDestroy(pMsgArr); } TEST(vnodeApiTest, vnode_simple_insert_test) { From 13fedf14e1de032f54448bfa8f0d602ee5cf417f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 17:57:51 +0800 Subject: [PATCH 03/14] more --- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index bbb1ef5a8b..e32937b505 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -121,6 +121,14 @@ static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { *ppMsg = pMsg; } +static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) { + // TODO +} + +static void vtBuildSubmitReq(SRpcMsg **ppMsg) { + // TODO +} + static void vtClearMsgBatch(SArray *pMsgArr) { SRpcMsg *pMsg; for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { From 1db84c33aa8a836b30522f751e15b4e79557ed69 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 18:03:02 +0800 Subject: [PATCH 04/14] more --- source/dnode/vnode/impl/test/vBenchmarkTest.cpp | 1 + 1 file changed, 1 insertion(+) create mode 100644 source/dnode/vnode/impl/test/vBenchmarkTest.cpp diff --git a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp new file mode 100644 index 0000000000..19a033d237 --- /dev/null +++ b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp @@ -0,0 +1 @@ +// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest \ No newline at end of file From b383ae5d02bca43113409332e2fc7a6749cfc269 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 19:24:07 +0800 Subject: [PATCH 05/14] try to integrate benchmark test --- source/dnode/vnode/impl/test/vBenchmarkTest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp index 19a033d237..e218886231 100644 --- a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp +++ b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp @@ -1 +1,2 @@ -// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest \ No newline at end of file +// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest +// https://github.com/google/benchmark \ No newline at end of file From 684f1d11e2215e93ff9e5c6454c30770f42938bf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 10:39:24 +0800 Subject: [PATCH 06/14] fix coredump --- source/dnode/vnode/impl/CMakeLists.txt | 2 +- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 36 +++++++++++++++++-- source/dnode/vnode/tq/src/tq.c | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 3623516624..6972605afd 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -19,5 +19,5 @@ target_link_libraries( # test if(${BUILD_TEST}) - #add_subdirectory(test) + add_subdirectory(test) endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index e32937b505..a876a25fba 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -139,7 +139,7 @@ static void vtClearMsgBatch(SArray *pMsgArr) { taosArrayClear(pMsgArr); } -TEST(vnodeApiTest, vnode_simple_create_table_test) { +TEST(vnodeApiTest, DISABLED_vnode_simple_create_table_test) { tb_uid_t suid = 1638166374163; SRpcMsg *pMsg; SArray * pMsgArr = NULL; @@ -192,12 +192,44 @@ TEST(vnodeApiTest, vnode_simple_create_table_test) { TEST(vnodeApiTest, vnode_simple_insert_test) { const char *vname = "vnode2"; + char tbname[128]; + tb_uid_t suid = 1638166374163; + SRpcMsg * pMsg; + SArray * pMsgArr; + int rcode; + SVnode * pVnode; + + pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg)); + vnodeDestroy(vname); GTEST_ASSERT_GE(vnodeInit(2), 0); - SVnode *pVnode = vnodeOpen(vname, NULL); + // Open a vnode + pVnode = vnodeOpen(vname, NULL); + GTEST_ASSERT_NE(pVnode, nullptr); + // 1. CREATE A SUPER TABLE + sprintf(tbname, "st"); + vtBuildCreateStbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); + + // 2. CREATE A CHILD TABLE + sprintf(tbname, "t0"); + vtBuildCreateCtbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); + + // 3. WRITE A LOT OF TIME-SERIES DATA + + // Close the vnode vnodeClose(pVnode); vnodeClear(); + + taosArrayDestroy(pMsgArr); } \ No newline at end of file diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 5888141c58..249ffd7ae3 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -50,7 +50,7 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAl pTq->tqConfig = tqConfig; pTq->tqLogReader = tqLogReader; pTq->tqMemRef.pAlloctorFactory = allocFac; - pTq->tqMemRef.pAllocator = allocFac->create(allocFac); + // pTq->tqMemRef.pAllocator = allocFac->create(allocFac); if(pTq->tqMemRef.pAllocator == NULL) { //TODO } From ec004e5d59dca2de9dfdd817fdbab845330e7319 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 10:55:21 +0800 Subject: [PATCH 07/14] more --- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index a876a25fba..0f330d7b4c 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -139,7 +139,7 @@ static void vtClearMsgBatch(SArray *pMsgArr) { taosArrayClear(pMsgArr); } -TEST(vnodeApiTest, DISABLED_vnode_simple_create_table_test) { +TEST(vnodeApiTest, vnode_simple_create_table_test) { tb_uid_t suid = 1638166374163; SRpcMsg *pMsg; SArray * pMsgArr = NULL; From 5dc96f9de2d6e4006f524376ec0bcbb8dda9ad81 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 11:36:34 +0800 Subject: [PATCH 08/14] more --- source/dnode/vnode/impl/src/vnodeRequest.c | 3 ++- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 22 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeRequest.c b/source/dnode/vnode/impl/src/vnodeRequest.c index be5f5c890c..249bde4e56 100644 --- a/source/dnode/vnode/impl/src/vnodeRequest.c +++ b/source/dnode/vnode/impl/src/vnodeRequest.c @@ -25,9 +25,10 @@ int vnodeBuildReq(void **buf, const SVnodeReq *pReq, uint8_t type) { switch (type) { case TSDB_MSG_TYPE_CREATE_TABLE: tsize += vnodeBuildCreateTableReq(buf, &(pReq->ctReq)); + break; + case TSDB_MSG_TYPE_SUBMIT: /* code */ break; - default: break; } diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 0f330d7b4c..8f33dbba85 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -126,7 +126,19 @@ static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) { } static void vtBuildSubmitReq(SRpcMsg **ppMsg) { - // TODO + SRpcMsg *pMsg; + void * pBuf; + int tz = 0; + + pMsg = (SRpcMsg *)malloc(sizeof(*pMsg) + tz); + pMsg->msgType = TSDB_MSG_TYPE_SUBMIT; + pMsg->contLen = tz; + pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); + + pBuf = pMsg->pCont; + vnodeBuildReq(&pBuf, NULL /*TODO*/, TSDB_MSG_TYPE_SUBMIT); + + *ppMsg = pMsg; } static void vtClearMsgBatch(SArray *pMsgArr) { @@ -198,6 +210,7 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { SArray * pMsgArr; int rcode; SVnode * pVnode; + int batch = 1; pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg)); @@ -226,6 +239,13 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { vtClearMsgBatch(pMsgArr); // 3. WRITE A LOT OF TIME-SERIES DATA + for (int i = 0; i < batch; i++) { + vtBuildSubmitReq(&pMsg); + taosArrayPush(pMsgArr, &pMsg); + } + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); // Close the vnode vnodeClose(pVnode); From 1ab4f2c443fb7a94518699489e0ad16429e09651 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 14:39:20 +0800 Subject: [PATCH 09/14] solve memory problem --- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 8f33dbba85..4a60fbe3df 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -102,9 +102,8 @@ static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { SRpcMsg *pMsg; int tz; - SKVRow pTag; + SKVRow pTag = vtCreateBasicTag(); - pTag = vtCreateBasicTag(); SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); @@ -126,17 +125,33 @@ static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) { } static void vtBuildSubmitReq(SRpcMsg **ppMsg) { - SRpcMsg *pMsg; - void * pBuf; - int tz = 0; + SRpcMsg * pMsg; + SSubmitMsg *pSubmitMsg; + SSubmitBlk *pSubmitBlk; + int tz = 1024; // TODO pMsg = (SRpcMsg *)malloc(sizeof(*pMsg) + tz); pMsg->msgType = TSDB_MSG_TYPE_SUBMIT; pMsg->contLen = tz; pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); - pBuf = pMsg->pCont; - vnodeBuildReq(&pBuf, NULL /*TODO*/, TSDB_MSG_TYPE_SUBMIT); + // For submit msg header + pSubmitMsg = (SSubmitMsg *)(pMsg->pCont); + // pSubmitMsg->header.contLen = 0; + // pSubmitMsg->header.vgId = 0; + // pSubmitMsg->length = 0; + pSubmitMsg->numOfBlocks = 1; + + // For submit blk + pSubmitBlk = (SSubmitBlk *)(pSubmitMsg->blocks); + pSubmitBlk->uid = 0; + pSubmitBlk->tid = 0; + pSubmitBlk->padding = 0; + pSubmitBlk->sversion = 0; + pSubmitBlk->dataLen = 0; + pSubmitBlk->numOfRows = 0; + + // For row batch *ppMsg = pMsg; } From d0d56645a9b9bb0aefa00208453f6c7f34f46433 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 15:24:33 +0800 Subject: [PATCH 10/14] more --- include/dnode/vnode/meta/meta.h | 6 ++-- include/dnode/vnode/tq/tq.h | 2 +- include/dnode/vnode/tsdb/tsdb.h | 4 ++- source/dnode/vnode/impl/inc/vnodeBufferPool.h | 2 ++ source/dnode/vnode/impl/inc/vnodeMAF.h | 32 +++++++++++++++++++ source/dnode/vnode/impl/src/vnodeBufferPool.c | 6 ++-- source/dnode/vnode/impl/src/vnodeMain.c | 8 +++-- source/dnode/vnode/meta/src/metaMain.c | 16 +++++----- source/dnode/vnode/tq/src/tq.c | 4 +++ source/dnode/vnode/tsdb/src/tsdbMain.c | 16 +++++----- 10 files changed, 70 insertions(+), 26 deletions(-) create mode 100644 source/dnode/vnode/impl/inc/vnodeMAF.h diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index 44ae1bb79f..b5994a50e1 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -71,7 +71,7 @@ typedef struct STbCfg { } STbCfg; // SMeta operations -SMeta *metaOpen(const char *path, const SMetaCfg *pOptions); +SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg); void metaClose(SMeta *pMeta); void metaRemove(const char *path); int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg); @@ -79,8 +79,8 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); // Options -void metaOptionsInit(SMetaCfg *pOptions); -void metaOptionsClear(SMetaCfg *pOptions); +void metaOptionsInit(SMetaCfg *pMetaCfg); +void metaOptionsClear(SMetaCfg *pMetaCfg); // STbCfg #define META_INIT_STB_CFG(NAME, TTL, KEEP, SUID, PSCHEMA, PTAGSCHEMA) \ diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 747d97b7a0..074678f248 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -264,7 +264,7 @@ typedef struct STQ { // open in each vnode STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac); -void tqDestroy(STQ*); +void tqClose(STQ*); // void* will be replace by a msg type int tqPushMsg(STQ*, void* msg, int64_t version); diff --git a/include/dnode/vnode/tsdb/tsdb.h b/include/dnode/vnode/tsdb/tsdb.h index f8eac9768f..b85c6b64f6 100644 --- a/include/dnode/vnode/tsdb/tsdb.h +++ b/include/dnode/vnode/tsdb/tsdb.h @@ -16,6 +16,8 @@ #ifndef _TD_TSDB_H_ #define _TD_TSDB_H_ +#include "mallocator.h" + #ifdef __cplusplus extern "C" { #endif @@ -25,7 +27,7 @@ typedef struct STsdb STsdb; typedef struct STsdbCfg STsdbCfg; // STsdb -STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg); +STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF); void tsdbClose(STsdb *); void tsdbRemove(const char *path); int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg); diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/impl/inc/vnodeBufferPool.h index d96671d2bd..b4535597ee 100644 --- a/source/dnode/vnode/impl/inc/vnodeBufferPool.h +++ b/source/dnode/vnode/impl/inc/vnodeBufferPool.h @@ -32,6 +32,8 @@ int vnodeBufPoolRecycle(SVnode *pVnode); void *vnodeMalloc(SVnode *pVnode, uint64_t size); bool vnodeBufPoolIsFull(SVnode *pVnode); +SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/impl/inc/vnodeMAF.h b/source/dnode/vnode/impl/inc/vnodeMAF.h new file mode 100644 index 0000000000..7aa405103c --- /dev/null +++ b/source/dnode/vnode/impl/inc/vnodeMAF.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_MAF_H_ +#define _TD_VNODE_MAF_H_ + +#include "vnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int vnodeOpenMAF(SVnode *pVnode); +void vnodeCloseMAF(SVnode *pVnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_MAF_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 152a346f0a..49e7889517 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -24,8 +24,8 @@ struct SVBufPool { TD_DLIST(SVMemAllocator) free; TD_DLIST(SVMemAllocator) incycle; SVMemAllocator *inuse; - // MAF for submodules - // SMemAllocatorFactory maf; + // MAF for submodules to use + SMemAllocatorFactory *pMAF; }; int vnodeOpenBufPool(SVnode *pVnode) { @@ -125,6 +125,8 @@ bool vnodeBufPoolIsFull(SVnode *pVnode) { return vmaIsFull(pVnode->pBufPool->inuse); } +SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode) { return pVnode->pBufPool->pMAF; } + #if 0 typedef enum { diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index 59e3bae5d7..c1539659ae 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -102,7 +102,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open tsdb sprintf(dir, "%s/tsdb", pVnode->path); - pVnode->pTsdb = tsdbOpen(dir, &(pVnode->config.tsdbCfg)); + pVnode->pTsdb = tsdbOpen(dir, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pTsdb == NULL) { // TODO: handle error return -1; @@ -110,7 +110,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // TODO: Open TQ sprintf(dir, "%s/tq", pVnode->path); - pVnode->pTq = tqOpen(dir, &(pVnode->config.tqCfg), NULL, NULL); + pVnode->pTq = tqOpen(dir, &(pVnode->config.tqCfg), NULL, vBufPoolGetMAF(pVnode)); if (pVnode->pTq == NULL) { // TODO: handle error return -1; @@ -131,7 +131,9 @@ static int vnodeOpenImpl(SVnode *pVnode) { static void vnodeCloseImpl(SVnode *pVnode) { if (pVnode) { vnodeCloseBufPool(pVnode); - tsdbClose(pVnode->pTsdb); metaClose(pVnode->pMeta); + tsdbClose(pVnode->pTsdb); + tqClose(pVnode->pTq); + walClose(pVnode->pWal); } } \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaMain.c b/source/dnode/vnode/meta/src/metaMain.c index 53055dcea3..f2602fbad7 100644 --- a/source/dnode/vnode/meta/src/metaMain.c +++ b/source/dnode/vnode/meta/src/metaMain.c @@ -17,27 +17,27 @@ #include "metaDef.h" -static SMeta *metaNew(const char *path, const SMetaCfg *pMetaOptions); +static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg); static void metaFree(SMeta *pMeta); static int metaOpenImpl(SMeta *pMeta); static void metaCloseImpl(SMeta *pMeta); -SMeta *metaOpen(const char *path, const SMetaCfg *pMetaOptions) { +SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg) { SMeta *pMeta = NULL; // Set default options - if (pMetaOptions == NULL) { - pMetaOptions = &defaultMetaOptions; + if (pMetaCfg == NULL) { + pMetaCfg = &defaultMetaOptions; } // Validate the options - if (metaValidateOptions(pMetaOptions) < 0) { + if (metaValidateOptions(pMetaCfg) < 0) { // TODO: deal with error return NULL; } // Allocate handle - pMeta = metaNew(path, pMetaOptions); + pMeta = metaNew(path, pMetaCfg); if (pMeta == NULL) { // TODO: handle error return NULL; @@ -65,7 +65,7 @@ void metaClose(SMeta *pMeta) { void metaRemove(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static SMeta *metaNew(const char *path, const SMetaCfg *pMetaOptions) { +static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg) { SMeta *pMeta; size_t psize = strlen(path); @@ -80,7 +80,7 @@ static SMeta *metaNew(const char *path, const SMetaCfg *pMetaOptions) { return NULL; } - metaOptionsCopy(&(pMeta->options), pMetaOptions); + metaOptionsCopy(&(pMeta->options), pMetaCfg); return pMeta; }; diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 249ffd7ae3..4aabf07c2d 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -66,6 +66,10 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAl return pTq; } +void tqClose(STQ*pTq) { + // TODO +} + static int tqProtoCheck(TmqMsgHead *pMsg) { return pMsg->protoVer == 0; } diff --git a/source/dnode/vnode/tsdb/src/tsdbMain.c b/source/dnode/vnode/tsdb/src/tsdbMain.c index 2fe7a61930..20b75aa78f 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMain.c +++ b/source/dnode/vnode/tsdb/src/tsdbMain.c @@ -15,27 +15,27 @@ #include "tsdbDef.h" -static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbOptions); +static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg); static void tsdbFree(STsdb *pTsdb); static int tsdbOpenImpl(STsdb *pTsdb); static void tsdbCloseImpl(STsdb *pTsdb); -STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbOptions) { +STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) { STsdb *pTsdb = NULL; // Set default TSDB Options - if (pTsdbOptions == NULL) { - pTsdbOptions = &defautlTsdbOptions; + if (pTsdbCfg == NULL) { + pTsdbCfg = &defautlTsdbOptions; } // Validate the options - if (tsdbValidateOptions(pTsdbOptions) < 0) { + if (tsdbValidateOptions(pTsdbCfg) < 0) { // TODO: handle error return NULL; } // Create the handle - pTsdb = tsdbNew(path, pTsdbOptions); + pTsdb = tsdbNew(path, pTsdbCfg); if (pTsdb == NULL) { // TODO: handle error return NULL; @@ -62,7 +62,7 @@ void tsdbClose(STsdb *pTsdb) { void tsdbRemove(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbOptions) { +static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg) { STsdb *pTsdb = NULL; pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); @@ -72,7 +72,7 @@ static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbOptions) { } pTsdb->path = strdup(path); - tsdbOptionsCopy(&(pTsdb->options), pTsdbOptions); + tsdbOptionsCopy(&(pTsdb->options), pTsdbCfg); return pTsdb; } From 835c3030c591749c46da0dd4a7c03e07dce1c68e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 15:28:53 +0800 Subject: [PATCH 11/14] more --- include/dnode/vnode/meta/meta.h | 3 ++- source/dnode/vnode/impl/src/vnodeMain.c | 2 +- source/dnode/vnode/meta/inc/metaDef.h | 2 +- source/dnode/vnode/meta/src/metaMain.c | 9 +++++---- source/dnode/vnode/tsdb/src/tsdbMain.c | 7 ++++--- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index b5994a50e1..113a970548 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -16,6 +16,7 @@ #ifndef _TD_META_H_ #define _TD_META_H_ +#include "mallocator.h" #include "os.h" #include "trow.h" @@ -71,7 +72,7 @@ typedef struct STbCfg { } STbCfg; // SMeta operations -SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg); +SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); void metaClose(SMeta *pMeta); void metaRemove(const char *path); int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg); diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index c1539659ae..70d9c7d4b0 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -94,7 +94,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open meta sprintf(dir, "%s/meta", pVnode->path); - pVnode->pMeta = metaOpen(dir, &(pVnode->config.metaCfg)); + pVnode->pMeta = metaOpen(dir, &(pVnode->config.metaCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pMeta == NULL) { // TODO: handle error return -1; diff --git a/source/dnode/vnode/meta/inc/metaDef.h b/source/dnode/vnode/meta/inc/metaDef.h index 0204031e76..e1c15af5aa 100644 --- a/source/dnode/vnode/meta/inc/metaDef.h +++ b/source/dnode/vnode/meta/inc/metaDef.h @@ -34,7 +34,7 @@ extern "C" { struct SMeta { char* path; SMetaCfg options; - SMetaDB* pDB; + SMetaDB* pDB; SMetaIdx* pIdx; SMetaCache* pCache; STbUidGenerator uidGnrt; diff --git a/source/dnode/vnode/meta/src/metaMain.c b/source/dnode/vnode/meta/src/metaMain.c index f2602fbad7..b6aa029b19 100644 --- a/source/dnode/vnode/meta/src/metaMain.c +++ b/source/dnode/vnode/meta/src/metaMain.c @@ -17,12 +17,12 @@ #include "metaDef.h" -static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg); +static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); static void metaFree(SMeta *pMeta); static int metaOpenImpl(SMeta *pMeta); static void metaCloseImpl(SMeta *pMeta); -SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg) { +SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF) { SMeta *pMeta = NULL; // Set default options @@ -37,7 +37,7 @@ SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg) { } // Allocate handle - pMeta = metaNew(path, pMetaCfg); + pMeta = metaNew(path, pMetaCfg, pMAF); if (pMeta == NULL) { // TODO: handle error return NULL; @@ -65,7 +65,7 @@ void metaClose(SMeta *pMeta) { void metaRemove(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg) { +static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF) { SMeta *pMeta; size_t psize = strlen(path); @@ -81,6 +81,7 @@ static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg) { } metaOptionsCopy(&(pMeta->options), pMetaCfg); + pMeta->pmaf = pMAF; return pMeta; }; diff --git a/source/dnode/vnode/tsdb/src/tsdbMain.c b/source/dnode/vnode/tsdb/src/tsdbMain.c index 20b75aa78f..d67d45660d 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMain.c +++ b/source/dnode/vnode/tsdb/src/tsdbMain.c @@ -15,7 +15,7 @@ #include "tsdbDef.h" -static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg); +static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF); static void tsdbFree(STsdb *pTsdb); static int tsdbOpenImpl(STsdb *pTsdb); static void tsdbCloseImpl(STsdb *pTsdb); @@ -35,7 +35,7 @@ STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory } // Create the handle - pTsdb = tsdbNew(path, pTsdbCfg); + pTsdb = tsdbNew(path, pTsdbCfg, pMAF); if (pTsdb == NULL) { // TODO: handle error return NULL; @@ -62,7 +62,7 @@ void tsdbClose(STsdb *pTsdb) { void tsdbRemove(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg) { +static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) { STsdb *pTsdb = NULL; pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); @@ -73,6 +73,7 @@ static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg) { pTsdb->path = strdup(path); tsdbOptionsCopy(&(pTsdb->options), pTsdbCfg); + pTsdb->pmaf = pMAF; return pTsdb; } From d1e0a9a780538f399ce00211900cd82f372b1c57 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 16:20:46 +0800 Subject: [PATCH 12/14] make test run --- source/dnode/vnode/impl/src/vnodeBufferPool.c | 223 ++++-------------- 1 file changed, 46 insertions(+), 177 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 49e7889517..6c1ededfc9 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -28,6 +28,9 @@ struct SVBufPool { SMemAllocatorFactory *pMAF; }; +static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF); +static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA); + int vnodeOpenBufPool(SVnode *pVnode) { uint64_t capacity; @@ -54,6 +57,15 @@ int vnodeOpenBufPool(SVnode *pVnode) { tDListAppend(&(pVnode->pBufPool->free), pVMA); } + pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)malloc(sizeof(SMemAllocatorFactory)); + if (pVnode->pBufPool->pMAF == NULL) { + // TODO: handle error + return -1; + } + pVnode->pBufPool->pMAF->impl = pVnode; + pVnode->pBufPool->pMAF->create = vBufPoolCreateMA; + pVnode->pBufPool->pMAF->destroy = vBufPoolDestroyMA; + return 0; } @@ -127,195 +139,52 @@ bool vnodeBufPoolIsFull(SVnode *pVnode) { SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode) { return pVnode->pBufPool->pMAF; } -#if 0 - -typedef enum { - // Heap allocator - E_V_HEAP_ALLOCATOR = 0, - // Arena allocator - E_V_ARENA_ALLOCATOR -} EVMemAllocatorT; - +/* ------------------------ STATIC METHODS ------------------------ */ typedef struct { - /* TODO */ -} SVHeapAllocator; - -typedef struct SVArenaNode { - struct SVArenaNode *prev; - uint64_t size; - void * ptr; - char data[]; -} SVArenaNode; - -typedef struct { - uint64_t ssize; // step size - uint64_t lsize; // limit size - SVArenaNode *inuse; - SVArenaNode node; -} SVArenaAllocator; - -typedef struct { - SVnode * pVnode; - SListNode *pNode; + SVnode * pVnode; + SVMemAllocator *pVMA; } SVMAWrapper; +static FORCE_INLINE void *vmaMaloocCb(SMemAllocator *pMA, uint64_t size) { + SVMAWrapper *pWrapper = (SVMAWrapper *)(pMA->impl); -static SListNode * vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); -static void vBufPoolFreeNode(SListNode *pNode); -static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf); -static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); -static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); + return vmaMalloc(pWrapper->pVMA, size); +} -/* ------------------------ STATIC METHODS ------------------------ */ -static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { - SListNode * pNode; - SVMemAllocator *pvma; - uint64_t msize; - uint64_t ssize = 4096; // TODO - uint64_t lsize = 1024; // TODO +// TODO: Add atomic operations here +static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF) { + SMemAllocator *pMA; + SVnode * pVnode = (SVnode *)(pMAF->impl); + SVMAWrapper * pWrapper; - msize = sizeof(SListNode) + sizeof(SVMemAllocator); - if (type == E_V_ARENA_ALLOCATOR) { - msize += capacity; - } - - pNode = (SListNode *)calloc(1, msize); - if (pNode == NULL) { - // TODO: handle error + pMA = (SMemAllocator *)calloc(1, sizeof(*pMA) + sizeof(SVMAWrapper)); + if (pMA == NULL) { return NULL; } - pvma = (SVMemAllocator *)(pNode->data); - pvma->capacity = capacity; - pvma->type = type; + pVnode->pBufPool->inuse->_ref.val++; + pWrapper = POINTER_SHIFT(pMA, sizeof(*pMA)); + pWrapper->pVnode = pVnode; + pWrapper->pVMA = pVnode->pBufPool->inuse; - switch (type) { - case E_V_ARENA_ALLOCATOR: - vArenaAllocatorInit(&(pvma->vaa), capacity, ssize, lsize); - break; - case E_V_HEAP_ALLOCATOR: - // vHeapAllocatorInit(&(pvma->vha)); - break; - default: - ASSERT(0); - } + pMA->impl = pWrapper; + pMA->malloc = vmaMaloocCb; + pMA->calloc = NULL; + pMA->realloc = NULL; + pMA->free = NULL; + pMA->usage = NULL; - return pNode; + return pMA; } -static void vBufPoolFreeNode(SListNode *pNode) { - SVMemAllocator *pvma = (SVMemAllocator *)(pNode->data); +static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA) { + SVMAWrapper * pWrapper = (SVMAWrapper *)(pMA->impl); + SVnode * pVnode = pWrapper->pVnode; + SVMemAllocator *pVMA = pWrapper->pVMA; - switch (pvma->type) { - case E_V_ARENA_ALLOCATOR: - vArenaAllocatorClear(&(pvma->vaa)); - break; - case E_V_HEAP_ALLOCATOR: - // vHeapAllocatorClear(&(pvma->vha)); - break; - default: - break; + free(pMA); + if (--pVMA->_ref.val == 0) { + tDListPop(&(pVnode->pBufPool->incycle), pVMA); + tDListAppend(&(pVnode->pBufPool->free), pVMA); } - - free(pNode); -} - -static void *vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size) { - void *ptr = NULL; - - if (pvma->type == E_V_ARENA_ALLOCATOR) { - SVArenaAllocator *pvaa = &(pvma->vaa); - - if (POINTER_DISTANCE(pvaa->inuse->ptr, pvaa->inuse->data) + size > pvaa->inuse->size) { - SVArenaNode *pNode = (SVArenaNode *)malloc(sizeof(*pNode) + MAX(size, pvaa->ssize)); - if (pNode == NULL) { - // TODO: handle error - return NULL; - } - - pNode->prev = pvaa->inuse; - pNode->size = MAX(size, pvaa->ssize); - pNode->ptr = pNode->data; - - pvaa->inuse = pNode; - } - - ptr = pvaa->inuse->ptr; - pvaa->inuse->ptr = POINTER_SHIFT(ptr, size); - } else if (pvma->type == E_V_HEAP_ALLOCATOR) { - /* TODO */ - } - - return ptr; -} - -static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf) { - SVnode * pVnode; - SMemAllocator * pma; - SVMemAllocator *pvma; - SVMAWrapper * pvmaw; - - pVnode = (SVnode *)(pmaf->impl); - pma = (SMemAllocator *)calloc(1, sizeof(*pma) + sizeof(SVMAWrapper)); - if (pma == NULL) { - // TODO: handle error - return NULL; - } - pvmaw = (SVMAWrapper *)POINTER_SHIFT(pma, sizeof(*pma)); - - // No allocator used currently - if (pVnode->pBufPool->inuse == NULL) { - while (listNEles(&(pVnode->pBufPool->free)) == 0) { - // TODO: wait until all released ro kill query - // tsem_wait(); - ASSERT(0); - } - - pVnode->pBufPool->inuse = tdListPopHead(&(pVnode->pBufPool->free)); - pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - T_REF_INIT_VAL(pvma, 1); - } else { - pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - } - - T_REF_INC(pvma); - - pvmaw->pVnode = pVnode; - pvmaw->pNode = pVnode->pBufPool->inuse; - - pma->impl = pvmaw; - pma->malloc = NULL; - pma->calloc = NULL; /* TODO */ - pma->realloc = NULL; /* TODO */ - pma->free = NULL; /* TODO */ - pma->usage = NULL; /* TODO */ - - return pma; -} - -static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma) { /* TODO */ - SVnode * pVnode = (SVnode *)(pmaf->impl); - SListNode * pNode = ((SVMAWrapper *)(pma->impl))->pNode; - SVMemAllocator *pvma = (SVMemAllocator *)(pNode->data); - - if (T_REF_DEC(pvma) == 0) { - if (pvma->type == E_V_ARENA_ALLOCATOR) { - SVArenaAllocator *pvaa = &(pvma->vaa); - while (pvaa->inuse != &(pvaa->node)) { - SVArenaNode *pNode = pvaa->inuse; - pvaa->inuse = pNode->prev; - /* code */ - } - - pvaa->inuse->ptr = pvaa->inuse->data; - } else if (pvma->type == E_V_HEAP_ALLOCATOR) { - } else { - ASSERT(0); - } - - // Move node from incycle to free - tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode)); - // tsem_post(); todo: sem_post - } -} -#endif \ No newline at end of file +} \ No newline at end of file From 2c6dd6c3a0b93fde09d9443da660090721c6b92f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 16:30:02 +0800 Subject: [PATCH 13/14] more --- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 4a60fbe3df..6f2e6f721a 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -226,6 +226,7 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { int rcode; SVnode * pVnode; int batch = 1; + int loop = 1000000; pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg)); @@ -254,13 +255,15 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { vtClearMsgBatch(pMsgArr); // 3. WRITE A LOT OF TIME-SERIES DATA - for (int i = 0; i < batch; i++) { - vtBuildSubmitReq(&pMsg); - taosArrayPush(pMsgArr, &pMsg); + for (int j = 0; j < loop; j++) { + for (int i = 0; i < batch; i++) { + vtBuildSubmitReq(&pMsg); + taosArrayPush(pMsgArr, &pMsg); + } + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); } - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - GTEST_ASSERT_EQ(rcode, 0); - vtClearMsgBatch(pMsgArr); // Close the vnode vnodeClose(pVnode); From f57407f3029a205246857b46347a40becf6312ea Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 17 Dec 2021 13:47:58 +0800 Subject: [PATCH 14/14] make it compile --- include/util/tmacro.h | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 include/util/tmacro.h diff --git a/include/util/tmacro.h b/include/util/tmacro.h new file mode 100644 index 0000000000..5cca8a1062 --- /dev/null +++ b/include/util/tmacro.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_MACRO_H_ +#define _TD_UTIL_MACRO_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Module init/clear MACRO definitions +#define TD_MOD_UNINITIALIZED 0 +#define TD_MOD_INITIALIZED 1 + +#define TD_MOD_UNCLEARD 0 +#define TD_MOD_CLEARD 1 + +typedef int8_t td_mode_flag_t; + +#define TD_CHECK_AND_SET_MODE_INIT(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED) + +#define TD_CHECK_AND_SET_MOD_CLEAR(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNCLEARD, TD_MOD_CLEARD) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_MACRO_H_*/ \ No newline at end of file