From cb5c12f1c60a507769589ef4a969329782f0961f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 16:55:31 +0800 Subject: [PATCH 01/50] refact test --- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 51 ++++++------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index f0bca3aa2e..49f3ff0160 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -14,7 +14,7 @@ #include "vnode.h" -static STSchema *createBasicSchema() { +static STSchema *vtCreateBasicSchema() { STSchemaBuilder sb; STSchema * pSchema = NULL; @@ -32,7 +32,7 @@ static STSchema *createBasicSchema() { return pSchema; } -static STSchema *createBasicTagSchema() { +static STSchema *vtCreateBasicTagSchema() { STSchemaBuilder sb; STSchema * pSchema = NULL; @@ -50,7 +50,7 @@ static STSchema *createBasicTagSchema() { return pSchema; } -static SKVRow createBasicTag() { +static SKVRow vtCreateBasicTag() { SKVRowBuilder rb; SKVRow pTag; @@ -71,27 +71,7 @@ static SKVRow createBasicTag() { return pTag; } -#if 0 -TEST(vnodeApiTest, test_create_table_encode_and_decode_function) { - tb_uid_t suid = 1638166374163; - STSchema *pSchema = createBasicSchema(); - STSchema *pTagSchema = createBasicTagSchema(); - char tbname[128] = "st"; - char * buffer = new char[1024]; - void * pBuf = (void *)buffer; - SVnodeReq vCreateSTbReq = VNODE_INIT_CREATE_STB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); - - vnodeBuildReq(&pBuf, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - - SVnodeReq decoded_req; - - vnodeParseReq(buffer, &decoded_req, TSDB_MSG_TYPE_CREATE_TABLE); - - int k = 10; -} -#endif - -TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { +TEST(vnodeApiTest, vnode_simple_create_table_test) { vnodeDestroy("vnode1"); GTEST_ASSERT_GE(vnodeInit(2), 0); @@ -103,8 +83,8 @@ TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { tb_uid_t suid = 1638166374163; { // Create a super table - STSchema *pSchema = createBasicSchema(); - STSchema *pTagSchema = createBasicTagSchema(); + STSchema *pSchema = vtCreateBasicSchema(); + STSchema *pTagSchema = vtCreateBasicTagSchema(); char tbname[128] = "st"; SArray * pMsgs = (SArray *)taosArrayInit(1, sizeof(SRpcMsg *)); @@ -138,7 +118,7 @@ TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { for (int i = 0; i < ntables / batch; i++) { SArray *pMsgs = (SArray *)taosArrayInit(batch, sizeof(SRpcMsg *)); for (int j = 0; j < batch; j++) { - SKVRow pTag = createBasicTag(); + SKVRow pTag = vtCreateBasicTag(); char tbname[128]; sprintf(tbname, "tb%d", i * batch + j); SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); @@ -176,13 +156,14 @@ TEST(vnodeApiTest, vnodeOpen_vnodeClose_test) { vnodeClear(); } -TEST(vnodeApiTest, DISABLED_vnode_process_create_table) { - STSchema * pSchema = NULL; - STSchema * pTagSchema = NULL; - char stname[15]; - SVCreateTableReq pReq = META_INIT_STB_CFG(stname, UINT32_MAX, UINT32_MAX, 0, pSchema, pTagSchema); +TEST(vnodeApiTest, vnode_simple_insert_test) { + const char *vname = "vnode2"; + vnodeDestroy(vname); - int k = 10; + GTEST_ASSERT_GE(vnodeInit(2), 0); - META_CLEAR_TB_CFG(pReq); -} + SVnode *pVnode = vnodeOpen(vname, NULL); + + vnodeClose(pVnode); + vnodeClear(); +} \ No newline at end of file From 2ad2efdfa37c5dc90465b3af2aff7bb9c7875d4e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 17:52:10 +0800 Subject: [PATCH 02/50] more --- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 170 ++++++++++-------- 1 file changed, 98 insertions(+), 72 deletions(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 49f3ff0160..bbb1ef5a8b 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -71,89 +71,115 @@ static SKVRow vtCreateBasicTag() { return pTag; } -TEST(vnodeApiTest, vnode_simple_create_table_test) { - vnodeDestroy("vnode1"); +static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { + SRpcMsg * pMsg; + STSchema *pSchema; + STSchema *pTagSchema; + int zs; + void * pBuf; + pSchema = vtCreateBasicSchema(); + pTagSchema = vtCreateBasicTagSchema(); + + SVnodeReq vCreateSTbReq = VNODE_INIT_CREATE_STB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); + + zs = vnodeBuildReq(NULL, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); + pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + pMsg->contLen = zs; + pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg)); + + pBuf = pMsg->pCont; + vnodeBuildReq(&pBuf, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + META_CLEAR_TB_CFG(&vCreateSTbReq); + + tdFreeSchema(pSchema); + tdFreeSchema(pTagSchema); + + *ppMsg = pMsg; +} + +static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { + SRpcMsg *pMsg; + int tz; + SKVRow pTag; + + pTag = vtCreateBasicTag(); + SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); + + tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); + pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; + pMsg->contLen = tz; + pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); + void *pBuf = pMsg->pCont; + + vnodeBuildReq(&pBuf, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); + META_CLEAR_TB_CFG(&vCreateCTbReq); + free(pTag); + + *ppMsg = pMsg; +} + +static void vtClearMsgBatch(SArray *pMsgArr) { + SRpcMsg *pMsg; + for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { + pMsg = *(SRpcMsg **)taosArrayGet(pMsgArr, i); + free(pMsg); + } + + taosArrayClear(pMsgArr); +} + +TEST(vnodeApiTest, vnode_simple_create_table_test) { + tb_uid_t suid = 1638166374163; + SRpcMsg *pMsg; + SArray * pMsgArr = NULL; + SVnode * pVnode; + int rcode; + int ntables = 1000000; + int batch = 10; + char tbname[128]; + + pMsgArr = (SArray *)taosArrayInit(batch, sizeof(pMsg)); + + vnodeDestroy("vnode1"); GTEST_ASSERT_GE(vnodeInit(2), 0); - // Create and open a vnode - SVnode *pVnode = vnodeOpen("vnode1", NULL); + // CREATE AND OPEN A VNODE + pVnode = vnodeOpen("vnode1", NULL); ASSERT_NE(pVnode, nullptr); - tb_uid_t suid = 1638166374163; - { - // Create a super table - STSchema *pSchema = vtCreateBasicSchema(); - STSchema *pTagSchema = vtCreateBasicTagSchema(); - char tbname[128] = "st"; + // CREATE A SUPER TABLE + sprintf(tbname, "st"); + vtBuildCreateStbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); - SArray * pMsgs = (SArray *)taosArrayInit(1, sizeof(SRpcMsg *)); - SVnodeReq vCreateSTbReq = VNODE_INIT_CREATE_STB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); - - int zs = vnodeBuildReq(NULL, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); - pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; - pMsg->contLen = zs; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg)); - - void *pBuf = pMsg->pCont; - - vnodeBuildReq(&pBuf, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - META_CLEAR_TB_CFG(&vCreateSTbReq); - - taosArrayPush(pMsgs, &(pMsg)); - - vnodeProcessWMsgs(pVnode, pMsgs); - - free(pMsg); - taosArrayDestroy(pMsgs); - tdFreeSchema(pSchema); - tdFreeSchema(pTagSchema); - } - - { - // Create some child tables - int ntables = 1000000; - int batch = 10; - for (int i = 0; i < ntables / batch; i++) { - SArray *pMsgs = (SArray *)taosArrayInit(batch, sizeof(SRpcMsg *)); - for (int j = 0; j < batch; j++) { - SKVRow pTag = vtCreateBasicTag(); - char tbname[128]; - sprintf(tbname, "tb%d", i * batch + j); - SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); - - int tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - SRpcMsg *pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); - pMsg->msgType = TSDB_MSG_TYPE_CREATE_TABLE; - pMsg->contLen = tz; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); - void *pBuf = pMsg->pCont; - - vnodeBuildReq(&pBuf, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); - META_CLEAR_TB_CFG(&vCreateCTbReq); - free(pTag); - - taosArrayPush(pMsgs, &(pMsg)); - } - - vnodeProcessWMsgs(pVnode, pMsgs); - - for (int j = 0; j < batch; j++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayPop(pMsgs); - free(pMsg); - } - - taosArrayDestroy(pMsgs); - - // std::cout << "the " << i << "th batch is created" << std::endl; + // CREATE A LOT OF CHILD TABLES + for (int i = 0; i < ntables / batch; i++) { + // Build request batch + for (int j = 0; j < batch; j++) { + sprintf(tbname, "ct%d", i * batch + j + 1); + vtBuildCreateCtbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); } + + // Process request batch + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + ASSERT_EQ(rcode, 0); + + // Clear request batch + vtClearMsgBatch(pMsgArr); } - // Close the vnode + // CLOSE THE VNODE vnodeClose(pVnode); - vnodeClear(); + + taosArrayDestroy(pMsgArr); } TEST(vnodeApiTest, vnode_simple_insert_test) { From 13fedf14e1de032f54448bfa8f0d602ee5cf417f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 17:57:51 +0800 Subject: [PATCH 03/50] more --- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index bbb1ef5a8b..e32937b505 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -121,6 +121,14 @@ static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { *ppMsg = pMsg; } +static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) { + // TODO +} + +static void vtBuildSubmitReq(SRpcMsg **ppMsg) { + // TODO +} + static void vtClearMsgBatch(SArray *pMsgArr) { SRpcMsg *pMsg; for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { From 1db84c33aa8a836b30522f751e15b4e79557ed69 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 18:03:02 +0800 Subject: [PATCH 04/50] more --- source/dnode/vnode/impl/test/vBenchmarkTest.cpp | 1 + 1 file changed, 1 insertion(+) create mode 100644 source/dnode/vnode/impl/test/vBenchmarkTest.cpp diff --git a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp new file mode 100644 index 0000000000..19a033d237 --- /dev/null +++ b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp @@ -0,0 +1 @@ +// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest \ No newline at end of file From b383ae5d02bca43113409332e2fc7a6749cfc269 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 15 Dec 2021 19:24:07 +0800 Subject: [PATCH 05/50] try to integrate benchmark test --- source/dnode/vnode/impl/test/vBenchmarkTest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp index 19a033d237..e218886231 100644 --- a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp +++ b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp @@ -1 +1,2 @@ -// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest \ No newline at end of file +// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest +// https://github.com/google/benchmark \ No newline at end of file From 684f1d11e2215e93ff9e5c6454c30770f42938bf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 10:39:24 +0800 Subject: [PATCH 06/50] fix coredump --- source/dnode/vnode/impl/CMakeLists.txt | 2 +- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 36 +++++++++++++++++-- source/dnode/vnode/tq/src/tq.c | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 3623516624..6972605afd 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -19,5 +19,5 @@ target_link_libraries( # test if(${BUILD_TEST}) - #add_subdirectory(test) + add_subdirectory(test) endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index e32937b505..a876a25fba 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -139,7 +139,7 @@ static void vtClearMsgBatch(SArray *pMsgArr) { taosArrayClear(pMsgArr); } -TEST(vnodeApiTest, vnode_simple_create_table_test) { +TEST(vnodeApiTest, DISABLED_vnode_simple_create_table_test) { tb_uid_t suid = 1638166374163; SRpcMsg *pMsg; SArray * pMsgArr = NULL; @@ -192,12 +192,44 @@ TEST(vnodeApiTest, vnode_simple_create_table_test) { TEST(vnodeApiTest, vnode_simple_insert_test) { const char *vname = "vnode2"; + char tbname[128]; + tb_uid_t suid = 1638166374163; + SRpcMsg * pMsg; + SArray * pMsgArr; + int rcode; + SVnode * pVnode; + + pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg)); + vnodeDestroy(vname); GTEST_ASSERT_GE(vnodeInit(2), 0); - SVnode *pVnode = vnodeOpen(vname, NULL); + // Open a vnode + pVnode = vnodeOpen(vname, NULL); + GTEST_ASSERT_NE(pVnode, nullptr); + // 1. CREATE A SUPER TABLE + sprintf(tbname, "st"); + vtBuildCreateStbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); + + // 2. CREATE A CHILD TABLE + sprintf(tbname, "t0"); + vtBuildCreateCtbReq(suid, tbname, &pMsg); + taosArrayPush(pMsgArr, &pMsg); + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); + + // 3. WRITE A LOT OF TIME-SERIES DATA + + // Close the vnode vnodeClose(pVnode); vnodeClear(); + + taosArrayDestroy(pMsgArr); } \ No newline at end of file diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 5888141c58..249ffd7ae3 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -50,7 +50,7 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAl pTq->tqConfig = tqConfig; pTq->tqLogReader = tqLogReader; pTq->tqMemRef.pAlloctorFactory = allocFac; - pTq->tqMemRef.pAllocator = allocFac->create(allocFac); + // pTq->tqMemRef.pAllocator = allocFac->create(allocFac); if(pTq->tqMemRef.pAllocator == NULL) { //TODO } From ec004e5d59dca2de9dfdd817fdbab845330e7319 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 10:55:21 +0800 Subject: [PATCH 07/50] more --- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index a876a25fba..0f330d7b4c 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -139,7 +139,7 @@ static void vtClearMsgBatch(SArray *pMsgArr) { taosArrayClear(pMsgArr); } -TEST(vnodeApiTest, DISABLED_vnode_simple_create_table_test) { +TEST(vnodeApiTest, vnode_simple_create_table_test) { tb_uid_t suid = 1638166374163; SRpcMsg *pMsg; SArray * pMsgArr = NULL; From 5dc96f9de2d6e4006f524376ec0bcbb8dda9ad81 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 11:36:34 +0800 Subject: [PATCH 08/50] more --- source/dnode/vnode/impl/src/vnodeRequest.c | 3 ++- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 22 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeRequest.c b/source/dnode/vnode/impl/src/vnodeRequest.c index be5f5c890c..249bde4e56 100644 --- a/source/dnode/vnode/impl/src/vnodeRequest.c +++ b/source/dnode/vnode/impl/src/vnodeRequest.c @@ -25,9 +25,10 @@ int vnodeBuildReq(void **buf, const SVnodeReq *pReq, uint8_t type) { switch (type) { case TSDB_MSG_TYPE_CREATE_TABLE: tsize += vnodeBuildCreateTableReq(buf, &(pReq->ctReq)); + break; + case TSDB_MSG_TYPE_SUBMIT: /* code */ break; - default: break; } diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 0f330d7b4c..8f33dbba85 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -126,7 +126,19 @@ static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) { } static void vtBuildSubmitReq(SRpcMsg **ppMsg) { - // TODO + SRpcMsg *pMsg; + void * pBuf; + int tz = 0; + + pMsg = (SRpcMsg *)malloc(sizeof(*pMsg) + tz); + pMsg->msgType = TSDB_MSG_TYPE_SUBMIT; + pMsg->contLen = tz; + pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); + + pBuf = pMsg->pCont; + vnodeBuildReq(&pBuf, NULL /*TODO*/, TSDB_MSG_TYPE_SUBMIT); + + *ppMsg = pMsg; } static void vtClearMsgBatch(SArray *pMsgArr) { @@ -198,6 +210,7 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { SArray * pMsgArr; int rcode; SVnode * pVnode; + int batch = 1; pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg)); @@ -226,6 +239,13 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { vtClearMsgBatch(pMsgArr); // 3. WRITE A LOT OF TIME-SERIES DATA + for (int i = 0; i < batch; i++) { + vtBuildSubmitReq(&pMsg); + taosArrayPush(pMsgArr, &pMsg); + } + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); // Close the vnode vnodeClose(pVnode); From 1ab4f2c443fb7a94518699489e0ad16429e09651 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 14:39:20 +0800 Subject: [PATCH 09/50] solve memory problem --- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 8f33dbba85..4a60fbe3df 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -102,9 +102,8 @@ static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { SRpcMsg *pMsg; int tz; - SKVRow pTag; + SKVRow pTag = vtCreateBasicTag(); - pTag = vtCreateBasicTag(); SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag); tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE); @@ -126,17 +125,33 @@ static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) { } static void vtBuildSubmitReq(SRpcMsg **ppMsg) { - SRpcMsg *pMsg; - void * pBuf; - int tz = 0; + SRpcMsg * pMsg; + SSubmitMsg *pSubmitMsg; + SSubmitBlk *pSubmitBlk; + int tz = 1024; // TODO pMsg = (SRpcMsg *)malloc(sizeof(*pMsg) + tz); pMsg->msgType = TSDB_MSG_TYPE_SUBMIT; pMsg->contLen = tz; pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); - pBuf = pMsg->pCont; - vnodeBuildReq(&pBuf, NULL /*TODO*/, TSDB_MSG_TYPE_SUBMIT); + // For submit msg header + pSubmitMsg = (SSubmitMsg *)(pMsg->pCont); + // pSubmitMsg->header.contLen = 0; + // pSubmitMsg->header.vgId = 0; + // pSubmitMsg->length = 0; + pSubmitMsg->numOfBlocks = 1; + + // For submit blk + pSubmitBlk = (SSubmitBlk *)(pSubmitMsg->blocks); + pSubmitBlk->uid = 0; + pSubmitBlk->tid = 0; + pSubmitBlk->padding = 0; + pSubmitBlk->sversion = 0; + pSubmitBlk->dataLen = 0; + pSubmitBlk->numOfRows = 0; + + // For row batch *ppMsg = pMsg; } From d0d56645a9b9bb0aefa00208453f6c7f34f46433 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 15:24:33 +0800 Subject: [PATCH 10/50] more --- include/dnode/vnode/meta/meta.h | 6 ++-- include/dnode/vnode/tq/tq.h | 2 +- include/dnode/vnode/tsdb/tsdb.h | 4 ++- source/dnode/vnode/impl/inc/vnodeBufferPool.h | 2 ++ source/dnode/vnode/impl/inc/vnodeMAF.h | 32 +++++++++++++++++++ source/dnode/vnode/impl/src/vnodeBufferPool.c | 6 ++-- source/dnode/vnode/impl/src/vnodeMain.c | 8 +++-- source/dnode/vnode/meta/src/metaMain.c | 16 +++++----- source/dnode/vnode/tq/src/tq.c | 4 +++ source/dnode/vnode/tsdb/src/tsdbMain.c | 16 +++++----- 10 files changed, 70 insertions(+), 26 deletions(-) create mode 100644 source/dnode/vnode/impl/inc/vnodeMAF.h diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index 44ae1bb79f..b5994a50e1 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -71,7 +71,7 @@ typedef struct STbCfg { } STbCfg; // SMeta operations -SMeta *metaOpen(const char *path, const SMetaCfg *pOptions); +SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg); void metaClose(SMeta *pMeta); void metaRemove(const char *path); int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg); @@ -79,8 +79,8 @@ int metaDropTable(SMeta *pMeta, tb_uid_t uid); int metaCommit(SMeta *pMeta); // Options -void metaOptionsInit(SMetaCfg *pOptions); -void metaOptionsClear(SMetaCfg *pOptions); +void metaOptionsInit(SMetaCfg *pMetaCfg); +void metaOptionsClear(SMetaCfg *pMetaCfg); // STbCfg #define META_INIT_STB_CFG(NAME, TTL, KEEP, SUID, PSCHEMA, PTAGSCHEMA) \ diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 747d97b7a0..074678f248 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -264,7 +264,7 @@ typedef struct STQ { // open in each vnode STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac); -void tqDestroy(STQ*); +void tqClose(STQ*); // void* will be replace by a msg type int tqPushMsg(STQ*, void* msg, int64_t version); diff --git a/include/dnode/vnode/tsdb/tsdb.h b/include/dnode/vnode/tsdb/tsdb.h index f8eac9768f..b85c6b64f6 100644 --- a/include/dnode/vnode/tsdb/tsdb.h +++ b/include/dnode/vnode/tsdb/tsdb.h @@ -16,6 +16,8 @@ #ifndef _TD_TSDB_H_ #define _TD_TSDB_H_ +#include "mallocator.h" + #ifdef __cplusplus extern "C" { #endif @@ -25,7 +27,7 @@ typedef struct STsdb STsdb; typedef struct STsdbCfg STsdbCfg; // STsdb -STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg); +STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF); void tsdbClose(STsdb *); void tsdbRemove(const char *path); int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg); diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/impl/inc/vnodeBufferPool.h index d96671d2bd..b4535597ee 100644 --- a/source/dnode/vnode/impl/inc/vnodeBufferPool.h +++ b/source/dnode/vnode/impl/inc/vnodeBufferPool.h @@ -32,6 +32,8 @@ int vnodeBufPoolRecycle(SVnode *pVnode); void *vnodeMalloc(SVnode *pVnode, uint64_t size); bool vnodeBufPoolIsFull(SVnode *pVnode); +SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/impl/inc/vnodeMAF.h b/source/dnode/vnode/impl/inc/vnodeMAF.h new file mode 100644 index 0000000000..7aa405103c --- /dev/null +++ b/source/dnode/vnode/impl/inc/vnodeMAF.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_MAF_H_ +#define _TD_VNODE_MAF_H_ + +#include "vnode.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int vnodeOpenMAF(SVnode *pVnode); +void vnodeCloseMAF(SVnode *pVnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_MAF_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 152a346f0a..49e7889517 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -24,8 +24,8 @@ struct SVBufPool { TD_DLIST(SVMemAllocator) free; TD_DLIST(SVMemAllocator) incycle; SVMemAllocator *inuse; - // MAF for submodules - // SMemAllocatorFactory maf; + // MAF for submodules to use + SMemAllocatorFactory *pMAF; }; int vnodeOpenBufPool(SVnode *pVnode) { @@ -125,6 +125,8 @@ bool vnodeBufPoolIsFull(SVnode *pVnode) { return vmaIsFull(pVnode->pBufPool->inuse); } +SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode) { return pVnode->pBufPool->pMAF; } + #if 0 typedef enum { diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index 59e3bae5d7..c1539659ae 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -102,7 +102,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open tsdb sprintf(dir, "%s/tsdb", pVnode->path); - pVnode->pTsdb = tsdbOpen(dir, &(pVnode->config.tsdbCfg)); + pVnode->pTsdb = tsdbOpen(dir, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pTsdb == NULL) { // TODO: handle error return -1; @@ -110,7 +110,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // TODO: Open TQ sprintf(dir, "%s/tq", pVnode->path); - pVnode->pTq = tqOpen(dir, &(pVnode->config.tqCfg), NULL, NULL); + pVnode->pTq = tqOpen(dir, &(pVnode->config.tqCfg), NULL, vBufPoolGetMAF(pVnode)); if (pVnode->pTq == NULL) { // TODO: handle error return -1; @@ -131,7 +131,9 @@ static int vnodeOpenImpl(SVnode *pVnode) { static void vnodeCloseImpl(SVnode *pVnode) { if (pVnode) { vnodeCloseBufPool(pVnode); - tsdbClose(pVnode->pTsdb); metaClose(pVnode->pMeta); + tsdbClose(pVnode->pTsdb); + tqClose(pVnode->pTq); + walClose(pVnode->pWal); } } \ No newline at end of file diff --git a/source/dnode/vnode/meta/src/metaMain.c b/source/dnode/vnode/meta/src/metaMain.c index 53055dcea3..f2602fbad7 100644 --- a/source/dnode/vnode/meta/src/metaMain.c +++ b/source/dnode/vnode/meta/src/metaMain.c @@ -17,27 +17,27 @@ #include "metaDef.h" -static SMeta *metaNew(const char *path, const SMetaCfg *pMetaOptions); +static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg); static void metaFree(SMeta *pMeta); static int metaOpenImpl(SMeta *pMeta); static void metaCloseImpl(SMeta *pMeta); -SMeta *metaOpen(const char *path, const SMetaCfg *pMetaOptions) { +SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg) { SMeta *pMeta = NULL; // Set default options - if (pMetaOptions == NULL) { - pMetaOptions = &defaultMetaOptions; + if (pMetaCfg == NULL) { + pMetaCfg = &defaultMetaOptions; } // Validate the options - if (metaValidateOptions(pMetaOptions) < 0) { + if (metaValidateOptions(pMetaCfg) < 0) { // TODO: deal with error return NULL; } // Allocate handle - pMeta = metaNew(path, pMetaOptions); + pMeta = metaNew(path, pMetaCfg); if (pMeta == NULL) { // TODO: handle error return NULL; @@ -65,7 +65,7 @@ void metaClose(SMeta *pMeta) { void metaRemove(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static SMeta *metaNew(const char *path, const SMetaCfg *pMetaOptions) { +static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg) { SMeta *pMeta; size_t psize = strlen(path); @@ -80,7 +80,7 @@ static SMeta *metaNew(const char *path, const SMetaCfg *pMetaOptions) { return NULL; } - metaOptionsCopy(&(pMeta->options), pMetaOptions); + metaOptionsCopy(&(pMeta->options), pMetaCfg); return pMeta; }; diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 249ffd7ae3..4aabf07c2d 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -66,6 +66,10 @@ STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAl return pTq; } +void tqClose(STQ*pTq) { + // TODO +} + static int tqProtoCheck(TmqMsgHead *pMsg) { return pMsg->protoVer == 0; } diff --git a/source/dnode/vnode/tsdb/src/tsdbMain.c b/source/dnode/vnode/tsdb/src/tsdbMain.c index 2fe7a61930..20b75aa78f 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMain.c +++ b/source/dnode/vnode/tsdb/src/tsdbMain.c @@ -15,27 +15,27 @@ #include "tsdbDef.h" -static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbOptions); +static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg); static void tsdbFree(STsdb *pTsdb); static int tsdbOpenImpl(STsdb *pTsdb); static void tsdbCloseImpl(STsdb *pTsdb); -STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbOptions) { +STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) { STsdb *pTsdb = NULL; // Set default TSDB Options - if (pTsdbOptions == NULL) { - pTsdbOptions = &defautlTsdbOptions; + if (pTsdbCfg == NULL) { + pTsdbCfg = &defautlTsdbOptions; } // Validate the options - if (tsdbValidateOptions(pTsdbOptions) < 0) { + if (tsdbValidateOptions(pTsdbCfg) < 0) { // TODO: handle error return NULL; } // Create the handle - pTsdb = tsdbNew(path, pTsdbOptions); + pTsdb = tsdbNew(path, pTsdbCfg); if (pTsdb == NULL) { // TODO: handle error return NULL; @@ -62,7 +62,7 @@ void tsdbClose(STsdb *pTsdb) { void tsdbRemove(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbOptions) { +static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg) { STsdb *pTsdb = NULL; pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); @@ -72,7 +72,7 @@ static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbOptions) { } pTsdb->path = strdup(path); - tsdbOptionsCopy(&(pTsdb->options), pTsdbOptions); + tsdbOptionsCopy(&(pTsdb->options), pTsdbCfg); return pTsdb; } From 835c3030c591749c46da0dd4a7c03e07dce1c68e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 15:28:53 +0800 Subject: [PATCH 11/50] more --- include/dnode/vnode/meta/meta.h | 3 ++- source/dnode/vnode/impl/src/vnodeMain.c | 2 +- source/dnode/vnode/meta/inc/metaDef.h | 2 +- source/dnode/vnode/meta/src/metaMain.c | 9 +++++---- source/dnode/vnode/tsdb/src/tsdbMain.c | 7 ++++--- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/include/dnode/vnode/meta/meta.h b/include/dnode/vnode/meta/meta.h index b5994a50e1..113a970548 100644 --- a/include/dnode/vnode/meta/meta.h +++ b/include/dnode/vnode/meta/meta.h @@ -16,6 +16,7 @@ #ifndef _TD_META_H_ #define _TD_META_H_ +#include "mallocator.h" #include "os.h" #include "trow.h" @@ -71,7 +72,7 @@ typedef struct STbCfg { } STbCfg; // SMeta operations -SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg); +SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); void metaClose(SMeta *pMeta); void metaRemove(const char *path); int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg); diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index c1539659ae..70d9c7d4b0 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -94,7 +94,7 @@ static int vnodeOpenImpl(SVnode *pVnode) { // Open meta sprintf(dir, "%s/meta", pVnode->path); - pVnode->pMeta = metaOpen(dir, &(pVnode->config.metaCfg)); + pVnode->pMeta = metaOpen(dir, &(pVnode->config.metaCfg), vBufPoolGetMAF(pVnode)); if (pVnode->pMeta == NULL) { // TODO: handle error return -1; diff --git a/source/dnode/vnode/meta/inc/metaDef.h b/source/dnode/vnode/meta/inc/metaDef.h index 0204031e76..e1c15af5aa 100644 --- a/source/dnode/vnode/meta/inc/metaDef.h +++ b/source/dnode/vnode/meta/inc/metaDef.h @@ -34,7 +34,7 @@ extern "C" { struct SMeta { char* path; SMetaCfg options; - SMetaDB* pDB; + SMetaDB* pDB; SMetaIdx* pIdx; SMetaCache* pCache; STbUidGenerator uidGnrt; diff --git a/source/dnode/vnode/meta/src/metaMain.c b/source/dnode/vnode/meta/src/metaMain.c index f2602fbad7..b6aa029b19 100644 --- a/source/dnode/vnode/meta/src/metaMain.c +++ b/source/dnode/vnode/meta/src/metaMain.c @@ -17,12 +17,12 @@ #include "metaDef.h" -static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg); +static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF); static void metaFree(SMeta *pMeta); static int metaOpenImpl(SMeta *pMeta); static void metaCloseImpl(SMeta *pMeta); -SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg) { +SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF) { SMeta *pMeta = NULL; // Set default options @@ -37,7 +37,7 @@ SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg) { } // Allocate handle - pMeta = metaNew(path, pMetaCfg); + pMeta = metaNew(path, pMetaCfg, pMAF); if (pMeta == NULL) { // TODO: handle error return NULL; @@ -65,7 +65,7 @@ void metaClose(SMeta *pMeta) { void metaRemove(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg) { +static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF) { SMeta *pMeta; size_t psize = strlen(path); @@ -81,6 +81,7 @@ static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg) { } metaOptionsCopy(&(pMeta->options), pMetaCfg); + pMeta->pmaf = pMAF; return pMeta; }; diff --git a/source/dnode/vnode/tsdb/src/tsdbMain.c b/source/dnode/vnode/tsdb/src/tsdbMain.c index 20b75aa78f..d67d45660d 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMain.c +++ b/source/dnode/vnode/tsdb/src/tsdbMain.c @@ -15,7 +15,7 @@ #include "tsdbDef.h" -static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg); +static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF); static void tsdbFree(STsdb *pTsdb); static int tsdbOpenImpl(STsdb *pTsdb); static void tsdbCloseImpl(STsdb *pTsdb); @@ -35,7 +35,7 @@ STsdb *tsdbOpen(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory } // Create the handle - pTsdb = tsdbNew(path, pTsdbCfg); + pTsdb = tsdbNew(path, pTsdbCfg, pMAF); if (pTsdb == NULL) { // TODO: handle error return NULL; @@ -62,7 +62,7 @@ void tsdbClose(STsdb *pTsdb) { void tsdbRemove(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg) { +static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF) { STsdb *pTsdb = NULL; pTsdb = (STsdb *)calloc(1, sizeof(STsdb)); @@ -73,6 +73,7 @@ static STsdb *tsdbNew(const char *path, const STsdbCfg *pTsdbCfg) { pTsdb->path = strdup(path); tsdbOptionsCopy(&(pTsdb->options), pTsdbCfg); + pTsdb->pmaf = pMAF; return pTsdb; } From d1e0a9a780538f399ce00211900cd82f372b1c57 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 16:20:46 +0800 Subject: [PATCH 12/50] make test run --- source/dnode/vnode/impl/src/vnodeBufferPool.c | 223 ++++-------------- 1 file changed, 46 insertions(+), 177 deletions(-) diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 49e7889517..6c1ededfc9 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -28,6 +28,9 @@ struct SVBufPool { SMemAllocatorFactory *pMAF; }; +static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF); +static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA); + int vnodeOpenBufPool(SVnode *pVnode) { uint64_t capacity; @@ -54,6 +57,15 @@ int vnodeOpenBufPool(SVnode *pVnode) { tDListAppend(&(pVnode->pBufPool->free), pVMA); } + pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)malloc(sizeof(SMemAllocatorFactory)); + if (pVnode->pBufPool->pMAF == NULL) { + // TODO: handle error + return -1; + } + pVnode->pBufPool->pMAF->impl = pVnode; + pVnode->pBufPool->pMAF->create = vBufPoolCreateMA; + pVnode->pBufPool->pMAF->destroy = vBufPoolDestroyMA; + return 0; } @@ -127,195 +139,52 @@ bool vnodeBufPoolIsFull(SVnode *pVnode) { SMemAllocatorFactory *vBufPoolGetMAF(SVnode *pVnode) { return pVnode->pBufPool->pMAF; } -#if 0 - -typedef enum { - // Heap allocator - E_V_HEAP_ALLOCATOR = 0, - // Arena allocator - E_V_ARENA_ALLOCATOR -} EVMemAllocatorT; - +/* ------------------------ STATIC METHODS ------------------------ */ typedef struct { - /* TODO */ -} SVHeapAllocator; - -typedef struct SVArenaNode { - struct SVArenaNode *prev; - uint64_t size; - void * ptr; - char data[]; -} SVArenaNode; - -typedef struct { - uint64_t ssize; // step size - uint64_t lsize; // limit size - SVArenaNode *inuse; - SVArenaNode node; -} SVArenaAllocator; - -typedef struct { - SVnode * pVnode; - SListNode *pNode; + SVnode * pVnode; + SVMemAllocator *pVMA; } SVMAWrapper; +static FORCE_INLINE void *vmaMaloocCb(SMemAllocator *pMA, uint64_t size) { + SVMAWrapper *pWrapper = (SVMAWrapper *)(pMA->impl); -static SListNode * vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type); -static void vBufPoolFreeNode(SListNode *pNode); -static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf); -static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma); -static void * vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size); + return vmaMalloc(pWrapper->pVMA, size); +} -/* ------------------------ STATIC METHODS ------------------------ */ -static SListNode *vBufPoolNewNode(uint64_t capacity, EVMemAllocatorT type) { - SListNode * pNode; - SVMemAllocator *pvma; - uint64_t msize; - uint64_t ssize = 4096; // TODO - uint64_t lsize = 1024; // TODO +// TODO: Add atomic operations here +static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF) { + SMemAllocator *pMA; + SVnode * pVnode = (SVnode *)(pMAF->impl); + SVMAWrapper * pWrapper; - msize = sizeof(SListNode) + sizeof(SVMemAllocator); - if (type == E_V_ARENA_ALLOCATOR) { - msize += capacity; - } - - pNode = (SListNode *)calloc(1, msize); - if (pNode == NULL) { - // TODO: handle error + pMA = (SMemAllocator *)calloc(1, sizeof(*pMA) + sizeof(SVMAWrapper)); + if (pMA == NULL) { return NULL; } - pvma = (SVMemAllocator *)(pNode->data); - pvma->capacity = capacity; - pvma->type = type; + pVnode->pBufPool->inuse->_ref.val++; + pWrapper = POINTER_SHIFT(pMA, sizeof(*pMA)); + pWrapper->pVnode = pVnode; + pWrapper->pVMA = pVnode->pBufPool->inuse; - switch (type) { - case E_V_ARENA_ALLOCATOR: - vArenaAllocatorInit(&(pvma->vaa), capacity, ssize, lsize); - break; - case E_V_HEAP_ALLOCATOR: - // vHeapAllocatorInit(&(pvma->vha)); - break; - default: - ASSERT(0); - } + pMA->impl = pWrapper; + pMA->malloc = vmaMaloocCb; + pMA->calloc = NULL; + pMA->realloc = NULL; + pMA->free = NULL; + pMA->usage = NULL; - return pNode; + return pMA; } -static void vBufPoolFreeNode(SListNode *pNode) { - SVMemAllocator *pvma = (SVMemAllocator *)(pNode->data); +static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA) { + SVMAWrapper * pWrapper = (SVMAWrapper *)(pMA->impl); + SVnode * pVnode = pWrapper->pVnode; + SVMemAllocator *pVMA = pWrapper->pVMA; - switch (pvma->type) { - case E_V_ARENA_ALLOCATOR: - vArenaAllocatorClear(&(pvma->vaa)); - break; - case E_V_HEAP_ALLOCATOR: - // vHeapAllocatorClear(&(pvma->vha)); - break; - default: - break; + free(pMA); + if (--pVMA->_ref.val == 0) { + tDListPop(&(pVnode->pBufPool->incycle), pVMA); + tDListAppend(&(pVnode->pBufPool->free), pVMA); } - - free(pNode); -} - -static void *vBufPoolMalloc(SVMemAllocator *pvma, uint64_t size) { - void *ptr = NULL; - - if (pvma->type == E_V_ARENA_ALLOCATOR) { - SVArenaAllocator *pvaa = &(pvma->vaa); - - if (POINTER_DISTANCE(pvaa->inuse->ptr, pvaa->inuse->data) + size > pvaa->inuse->size) { - SVArenaNode *pNode = (SVArenaNode *)malloc(sizeof(*pNode) + MAX(size, pvaa->ssize)); - if (pNode == NULL) { - // TODO: handle error - return NULL; - } - - pNode->prev = pvaa->inuse; - pNode->size = MAX(size, pvaa->ssize); - pNode->ptr = pNode->data; - - pvaa->inuse = pNode; - } - - ptr = pvaa->inuse->ptr; - pvaa->inuse->ptr = POINTER_SHIFT(ptr, size); - } else if (pvma->type == E_V_HEAP_ALLOCATOR) { - /* TODO */ - } - - return ptr; -} - -static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pmaf) { - SVnode * pVnode; - SMemAllocator * pma; - SVMemAllocator *pvma; - SVMAWrapper * pvmaw; - - pVnode = (SVnode *)(pmaf->impl); - pma = (SMemAllocator *)calloc(1, sizeof(*pma) + sizeof(SVMAWrapper)); - if (pma == NULL) { - // TODO: handle error - return NULL; - } - pvmaw = (SVMAWrapper *)POINTER_SHIFT(pma, sizeof(*pma)); - - // No allocator used currently - if (pVnode->pBufPool->inuse == NULL) { - while (listNEles(&(pVnode->pBufPool->free)) == 0) { - // TODO: wait until all released ro kill query - // tsem_wait(); - ASSERT(0); - } - - pVnode->pBufPool->inuse = tdListPopHead(&(pVnode->pBufPool->free)); - pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - T_REF_INIT_VAL(pvma, 1); - } else { - pvma = (SVMemAllocator *)(pVnode->pBufPool->inuse->data); - } - - T_REF_INC(pvma); - - pvmaw->pVnode = pVnode; - pvmaw->pNode = pVnode->pBufPool->inuse; - - pma->impl = pvmaw; - pma->malloc = NULL; - pma->calloc = NULL; /* TODO */ - pma->realloc = NULL; /* TODO */ - pma->free = NULL; /* TODO */ - pma->usage = NULL; /* TODO */ - - return pma; -} - -static void vBufPoolDestroyMA(SMemAllocatorFactory *pmaf, SMemAllocator *pma) { /* TODO */ - SVnode * pVnode = (SVnode *)(pmaf->impl); - SListNode * pNode = ((SVMAWrapper *)(pma->impl))->pNode; - SVMemAllocator *pvma = (SVMemAllocator *)(pNode->data); - - if (T_REF_DEC(pvma) == 0) { - if (pvma->type == E_V_ARENA_ALLOCATOR) { - SVArenaAllocator *pvaa = &(pvma->vaa); - while (pvaa->inuse != &(pvaa->node)) { - SVArenaNode *pNode = pvaa->inuse; - pvaa->inuse = pNode->prev; - /* code */ - } - - pvaa->inuse->ptr = pvaa->inuse->data; - } else if (pvma->type == E_V_HEAP_ALLOCATOR) { - } else { - ASSERT(0); - } - - // Move node from incycle to free - tdListAppendNode(&(pVnode->pBufPool->free), tdListPopNode(&(pVnode->pBufPool->incycle), pNode)); - // tsem_post(); todo: sem_post - } -} -#endif \ No newline at end of file +} \ No newline at end of file From 2c6dd6c3a0b93fde09d9443da660090721c6b92f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 16 Dec 2021 16:30:02 +0800 Subject: [PATCH 13/50] more --- source/dnode/vnode/impl/test/vnodeApiTests.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 4a60fbe3df..6f2e6f721a 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -226,6 +226,7 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { int rcode; SVnode * pVnode; int batch = 1; + int loop = 1000000; pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg)); @@ -254,13 +255,15 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { vtClearMsgBatch(pMsgArr); // 3. WRITE A LOT OF TIME-SERIES DATA - for (int i = 0; i < batch; i++) { - vtBuildSubmitReq(&pMsg); - taosArrayPush(pMsgArr, &pMsg); + for (int j = 0; j < loop; j++) { + for (int i = 0; i < batch; i++) { + vtBuildSubmitReq(&pMsg); + taosArrayPush(pMsgArr, &pMsg); + } + rcode = vnodeProcessWMsgs(pVnode, pMsgArr); + GTEST_ASSERT_EQ(rcode, 0); + vtClearMsgBatch(pMsgArr); } - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - GTEST_ASSERT_EQ(rcode, 0); - vtClearMsgBatch(pMsgArr); // Close the vnode vnodeClose(pVnode); From 45391bcdb6349b86026410fc6d9ab6a3a38c9609 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Dec 2021 18:35:28 +0800 Subject: [PATCH 14/50] refact test files --- source/dnode/mgmt/impl/test/CMakeLists.txt | 2 +- .../mgmt/impl/test/cluster/CMakeLists.txt | 12 +- .../dnode/mgmt/impl/test/cluster/cluster.cpp | 267 +++++++++--------- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 4 +- .../dnode/mgmt/impl/test/show/CMakeLists.txt | 12 +- source/dnode/mnode/impl/src/mndCluster.c | 4 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- 7 files changed, 155 insertions(+), 148 deletions(-) diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 86186f77de..1dbf1160af 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -2,7 +2,7 @@ add_subdirectory(acct) # add_subdirectory(auth) # add_subdirectory(balance) -# add_subdirectory(cluster) +add_subdirectory(cluster) add_subdirectory(db) add_subdirectory(dnode) # add_subdirectory(func) diff --git a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt index b513f290e4..47b6009000 100644 --- a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt @@ -1,20 +1,20 @@ -add_executable(dndTestCluster "") +add_executable(dnode_test_cluster "") -target_sources(dndTestCluster +target_sources(dnode_test_cluster PRIVATE "cluster.cpp" "../sut/deploy.cpp" ) target_link_libraries( - dndTestCluster + dnode_test_cluster PUBLIC dnode PUBLIC util PUBLIC os PUBLIC gtest_main ) -target_include_directories(dndTestCluster +target_include_directories(dnode_test_cluster PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" @@ -24,6 +24,6 @@ target_include_directories(dndTestCluster enable_testing() add_test( - NAME dndTestCluster - COMMAND dndTestCluster + NAME dnode_test_cluster + COMMAND dnode_test_cluster ) diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index 4e1dbf8911..f822ba4c3d 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -13,154 +13,161 @@ class DndTestCluster : public ::testing::Test { protected: - void SetUp() override {} - void TearDown() override {} + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } static void SetUpTestSuite() { - const char* user = "root"; - const char* pass = "taosdata"; - const char* path = "/tmp/dndTestCluster"; - const char* fqdn = "localhost"; - uint16_t port = 9521; + initLog("/tmp/tdlog"); - pServer = createServer(path, fqdn, port); - ASSERT(pServer); - pClient = createClient(user, pass, fqdn, port); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9030"; + pServer = CreateServer("/tmp/dnode_test_cluster", fqdn, 9030, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9030); + taosMsleep(1100); } static void TearDownTestSuite() { stopServer(pServer); dropClient(pClient); + pServer = NULL; + pClient = NULL; } static SServer* pServer; static SClient* pClient; static int32_t connId; + + public: + void SetUp() override {} + void TearDown() override {} + + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = showType; + if (db != NULL) { + strcpy(pShow->db, db); + } + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, showName); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, columns); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + } + + void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, type); + EXPECT_EQ(pSchema->bytes, bytes); + EXPECT_STREQ(pSchema->name, name); + } + + void SendThenCheckShowRetrieveMsg(int32_t rows) { + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, rows); + EXPECT_EQ(pRetrieveRsp->offset, 0); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + // EXPECT_EQ(pRetrieveRsp->completed, completed); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->reserved, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + pData = pRetrieveRsp->data; + pos = 0; + } + + void CheckInt32() { + int32_t data = *((int32_t*)(pData + pos)); + pos += sizeof(int32_t); + EXPECT_GT(data, 0); + } + + void CheckTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(data, 0); + } + + void CheckBinary(int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + } + + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; }; SServer* DndTestCluster::pServer; SClient* DndTestCluster::pClient; int32_t DndTestCluster::connId; -TEST_F(DndTestCluster, ShowCluster) { - ASSERT_NE(pClient, nullptr); - int32_t showId = 0; +TEST_F(DndTestCluster, 01_ShowCluster) { + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_CLUSTER, "show cluster", 3, NULL); + CheckSchema(0, TSDB_DATA_TYPE_INT, 4, "id"); + CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, "name"); + CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); - { - SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); - pReq->type = TSDB_MGMT_TABLE_CLUSTER; - strcpy(pReq->db, ""); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SShowMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - - SShowRsp* pRsp = (SShowRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->showId = htonl(pRsp->showId); - STableMetaMsg* pMeta = &pRsp->tableMeta; - pMeta->contLen = htonl(pMeta->contLen); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tid = htonl(pMeta->tid); - pMeta->uid = htobe64(pMeta->uid); - pMeta->suid = htobe64(pMeta->suid); - - showId = pRsp->showId; - - EXPECT_NE(pRsp->showId, 0); - EXPECT_EQ(pMeta->contLen, 0); - EXPECT_STREQ(pMeta->tbFname, "show cluster"); - EXPECT_EQ(pMeta->numOfTags, 0); - EXPECT_EQ(pMeta->precision, 0); - EXPECT_EQ(pMeta->tableType, 0); - EXPECT_EQ(pMeta->numOfColumns, 3); - EXPECT_EQ(pMeta->sversion, 0); - EXPECT_EQ(pMeta->tversion, 0); - EXPECT_EQ(pMeta->tid, 0); - EXPECT_EQ(pMeta->uid, 0); - EXPECT_STREQ(pMeta->sTableName, ""); - EXPECT_EQ(pMeta->suid, 0); - - SSchema* pSchema = NULL; - pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); - EXPECT_EQ(pSchema->bytes, 4); - EXPECT_STREQ(pSchema->name, "id"); - - pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); - EXPECT_EQ(pSchema->bytes, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE); - EXPECT_STREQ(pSchema->name, "name"); - - pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); - EXPECT_EQ(pSchema->colId, 0); - EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); - EXPECT_EQ(pSchema->bytes, 8); - EXPECT_STREQ(pSchema->name, "create_time"); - } - - { - SRetrieveTableMsg* pReq = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); - pReq->showId = htonl(showId); - pReq->free = 0; - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SRetrieveTableMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->offset = htobe64(pRsp->offset); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->compLen = htonl(pRsp->compLen); - - EXPECT_EQ(pRsp->numOfRows, 1); - EXPECT_EQ(pRsp->offset, 0); - EXPECT_EQ(pRsp->useconds, 0); - EXPECT_EQ(pRsp->completed, 1); - EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->reserved, 0); - EXPECT_EQ(pRsp->compLen, 0); - - char* pData = pRsp->data; - int32_t pos = 0; - - int32_t id = *((int32_t*)(pData + pos)); - pos += sizeof(int32_t); - - int32_t nameLen = varDataLen(pData + pos); - pos += sizeof(VarDataLenT); - - char* name = (char*)(pData + pos); - pos += TSDB_CLUSTER_ID_LEN; - - int64_t create_time = *((int64_t*)(pData + pos)); - pos += sizeof(int64_t); - - EXPECT_NE(id, 0); - EXPECT_EQ(nameLen, 36); - EXPECT_STRNE(name, ""); - EXPECT_GT(create_time, 0); - printf("--- id:%d nameLen:%d name:%s time:%" PRId64 " --- \n", id, nameLen, name, create_time); - } + SendThenCheckShowRetrieveMsg(1); + CheckInt32(); + CheckBinary(TSDB_CLUSTER_ID_LEN); + CheckTimestamp(); } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index f382f385d7..f2b3ce1350 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -81,8 +81,8 @@ class DndTestDnode : public ::testing::Test { pMeta->numOfTags = htonl(pMeta->numOfTags); pMeta->numOfColumns = htonl(pMeta->numOfColumns); pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); - pMeta->tuid = htonl(pMeta->tuid); + pMeta->tversion = htonl(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); pMeta->suid = htobe64(pMeta->suid); showId = pShowRsp->showId; diff --git a/source/dnode/mgmt/impl/test/show/CMakeLists.txt b/source/dnode/mgmt/impl/test/show/CMakeLists.txt index 5f1596ce95..b0a799d84b 100644 --- a/source/dnode/mgmt/impl/test/show/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/show/CMakeLists.txt @@ -1,20 +1,20 @@ -add_executable(dndTestShow "") +add_executable(dnode_test_show "") -target_sources(dndTestShow +target_sources(dnode_test_show PRIVATE "show.cpp" "../sut/deploy.cpp" ) target_link_libraries( - dndTestShow + dnode_test_show PUBLIC dnode PUBLIC util PUBLIC os PUBLIC gtest_main ) -target_include_directories(dndTestShow +target_include_directories(dnode_test_show PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" @@ -24,6 +24,6 @@ target_include_directories(dndTestShow enable_testing() add_test( - NAME dndTestShow - COMMAND dndTestShow + NAME dnode_test_show + COMMAND dnode_test_show ) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 0eaa184907..3d2c12c4ab 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -154,11 +154,11 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create_time"); + strcpy(pSchema[cols].name, "create time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); strcpy(pMeta->tbFname, mndShowStr(pShow->type)); pShow->numOfColumns = cols; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 91a229956f..db6777ebf8 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -291,7 +291,7 @@ char *mndShowStr(int32_t showType) { case TSDB_MGMT_TABLE_VNODES: return "show vnodes"; case TSDB_MGMT_TABLE_CLUSTER: - return "show clusters"; + return "show cluster"; case TSDB_MGMT_TABLE_STREAMTABLES: return "show streamtables"; case TSDB_MGMT_TABLE_TP: From 22ed98dd56ce1c3ca00935ab8b8380b57d95798e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Dec 2021 20:16:35 +0800 Subject: [PATCH 15/50] rename test cases --- source/dnode/mgmt/impl/test/CMakeLists.txt | 3 +- .../dnode/mgmt/impl/test/acct/CMakeLists.txt | 2 - source/dnode/mgmt/impl/test/acct/acct.cpp | 2 +- .../mgmt/impl/test/cluster/CMakeLists.txt | 2 - .../dnode/mgmt/impl/test/cluster/cluster.cpp | 2 +- source/dnode/mgmt/impl/test/db/CMakeLists.txt | 2 - source/dnode/mgmt/impl/test/db/db.cpp | 2 +- .../dnode/mgmt/impl/test/dnode/CMakeLists.txt | 2 - source/dnode/mgmt/impl/test/dnode/dnode.cpp | 2 +- .../mgmt/impl/test/profile/CMakeLists.txt | 2 - .../dnode/mgmt/impl/test/profile/profile.cpp | 2 +- .../dnode/mgmt/impl/test/show/CMakeLists.txt | 2 - source/dnode/mgmt/impl/test/show/show.cpp | 2 +- .../dnode/mgmt/impl/test/stb/CMakeLists.txt | 27 + source/dnode/mgmt/impl/test/stb/stb.cpp | 479 ++++++++++++++++++ .../dnode/mgmt/impl/test/user/CMakeLists.txt | 2 - source/dnode/mgmt/impl/test/user/user.cpp | 2 +- 17 files changed, 515 insertions(+), 22 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/stb/CMakeLists.txt create mode 100644 source/dnode/mgmt/impl/test/stb/stb.cpp diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index 1dbf1160af..8c6d146fb6 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,3 +1,4 @@ +enable_testing() add_subdirectory(acct) # add_subdirectory(auth) @@ -9,7 +10,7 @@ add_subdirectory(dnode) # add_subdirectory(mnode) # add_subdirectory(profile) # add_subdirectory(show) -# add_subdirectory(stb) +add_subdirectory(stb) # add_subdirectory(sync) # add_subdirectory(telem) # add_subdirectory(trans) diff --git a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt index a548c2adc2..8b160fc446 100644 --- a/source/dnode/mgmt/impl/test/acct/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/acct/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dnode_test_acct "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dnode_test_acct COMMAND dnode_test_acct diff --git a/source/dnode/mgmt/impl/test/acct/acct.cpp b/source/dnode/mgmt/impl/test/acct/acct.cpp index 6de8c70ba9..e2b819da28 100644 --- a/source/dnode/mgmt/impl/test/acct/acct.cpp +++ b/source/dnode/mgmt/impl/test/acct/acct.cpp @@ -1,5 +1,5 @@ /** - * @file vnodeApiTests.cpp + * @file acct.cpp * @author slguan (slguan@taosdata.com) * @brief DNODE module acct-msg tests * @version 0.1 diff --git a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt index 47b6009000..e97ef76881 100644 --- a/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/cluster/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dnode_test_cluster "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dnode_test_cluster COMMAND dnode_test_cluster diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index f822ba4c3d..0822be67f7 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -1,5 +1,5 @@ /** - * @file vnodeApiTests.cpp + * @file cluster.cpp * @author slguan (slguan@taosdata.com) * @brief DNODE module cluster-msg tests * @version 0.1 diff --git a/source/dnode/mgmt/impl/test/db/CMakeLists.txt b/source/dnode/mgmt/impl/test/db/CMakeLists.txt index b778e3854f..db96e2bad3 100644 --- a/source/dnode/mgmt/impl/test/db/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/db/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dnode_test_db "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dnode_test_db COMMAND dnode_test_db diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index 6821dec0a6..c568b1152d 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -1,5 +1,5 @@ /** - * @file vnodeApiTests.cpp + * @file db.cpp * @author slguan (slguan@taosdata.com) * @brief DNODE module db-msg tests * @version 0.1 diff --git a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt index ebe2f3c5eb..48b74a4eff 100644 --- a/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/dnode/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dnode_test_dnode "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dnode_test_dnode COMMAND dnode_test_dnode diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index f2b3ce1350..0cadb3a4c4 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -1,5 +1,5 @@ /** - * @file vnodeApiTests.cpp + * @file dnode.cpp * @author slguan (slguan@taosdata.com) * @brief DNODE module dnode-msg tests * @version 0.1 diff --git a/source/dnode/mgmt/impl/test/profile/CMakeLists.txt b/source/dnode/mgmt/impl/test/profile/CMakeLists.txt index 2e99a2e53d..d8e4eae042 100644 --- a/source/dnode/mgmt/impl/test/profile/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/profile/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dndTestProfile "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dndTestProfile COMMAND dndTestProfile diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index e2dbeb1a5b..f5970b61c5 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -1,5 +1,5 @@ /** - * @file vnodeApiTests.cpp + * @file profile.cpp * @author slguan (slguan@taosdata.com) * @brief DNODE module profile-msg tests * @version 0.1 diff --git a/source/dnode/mgmt/impl/test/show/CMakeLists.txt b/source/dnode/mgmt/impl/test/show/CMakeLists.txt index b0a799d84b..b3dc3297a8 100644 --- a/source/dnode/mgmt/impl/test/show/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/show/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dnode_test_show "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dnode_test_show COMMAND dnode_test_show diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index 06e8d42ad4..eb925d407b 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -1,5 +1,5 @@ /** - * @file vnodeApiTests.cpp + * @file show.cpp * @author slguan (slguan@taosdata.com) * @brief DNODE module show-msg tests * @version 0.1 diff --git a/source/dnode/mgmt/impl/test/stb/CMakeLists.txt b/source/dnode/mgmt/impl/test/stb/CMakeLists.txt new file mode 100644 index 0000000000..524e3e5c7e --- /dev/null +++ b/source/dnode/mgmt/impl/test/stb/CMakeLists.txt @@ -0,0 +1,27 @@ +add_executable(dnode_test_stb "") + +target_sources(dnode_test_stb + PRIVATE + "stb.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dnode_test_stb + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dnode_test_stb + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +add_test( + NAME dnode_test_stb + COMMAND dnode_test_stb +) diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp new file mode 100644 index 0000000000..40af7fbb52 --- /dev/null +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -0,0 +1,479 @@ +/** + * @file stb.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module db-msg tests + * @version 0.1 + * @date 2021-12-17 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "deploy.h" + +class DndTestStb : public ::testing::Test { + protected: + static SServer* CreateServer(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SServer* pServer = createServer(path, fqdn, port, firstEp); + ASSERT(pServer); + return pServer; + } + + static void SetUpTestSuite() { + initLog("/tmp/tdlog"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9101"; + pServer = CreateServer("/tmp/dnode_test_stb", fqdn, 9101, firstEp); + pClient = createClient("root", "taosdata", fqdn, 9101); + taosMsleep(1100); + } + + static void TearDownTestSuite() { + stopServer(pServer); + dropClient(pClient); + pServer = NULL; + pClient = NULL; + } + + static SServer* pServer; + static SClient* pClient; + static int32_t connId; + + public: + void SetUp() override {} + void TearDown() override {} + + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = showType; + if (db != NULL) { + strcpy(pShow->db, db); + } + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + pMeta = &pShowRsp->tableMeta; + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); + pMeta->tuid = htobe64(pMeta->tuid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_STREQ(pMeta->tbFname, showName); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->numOfColumns, columns); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->update, 0); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tuid, 0); + EXPECT_EQ(pMeta->suid, 0); + } + + void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { + SSchema* pSchema = &pMeta->pSchema[index]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, type); + EXPECT_EQ(pSchema->bytes, bytes); + EXPECT_STREQ(pSchema->name, name); + } + + void SendThenCheckShowRetrieveMsg(int32_t rows) { + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + ASSERT_NE(pClient->pRsp->pCont, nullptr); + + pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, rows); + EXPECT_EQ(pRetrieveRsp->offset, 0); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + // EXPECT_EQ(pRetrieveRsp->completed, completed); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->reserved, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + pData = pRetrieveRsp->data; + pos = 0; + } + + void CheckInt8(int8_t val) { + int8_t data = *((int8_t*)(pData + pos)); + pos += sizeof(int8_t); + EXPECT_EQ(data, val); + } + + void CheckInt16(int16_t val) { + int16_t data = *((int16_t*)(pData + pos)); + pos += sizeof(int16_t); + EXPECT_EQ(data, val); + } + + void CheckInt32(int32_t val) { + int32_t data = *((int32_t*)(pData + pos)); + pos += sizeof(int32_t); + EXPECT_EQ(data, val); + } + + void CheckInt64(int64_t val) { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_EQ(data, val); + } + + void CheckTimestamp() { + int64_t data = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(data, 0); + } + + void CheckBinary(const char* val, int32_t len) { + pos += sizeof(VarDataLenT); + char* data = (char*)(pData + pos); + pos += len; + EXPECT_STREQ(data, val); + } + + int32_t showId; + STableMetaMsg* pMeta; + SRetrieveTableRsp* pRetrieveRsp; + char* pData; + int32_t pos; +}; + +SServer* DndTestStb::pServer; +SClient* DndTestStb::pClient; +int32_t DndTestStb::connId; + +TEST_F(DndTestStb, 01_Create_Alter_Drop_Stb) { + { + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + strcpy(pReq->db, "1.d1"); + pReq->numOfVgroups = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRowsPerFileBlock = htonl(100); + pReq->maxRowsPerFileBlock = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replications = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->ignoreExist = 1; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + { + int32_t tags = 2; + int32_t cols = 3; + int32_t size = (tags + cols) * sizeof(SSchema) + sizeof(SCreateStbMsg); + + SCreateStbMsg* pReq = (SCreateStbMsg*)rpcMallocCont(size); + strcpy(pReq->name, "1.d1.stb"); + pReq->numOfTags = htonl(tags); + pReq->numOfColumns = htonl(cols); + + { + SSchema* pSchema = &pReq->pSchema[0]; + pSchema->colId = htonl(0); + pSchema->bytes = htonl(8); + pSchema->type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema->name, "ts"); + } + + { + SSchema* pSchema = &pReq->pSchema[1]; + pSchema->colId = htonl(1); + pSchema->bytes = htonl(4); + pSchema->type = TSDB_DATA_TYPE_INT; + strcpy(pSchema->name, "col1"); + } + + { + SSchema* pSchema = &pReq->pSchema[2]; + pSchema->colId = htonl(2); + pSchema->bytes = htonl(2); + pSchema->type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema->name, "tag1"); + } + + { + SSchema* pSchema = &pReq->pSchema[3]; + pSchema->colId = htonl(3); + pSchema->bytes = htonl(8); + pSchema->type = TSDB_DATA_TYPE_BIGINT; + strcpy(pSchema->name, "tag2"); + } + + { + SSchema* pSchema = &pReq->pSchema[4]; + pSchema->colId = htonl(4); + pSchema->bytes = htonl(16); + pSchema->type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema->name, "tag3"); + } + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateStbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_STB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + taosMsleep(10000); + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show stables", 4, NULL); + CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); + CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(2, TSDB_DATA_TYPE_INT, 2, "columns"); + CheckSchema(3, TSDB_DATA_TYPE_INT, 2, "tags"); + + SendThenCheckShowRetrieveMsg(1); + CheckBinary("stb", TSDB_DB_NAME_LEN - 1); + CheckTimestamp(); + CheckInt16(2); + CheckInt16(3); + +#if 0 + { + SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); + strcpy(pReq->db, "1.d1"); + pReq->totalBlocks = htonl(12); + pReq->daysToKeep0 = htonl(300); + pReq->daysToKeep1 = htonl(400); + pReq->daysToKeep2 = htonl(500); + pReq->fsyncPeriod = htonl(4000); + pReq->walLevel = 2; + pReq->quorum = 2; + pReq->cacheLastRow = 1; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("d1", TSDB_DB_NAME_LEN - 1); + CheckTimestamp(); + CheckInt16(2); // vgroups + CheckInt16(1); // replica + CheckInt16(2); // quorum + CheckInt16(10); // days + CheckBinary("300,400,500", 24); // days + CheckInt32(16); // cache + CheckInt32(12); // blocks + CheckInt32(100); // minrows + CheckInt32(4096); // maxrows + CheckInt8(2); // wallevel + CheckInt32(4000); // fsync + CheckInt8(2); // comp + CheckInt8(1); // cachelast + CheckBinary("ms", 3); // precision + CheckInt8(0); // update + +#endif + + // restart + stopServer(pServer); + pServer = NULL; + + uInfo("start all server"); + + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9101"; + pServer = startServer("/tmp/dnode_test_stb", fqdn, 9101, firstEp); + + uInfo("all server is running"); + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show stables", 4, NULL); + CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); + CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(2, TSDB_DATA_TYPE_INT, 2, "columns"); + CheckSchema(3, TSDB_DATA_TYPE_INT, 2, "tags"); + + SendThenCheckShowRetrieveMsg(1); + CheckBinary("stb", TSDB_DB_NAME_LEN - 1); + CheckTimestamp(); + CheckInt16(2); + CheckInt16(3); + +#if 0 + { + SDropStbMsg* pReq = (SDropStbMsg*)rpcMallocCont(sizeof(SDropStbMsg)); + strcpy(pReq->name, "1.d1.stb"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropStbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_STB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show stables", 4, NULL); + SendThenCheckShowRetrieveMsg(0); +#endif +} + +#if 0 +TEST_F(DndTestStb, 03_Create_Use_Restart_Use_Db) { + { + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + strcpy(pReq->db, "1.d2"); + pReq->numOfVgroups = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRowsPerFileBlock = htonl(100); + pReq->maxRowsPerFileBlock = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replications = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->ignoreExist = 1; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("d2", TSDB_DB_NAME_LEN - 1); + + { + SUseDbMsg* pReq = (SUseDbMsg*)rpcMallocCont(sizeof(SUseDbMsg)); + strcpy(pReq->db, "1.d2"); + pReq->vgVersion = htonl(-1); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SUseDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_USE_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + SUseDbRsp* pRsp = (SUseDbRsp*)pMsg->pCont; + EXPECT_STREQ(pRsp->db, "1.d2"); + pRsp->vgVersion = htonl(pRsp->vgVersion); + pRsp->vgNum = htonl(pRsp->vgNum); + pRsp->hashMethod = pRsp->hashMethod; + EXPECT_EQ(pRsp->vgVersion, 1); + EXPECT_EQ(pRsp->vgNum, 2); + EXPECT_EQ(pRsp->hashMethod, 1); + + { + SVgroupInfo* pInfo = &pRsp->vgroupInfo[0]; + pInfo->vgId = htonl(pInfo->vgId); + pInfo->hashBegin = htonl(pInfo->hashBegin); + pInfo->hashEnd = htonl(pInfo->hashEnd); + EXPECT_GT(pInfo->vgId, 0); + EXPECT_EQ(pInfo->hashBegin, 0); + EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); + EXPECT_EQ(pInfo->inUse, 0); + EXPECT_EQ(pInfo->numOfEps, 1); + SEpAddrMsg* pAddr = &pInfo->epAddr[0]; + pAddr->port = htons(pAddr->port); + EXPECT_EQ(pAddr->port, 9101); + EXPECT_STREQ(pAddr->fqdn, "localhost"); + } + + { + SVgroupInfo* pInfo = &pRsp->vgroupInfo[1]; + pInfo->vgId = htonl(pInfo->vgId); + pInfo->hashBegin = htonl(pInfo->hashBegin); + pInfo->hashEnd = htonl(pInfo->hashEnd); + EXPECT_GT(pInfo->vgId, 0); + EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2); + EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); + EXPECT_EQ(pInfo->inUse, 0); + EXPECT_EQ(pInfo->numOfEps, 1); + SEpAddrMsg* pAddr = &pInfo->epAddr[0]; + pAddr->port = htons(pAddr->port); + EXPECT_EQ(pAddr->port, 9101); + EXPECT_STREQ(pAddr->fqdn, "localhost"); + } + } +} +#endif \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/user/CMakeLists.txt b/source/dnode/mgmt/impl/test/user/CMakeLists.txt index ca8f2ec6db..0d3215a4d0 100644 --- a/source/dnode/mgmt/impl/test/user/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/user/CMakeLists.txt @@ -21,8 +21,6 @@ target_include_directories(dnode_test_user "${CMAKE_CURRENT_SOURCE_DIR}/../sut" ) -enable_testing() - add_test( NAME dnode_test_user COMMAND dnode_test_user diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 74e8e0db44..2b7f70df71 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -1,5 +1,5 @@ /** - * @file vnodeApiTests.cpp + * @file user.cpp * @author slguan (slguan@taosdata.com) * @brief DNODE module user-msg tests * @version 0.1 From ff6453009414c3ecdc8b8b15f39643b8e7a99b2f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 16 Dec 2021 20:16:47 +0800 Subject: [PATCH 16/50] change some error number --- include/util/taoserror.h | 58 ++++++++++------------------ source/dnode/mnode/impl/src/mndStb.c | 43 ++++++++++++--------- source/util/src/terror.c | 23 +++++------ 3 files changed, 55 insertions(+), 69 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4f1ef7da7b..6dea5a4340 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -206,44 +206,28 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0392) // mnode-stable -#define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_IGEXIST TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_COLS_NUM TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_TAGS_NUM TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_COL_TYPE TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_COL_ID TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_COL_BYTES TAOS_DEF_ERROR_CODE(0, 0x0360) -#define TSDB_CODE_MND_STB_INVALID_COL_NAME TAOS_DEF_ERROR_CODE(0, 0x0360) - - -#define TSDB_CODE_MND_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0361) //"Table name too long") -#define TSDB_CODE_MND_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0362) //"Table does not exist") -#define TSDB_CODE_MND_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0363) //"Invalid table type in tsdb") -#define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x0364) //"Too many tags") -#define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x0365) //"Too many columns") -#define TSDB_CODE_MND_TOO_MANY_TIMESERIES TAOS_DEF_ERROR_CODE(0, 0x0366) //"Too many time series") -#define TSDB_CODE_MND_NOT_SUPER_TABLE TAOS_DEF_ERROR_CODE(0, 0x0367) //"Not super table") // operation only available for super table -#define TSDB_CODE_MND_COL_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x0368) //"Tag name too long") -#define TSDB_CODE_MND_TAG_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0369) //"Tag already exists") -#define TSDB_CODE_MND_TAG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036A) //"Tag does not exist") -#define TSDB_CODE_MND_FIELD_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x036B) //"Field already exists") -#define TSDB_CODE_MND_FIELD_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036C) //"Field does not exist") -#define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) //"Super table does not exist") -#define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG TAOS_DEF_ERROR_CODE(0, 0x036E) //"Invalid create table message") -#define TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) //"Exceed max row bytes") - -#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0370) -#define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0371) -#define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0372) -#define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0373) -#define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x0374) -#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x0375) -#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0376) - -#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") -#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") +#define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) +#define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A1) +#define TSDB_CODE_MND_TOO_MANY_STBS TAOS_DEF_ERROR_CODE(0, 0x03A2) +#define TSDB_CODE_MND_INVALID_STB TAOS_DEF_ERROR_CODE(0, 0x03A3) +#define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x03A4) +#define TSDB_CODE_MND_STB_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03A5) +#define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x03A6) +#define TSDB_CODE_MND_TAG_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A7) +#define TSDB_CODE_MND_TAG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A8) +#define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x03A9) +#define TSDB_CODE_MND_COLUMN_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AA) +#define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AB) +#define TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x03AC) +// mnode-func +#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0) +#define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C1) +#define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x03C2) +#define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x03C3) +#define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x03C4) +#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x03C5) +#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x03C6) // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 76edb556fa..0f10e853cd 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -157,11 +157,17 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb atomic_exchange_32(&pOldStb->version, pNewStb->version); taosWLockLatch(&pOldStb->lock); + pOldStb->numOfColumns = pNewStb->numOfColumns; + pOldStb->numOfTags = pNewStb->numOfTags; int32_t totalCols = pNewStb->numOfTags + pNewStb->numOfColumns; int32_t totalSize = totalCols * sizeof(SSchema); if (pOldStb->numOfTags + pOldStb->numOfColumns < totalCols) { - pOldStb->pSchema = malloc(totalSize); + void *pSchema = malloc(totalSize); + if (pSchema != NULL) { + free(pOldStb->pSchema); + pOldStb->pSchema = pSchema; + } } memcpy(pOldStb->pSchema, pNewStb->pSchema, totalSize); @@ -200,37 +206,37 @@ static int32_t mndCheckStbMsg(SCreateStbMsg *pCreate) { } if (pCreate->igExists < 0 || pCreate->igExists > 1) { - terrno = TSDB_CODE_MND_STB_INVALID_IGEXIST; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pCreate->numOfColumns < TSDB_MIN_COLUMNS || pCreate->numOfColumns > TSDB_MAX_COLUMNS) { - terrno = TSDB_CODE_MND_STB_INVALID_COLS_NUM; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pCreate->numOfTags <= 0 || pCreate->numOfTags > TSDB_MAX_TAGS) { - terrno = TSDB_CODE_MND_STB_INVALID_TAGS_NUM; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } int32_t maxColId = (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS); for (int32_t i = 0; i < totalCols; ++i) { SSchema *pSchema = &pCreate->pSchema[i]; - if (pSchema->type <= 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_TYPE; + if (pSchema->type < 0) { + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->colId < 0 || pSchema->colId >= maxColId) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_ID; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->bytes <= 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_BYTES; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->name[0] == 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_NAME; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } } @@ -245,6 +251,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre stbObj.createdTime = taosGetTimestampMs(); stbObj.updateTime = stbObj.createdTime; stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + stbObj.dbUid = pDb->uid; stbObj.version = 1; stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfTags = pCreate->numOfTags; @@ -350,19 +357,19 @@ static int32_t mndCheckAlterStbMsg(SAlterStbMsg *pAlter) { pSchema->bytes = htonl(pSchema->bytes); if (pSchema->type <= 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_TYPE; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->colId < 0 || pSchema->colId >= (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS)) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_ID; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->bytes <= 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_BYTES; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } if (pSchema->name[0] == 0) { - terrno = TSDB_CODE_MND_STB_INVALID_COL_NAME; + terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -495,7 +502,7 @@ static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { SStbObj *pStb = mndAcquireStb(pMnode, pInfo->name); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_MND_INVALID_TABLE_NAME; + terrno = TSDB_CODE_MND_INVALID_STB; mError("stb:%s, failed to get meta since %s", pInfo->name, terrstr()); return -1; } @@ -583,14 +590,14 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "columns"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "tags"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 70a3dc622f..0deac5f07e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -217,22 +217,20 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist // mnode-stable TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "Stable already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, "Table name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, "Table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, "Invalid table type in tsdb") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_NOT_EXIST, "Stable not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STBS, "Too many stables") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB, "Invalid stable name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_OPTION, "Invalid stable options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_OPTION_UNCHNAGED, "Stable options not changed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, "Too many time series") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, "Not super table") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, "Tag name too long") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST, "Tag already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST, "Field already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, "Field does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREAY_EXIST, "Column already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES, "Exceed max row bytes") +// mnode-func TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_NOT_EXIST, "Func not exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func") @@ -241,9 +239,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment" TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length") - // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_EXITING, "Dnode is exiting") From 48507aecc0c94226bdbe53050b63ba26547b6bcc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Dec 2021 11:22:31 +0800 Subject: [PATCH 17/50] [td-11818] support show databases; --- include/client/taos.h | 4 +- include/common/taosmsg.h | 5 +- include/common/tmsgtype.h | 2 +- include/os/os.h | 1 + include/os/osSysinfo.h | 12 ++- include/util/tmacro.h | 42 -------- source/client/inc/clientInt.h | 14 ++- source/client/src/clientImpl.c | 82 ++++++++++---- source/client/src/clientMain.c | 131 +++++++++++++++++++++-- source/client/src/clientMsgHandler.c | 79 +++++++++++--- source/client/src/tscEnv.c | 8 +- source/client/test/clientTests.cpp | 60 +++++++++-- source/dnode/mnode/impl/src/mndCluster.c | 8 +- source/dnode/mnode/impl/src/mndDb.c | 38 +++---- source/dnode/mnode/impl/src/mndDnode.c | 30 +++--- source/dnode/mnode/impl/src/mndFunc.c | 16 +-- source/dnode/mnode/impl/src/mndMnode.c | 18 ++-- source/dnode/mnode/impl/src/mndProfile.c | 74 ++++++------- source/dnode/mnode/impl/src/mndStb.c | 12 +-- source/dnode/mnode/impl/src/mndSync.c | 36 ------- source/dnode/mnode/impl/src/mndUser.c | 8 +- source/dnode/mnode/impl/src/mndVgroup.c | 16 +-- source/libs/parser/src/parser.c | 13 +-- source/libs/query/src/querymsg.c | 4 +- source/os/src/osSysinfo.c | 10 +- src/client/src/tscAsync.c | 6 +- src/client/src/tscLocal.c | 2 +- src/client/src/tscServer.c | 10 +- src/client/src/tscSql.c | 6 +- src/client/src/tscSubquery.c | 2 +- 30 files changed, 466 insertions(+), 283 deletions(-) delete mode 100644 include/util/tmacro.h delete mode 100644 source/dnode/mnode/impl/src/mndSync.c diff --git a/include/client/taos.h b/include/client/taos.h index eb3332ed18..7357478555 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -80,8 +80,8 @@ typedef enum { typedef struct taosField { char name[65]; - uint8_t type; - int16_t bytes; + int8_t type; + int32_t bytes; } TAOS_FIELD; #ifdef _TD_GO_DLL_ diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 9f5dc0c666..7abeb97b27 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -572,14 +572,13 @@ typedef struct { } SRetrieveTableMsg; typedef struct SRetrieveTableRsp { - int32_t numOfRows; - int64_t offset; // updated offset value for multi-vnode projection query int64_t useconds; int8_t completed; // all results are returned to client int8_t precision; int8_t compressed; - int8_t reserved; int32_t compLen; + + int32_t numOfRows; char data[]; } SRetrieveTableRsp; diff --git a/include/common/tmsgtype.h b/include/common/tmsgtype.h index d357ca1f47..1fb10ae15b 100644 --- a/include/common/tmsgtype.h +++ b/include/common/tmsgtype.h @@ -62,7 +62,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_DNODE, "cfg-dnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_MNODE, "cfg-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SHOW, "show" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE, "retrieve" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_MNODE, "retrieve" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_QUERY, "kill-query" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_STREAM, "kill-stream" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_CONNECTION, "kill-connection" ) diff --git a/include/os/os.h b/include/os/os.h index 53a6cef96a..de2a8182db 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -46,6 +46,7 @@ extern "C" { #include #include #include +#include #include #include diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 56f6b3e0da..6952b91742 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "os.h" + #define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 96 @@ -57,11 +59,11 @@ char * taosGetCmdlineByPID(int pid); void taosSetCoreDump(bool enable); typedef struct { - const char *sysname; - const char *nodename; - const char *release; - const char *version; - const char *machine; + char sysname[_UTSNAME_MACHINE_LENGTH]; + char nodename[_UTSNAME_MACHINE_LENGTH]; + char release[_UTSNAME_MACHINE_LENGTH]; + char version[_UTSNAME_MACHINE_LENGTH]; + char machine[_UTSNAME_MACHINE_LENGTH]; } SysNameInfo; SysNameInfo taosGetSysNameInfo(); diff --git a/include/util/tmacro.h b/include/util/tmacro.h deleted file mode 100644 index 5cca8a1062..0000000000 --- a/include/util/tmacro.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_MACRO_H_ -#define _TD_UTIL_MACRO_H_ - -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Module init/clear MACRO definitions -#define TD_MOD_UNINITIALIZED 0 -#define TD_MOD_INITIALIZED 1 - -#define TD_MOD_UNCLEARD 0 -#define TD_MOD_CLEARD 1 - -typedef int8_t td_mode_flag_t; - -#define TD_CHECK_AND_SET_MODE_INIT(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED) - -#define TD_CHECK_AND_SET_MOD_CLEAR(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNCLEARD, TD_MOD_CLEARD) - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_MACRO_H_*/ \ No newline at end of file diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 87951d7944..9ef1d67e74 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,8 +20,8 @@ extern "C" { #endif -#include #include "taos.h" +#include "common.h" #include "taosmsg.h" #include "tdef.h" #include "tep.h" @@ -88,9 +88,15 @@ typedef struct STscObj { } STscObj; typedef struct SClientResultInfo { - SSDataBlock *pData; - TAOS_FIELD *resultFields; + const char *pMsg; + const char *pData; + TAOS_FIELD *fields; + int32_t numOfCols; + int32_t numOfRows; int32_t current; + int32_t *length; + TAOS_ROW row; + char **pCol; } SClientResultInfo; typedef struct SReqBody { @@ -98,6 +104,7 @@ typedef struct SReqBody { void* fp; void* param; int32_t paramLen; + int64_t execId; // showId/queryId SClientResultInfo* pResInfo; } SRequestBody; @@ -152,6 +159,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); void* doFetchRow(SRequestObj* pRequest); +void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); #ifdef __cplusplus } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index fb98a7da46..4815e862e8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -12,7 +12,7 @@ static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody); -static void destroyConnectMsg(SRequestMsgBody* pMsgBody); +static void destroyRequestMsgBody(SRequestMsgBody* pMsgBody); static int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId); @@ -99,17 +99,19 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, char* key = getClusterKey(user, secretEncrypt, ip, port); - SAppInstInfo* pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); + SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL) { - pInst = calloc(1, sizeof(struct SAppInstInfo)); + SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); - pInst->mgmtEp = epSet; - pInst->pTransporter = openTransporter(user, secretEncrypt); + p->mgmtEp = epSet; + p->pTransporter = openTransporter(user, secretEncrypt); + taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); - taosHashPut(appInfo.pInstMap, key, strlen(key), &pInst, POINTER_BYTES); + pInst = &p; } - return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, pInst); + tfree(key); + return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, *pInst); } TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { @@ -163,7 +165,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); tsem_wait(&pRequest->body.rspSem); - destroyConnectMsg(&body); + destroyRequestMsgBody(&body); } else { assert(0); } @@ -234,7 +236,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); tsem_wait(&pRequest->body.rspSem); - destroyConnectMsg(&body); + destroyRequestMsgBody(&body); if (pRequest->code != TSDB_CODE_SUCCESS) { const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); @@ -281,7 +283,7 @@ static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) return 0; } -static void destroyConnectMsg(SRequestMsgBody* pMsgBody) { +static void destroyRequestMsgBody(SRequestMsgBody* pMsgBody) { assert(pMsgBody != NULL); tfree(pMsgBody->pData); } @@ -337,7 +339,14 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%"PRId64 " ms", pRequest->requestId, taosMsg[pMsg->msgType], tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); if (handleRequestRspFp[pRequest->type]) { - pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen); + char *p = malloc(pMsg->contLen); + if (p == NULL) { + pRequest->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = pRequest->code; + } else { + memcpy(p, pMsg->pCont, pMsg->contLen); + pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, p, pMsg->contLen); + } } } else { tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d, elapsed time:%"PRId64" ms", pRequest->requestId, taosMsg[pMsg->msgType], @@ -381,15 +390,48 @@ void* doFetchRow(SRequestObj* pRequest) { assert(pRequest != NULL); SClientResultInfo* pResultInfo = pRequest->body.pResInfo; - if (pResultInfo == NULL || pResultInfo->current >= pResultInfo->pData->info.rows) { - if (pResultInfo == NULL) { - pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); -// pRequest->body.pResInfo. + if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { + pRequest->type = TSDB_SQL_RETRIEVE_MNODE; + + SRequestMsgBody body = {0}; + buildRequestMsgFp[pRequest->type](pRequest, &body); + + int64_t transporterId = 0; + STscObj* pTscObj = pRequest->pTscObj; + sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); + + tsem_wait(&pRequest->body.rspSem); + destroyRequestMsgBody(&body); + + pResultInfo->current = 0; + if (pResultInfo->numOfRows <= pResultInfo->current) { + return NULL; } - // current data set are exhausted, fetch more result from node -// if (pRes->row >= pRes->numOfRows && needToFetchNewBlock(pSql)) { -// taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj); -// tsem_wait(&pSql->rspSem); -// } + } + + for(int32_t i = 0; i < pResultInfo->numOfCols; ++i) { + pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current; + if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { + pResultInfo->length[i] = varDataLen(pResultInfo->row[i]); + pResultInfo->row[i] = varDataVal(pResultInfo->row[i]); + } + } + + pResultInfo->current += 1; + return pResultInfo->row; +} + +void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { + assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); + if (numOfRows == 0) { + return; + } + + int32_t offset = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + pResultInfo->length[i] = pResultInfo->fields[i].bytes; + pResultInfo->row[i] = pResultInfo->pData + offset * pResultInfo->numOfRows; + pResultInfo->pCol[i] = pResultInfo->row[i]; + offset += pResultInfo->fields[i].bytes; } } \ No newline at end of file diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 5a1f55e573..e3ec7c27ee 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -28,7 +28,6 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { } int ret = taos_options_imp(option, (const char*)arg); - atomic_store_32(&lock, 0); return ret; } @@ -58,18 +57,18 @@ void taos_cleanup(void) { } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - int32_t p = (port != 0)? port:tsServerPort; + int32_t p = (port != 0) ? port : tsServerPort; - tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); - if (user == NULL) { - user = TSDB_DEFAULT_USER; - } + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } - if (pass == NULL) { - pass = TSDB_DEFAULT_PASS; - } + if (pass == NULL) { + pass = TSDB_DEFAULT_PASS; + } - return taos_connect_internal(ip, user, pass, NULL, db, p); + return taos_connect_internal(ip, user, pass, NULL, db, p); } void taos_close(TAOS* taos) { @@ -110,6 +109,34 @@ void taos_free_result(TAOS_RES *res) { destroyRequest(pRequest); } +int taos_field_count(TAOS_RES *res) { + if (res == NULL) { + return 0; + } + + SRequestObj* pRequest = (SRequestObj*) res; + + SClientResultInfo* pResInfo = pRequest->body.pResInfo; + if (pResInfo == NULL) { + return 0; + } + + return pResInfo->numOfCols; +} + +int taos_num_fields(TAOS_RES *res) { + return taos_field_count(res); +} + +TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { + if (taos_num_fields(res) == 0) { + return NULL; + } + + SClientResultInfo* pResInfo = ((SRequestObj*) res)->body.pResInfo; + return pResInfo->fields; +} + TAOS_RES *taos_query(TAOS *taos, const char *sql) { if (taos == NULL || sql == NULL) { return NULL; @@ -131,3 +158,87 @@ TAOS_ROW taos_fetch_row(TAOS_RES *pRes) { return doFetchRow(pRequest); } + +int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { + int32_t len = 0; + for (int i = 0; i < num_fields; ++i) { + if (i > 0) { + str[len++] = ' '; + } + + if (row[i] == NULL) { + len += sprintf(str + len, "%s", TSDB_DATA_NULL_STR); + continue; + } + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(str + len, "%u", *((uint8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(str + len, "%d", *((int16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(str + len, "%u", *((uint16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_INT: + len += sprintf(str + len, "%d", *((int32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UINT: + len += sprintf(str + len, "%u", *((uint32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_FLOAT: { + float fv = 0; + fv = GET_FLOAT_VAL(row[i]); + len += sprintf(str + len, "%f", fv); + } break; + + case TSDB_DATA_TYPE_DOUBLE: { + double dv = 0; + dv = GET_DOUBLE_VAL(row[i]); + len += sprintf(str + len, "%lf", dv); + } break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE); + if (fields[i].type == TSDB_DATA_TYPE_BINARY) { + assert(charLen <= fields[i].bytes && charLen >= 0); + } else { + assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0); + } + + memcpy(str + len, row[i], charLen); + len += charLen; + } break; + + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BOOL: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + default: + break; + } + } + + return len; +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index d860d56f75..646964e319 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -529,7 +529,7 @@ int doBuildAndSendMsg(SSqlObj *pSql) { if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || @@ -2700,7 +2700,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { } STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if ((pCmd->command == TSDB_SQL_RETRIEVE) || + if ((pCmd->command == TSDB_SQL_RETRIEVE_MNODE) || ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && @@ -3158,6 +3158,9 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { pTscObj->pAppInfo->clusterId = pConnect->clusterId; atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); + pRequest->body.pResInfo->pMsg = pMsg; + tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); return 0; } @@ -3217,26 +3220,70 @@ int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) SSchema* pSchema = pMetaMsg->pSchema; pMetaMsg->tuid = htobe64(pMetaMsg->tuid); for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); pSchema++; } - STableMeta* pTableMeta = createTableMetaFromMsg(pMetaMsg); - SSchema *pTableSchema = pTableMeta->schema; - - TAOS_FIELD* pFields = calloc(1, pTableMeta->tableInfo.numOfColumns); - for (int16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; ++i, ++pSchema) { - tstrncpy(pFields[i].name, pTableSchema[i].name, tListLen(pFields[i].name)); - pFields[i].type = pTableSchema[i].type; - pFields[i].bytes = pTableSchema[i].bytes; + pSchema = pMetaMsg->pSchema; + TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); + for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) { + tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name)); + pFields[i].type = pSchema[i].type; + pFields[i].bytes = pSchema[i].bytes; } -// pRequest->body.resultFields = pFields; -// pRequest->body.numOfFields = pTableMeta->tableInfo.numOfColumns; + if (pRequest->body.pResInfo == NULL) { + pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); + } + pRequest->body.pResInfo->pMsg = pMsg; + SClientResultInfo* pResInfo = pRequest->body.pResInfo; + + pResInfo->fields = pFields; + pResInfo->numOfCols = pMetaMsg->numOfColumns; + pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); + + pRequest->body.execId = pShow->showId; return 0; } +int buildRetrieveMnodeMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + pMsgBody->msgLen = sizeof(SRetrieveTableMsg); + pMsgBody->requestObjRefId = pRequest->self; + + SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg)); + pRetrieveMsg->showId = htonl(pRequest->body.execId); + + pMsgBody->pData = pRetrieveMsg; + return TSDB_CODE_SUCCESS; +} + +int32_t processRetrieveMnodeRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + assert(msgLen >= sizeof(SRetrieveTableRsp)); + + tfree(pRequest->body.pResInfo->pMsg); + pRequest->body.pResInfo->pMsg = pMsg; + + SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg; + pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); + pRetrieve->precision = htons(pRetrieve->precision); + + SClientResultInfo* pResInfo = pRequest->body.pResInfo; + pResInfo->numOfRows = pRetrieve->numOfRows; + pResInfo->pData = pRetrieve->data; // todo fix this in async model + + pResInfo->current = 0; + setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); + + tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pRetrieve->numOfRows, + pRetrieve->completed, pRequest->body.execId); + return 0; +} + + void initMsgHandleFp() { #if 0 tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; @@ -3273,7 +3320,7 @@ void initMsgHandleFp() { tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg; tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; @@ -3291,7 +3338,7 @@ void initMsgHandleFp() { tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; @@ -3321,4 +3368,6 @@ void initMsgHandleFp() { buildRequestMsgFp[TSDB_SQL_SHOW] = buildShowMsg; handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp; + buildRequestMsgFp[TSDB_SQL_RETRIEVE_MNODE] = buildRetrieveMnodeMsg; + handleRequestRspFp[TSDB_SQL_RETRIEVE_MNODE]= processRetrieveMnodeRsp; } \ No newline at end of file diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index e378ebede4..023bd6ebe9 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -127,8 +127,6 @@ void destroyTscObj(void *pObj) { atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); - - closeTransporter(pTscObj); pthread_mutex_destroy(&pTscObj->mutex); tfree(pTscObj); } @@ -190,6 +188,12 @@ static void doDestroyRequest(void* p) { tfree(pRequest->sqlstr); tfree(pRequest->pInfo); + if (pRequest->body.pResInfo != NULL) { + tfree(pRequest->body.pResInfo->pData); + tfree(pRequest->body.pResInfo->pMsg); + tfree(pRequest->body.pResInfo); + } + deregisterRequest(pRequest); tfree(pRequest); } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 62d2cb5b18..46fd76234e 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -23,6 +23,7 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" +#include "../inc/clientInt.h" #include "taos.h" namespace { @@ -34,19 +35,62 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { + taos_init(); +} + +TEST(testCase, connect_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + taos_close(pConn); +} + +TEST(testCase, create_user_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); -// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +//TEST(testCase, show_user_Test) { +// TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show users"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); // } // -// taos_free_result(pRes); +// taos_close(pConn); +//} - TAOS_RES* pRes = taos_query(pConn, "show users"); - TAOS_ROW pRow = taos_fetch_row(pRes); - assert(pRow != NULL); +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show databases"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } taos_close(pConn); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 0eaa184907..7c53a4ebd0 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -143,22 +143,22 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); strcpy(pMeta->tbFname, mndShowStr(pShow->type)); pShow->numOfColumns = cols; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9f2b2d17f3..010b1abb78 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -709,100 +709,100 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "replica"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "quorum"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "days"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "keep0,keep1,keep2"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cache(MB)"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "cache"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "blocks"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "minrows"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "maxrows"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "wallevel"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "fsync"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "comp"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "cachelast"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 3 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "precision"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "update"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index ec5f68a713..32c1689589 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -552,16 +552,16 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -629,46 +629,46 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "vnodes"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "max vnodes"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "max_vnodes"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "offline reason"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "offline_reason"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b7158bb094..a513c9d57d 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -380,46 +380,46 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = PATH_MAX + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "comment"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "aggregate"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "outputtype"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "code_len"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "bufsize"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 7b6804d43e..16733cbf94 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -368,34 +368,34 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "role"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "role time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "role_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 45a63f2dc5..bf657fd27a 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -474,48 +474,48 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; // app name pShow->bytes[cols] = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "program"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; // app pid pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "pid"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "login_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "last_access"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -602,88 +602,88 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "queryId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 24; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "qid"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "created_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; strcpy(pSchema[cols].name, "time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sql_obj_id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "pid"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ep"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_BOOL; strcpy(pSchema[cols].name, "stable_query"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "sub_queries"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sub_query_info"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sql"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -818,64 +818,64 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "streamId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "dest table"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "destination"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "exec time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "exec"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; strcpy(pSchema[cols].name, "time(us)"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sql"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "cycles"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 791b6f5d12..e3b64b8b5c 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -570,28 +570,28 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "columns"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "tags"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c deleted file mode 100644 index 59161b32f2..0000000000 --- a/source/dnode/mnode/impl/src/mndSync.c +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" -#include "mndTrans.h" - -int32_t mndInitSync(SMnode *pMnode) { return 0; } -void mndCleanupSync(SMnode *pMnode) {} - -int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) { - int32_t code = 0; - - // int32_t len = sdbGetRawTotalSize(pRaw); - // SSdbRaw *pReceived = calloc(1, len); - // memcpy(pReceived, pRaw, len); - // mDebug("trans:%d, data:%p recv from sync, code:0x%x pMsg:%p", pMsg->id, pReceived, code & 0xFFFF, pMsg); - - // mndTransApply(pMnode, pReceived, code); - return code; -} - -bool mndIsMaster(SMnode *pMnode) { return true; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5fa4fe5359..c97e1ff7d6 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -410,25 +410,25 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "privilege"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "account"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pMeta->numOfColumns = htonl(cols); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 9a462c024b..fd711f7109 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -203,30 +203,30 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "vgId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "tables"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; for (int32_t i = 0; i < pShow->replica; ++i) { pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; } - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -309,16 +309,16 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "vgId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 1d883974b8..5e08859a66 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -32,12 +32,6 @@ bool qIsInsertSql(const char* pStr, size_t length) { } int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) { - SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); - if (pQueryInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. - return terrno; - } - SSqlInfo info = doGenerateAST(pStr); if (!info.valid) { strncpy(msg, info.msg, msgLen); @@ -51,6 +45,12 @@ int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *typ // do nothing } } else { + SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); + if (pQueryInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. + return terrno; + } + struct SCatalog* pCatalog = NULL; int32_t code = catalogGetHandle(NULL, &pCatalog); code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, id, msg, msgLen); @@ -59,6 +59,7 @@ int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *typ } } + destroySqlInfo(&info); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/query/src/querymsg.c b/source/libs/query/src/querymsg.c index 9d99b568a5..5b26d5bd02 100644 --- a/source/libs/query/src/querymsg.c +++ b/source/libs/query/src/querymsg.c @@ -305,7 +305,7 @@ void msgInit() { tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg; tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; @@ -323,7 +323,7 @@ void msgInit() { tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index f892b4d8c0..cb231e15a0 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -1120,11 +1120,11 @@ SysNameInfo taosGetSysNameInfo() { struct utsname uts; if (!uname(&uts)) { - info.sysname = strdup(uts.sysname); - info.nodename = strdup(uts.nodename); - info.release = strdup(uts.release); - info.version = strdup(uts.version); - info.machine = strdup(uts.machine); + tstrncpy(info.sysname, uts.sysname, sizeof(info.sysname)); + tstrncpy(info.nodename, uts.nodename, sizeof(info.nodename)); + tstrncpy(info.release, uts.release, sizeof(info.release)); + tstrncpy(info.version, uts.version, sizeof(info.version)); + tstrncpy(info.machine, uts.machine, sizeof(info.machine)); } return info; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 4a621d47c0..b2f8eda474 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -183,7 +183,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo pSql->fp = fp; if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; } if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { @@ -265,7 +265,7 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { } return; - } else if (pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE) { + } else if (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE) { // in case of show command, return no data (*pSql->fetchFp)(param, pSql, 0); } else { @@ -273,7 +273,7 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { } } else { // current query is not completed, continue retrieve from node if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; } SQueryInfo* pQueryInfo1 = tscGetQueryInfo(&pSql->cmd); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 0b4f399a1f..dcfb2d6a87 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -322,7 +322,7 @@ TAOS_ROW tscFetchRow(void *param) { // current data set are exhausted, fetch more data from node if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && - (pCmd->command == TSDB_SQL_RETRIEVE || + (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE || pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || pCmd->command == TSDB_SQL_FETCH || diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index caa334aaed..7a754ee698 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -539,7 +539,7 @@ int doBuildAndSendMsg(SSqlObj *pSql) { if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || @@ -2749,7 +2749,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { } STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if ((pCmd->command == TSDB_SQL_RETRIEVE) || + if ((pCmd->command == TSDB_SQL_RETRIEVE_MNODE) || ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && @@ -3174,7 +3174,7 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg; tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; @@ -3192,7 +3192,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; @@ -3214,7 +3214,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; tscKeepConn[TSDB_SQL_SHOW] = 1; - tscKeepConn[TSDB_SQL_RETRIEVE] = 1; + tscKeepConn[TSDB_SQL_RETRIEVE_MNODE] = 1; tscKeepConn[TSDB_SQL_SELECT] = 1; tscKeepConn[TSDB_SQL_FETCH] = 1; tscKeepConn[TSDB_SQL_HB] = 1; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 5fdaad0d66..ab1fffd5a2 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -460,7 +460,7 @@ static bool needToFetchNewBlock(SSqlObj* pSql) { SSqlCmd *pCmd = &pSql->cmd; return (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && - (pCmd->command == TSDB_SQL_RETRIEVE || + (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE || pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || pCmd->command == TSDB_SQL_FETCH || @@ -582,10 +582,10 @@ static bool tscKillQueryInDnode(SSqlObj* pSql) { if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) && (cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_SHOW || - cmd == TSDB_SQL_RETRIEVE || + cmd == TSDB_SQL_RETRIEVE_MNODE || cmd == TSDB_SQL_FETCH)) { pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; tscDebug("0x%"PRIx64" send msg to dnode to free qhandle ASAP before free sqlObj, command:%s", pSql->self, sqlCmd[pCmd->command]); tscBuildAndSendRequest(pSql, NULL); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 275042a238..24a6377a73 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1670,7 +1670,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { pSql1->fp = joinRetrieveFinalResCallback; if (pCmd1->command < TSDB_SQL_LOCAL) { - pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; } tscBuildAndSendRequest(pSql1, NULL); From 8f7a109851dc5ea5fa27c3aed67d1e75192660ef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Dec 2021 11:48:23 +0800 Subject: [PATCH 18/50] [td-11818] add new api implementation. --- source/client/src/clientMain.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index e3ec7c27ee..21e632db8d 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -242,3 +242,28 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) return len; } + +int* taos_fetch_lengths(TAOS_RES *res) { + if (res == NULL) { + return NULL; + } + + return ((SRequestObj*) res)->body.pResInfo->length; +} + +const char *taos_data_type(int type) { + switch (type) { + case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL"; + case TSDB_DATA_TYPE_BOOL: return "TSDB_DATA_TYPE_BOOL"; + case TSDB_DATA_TYPE_TINYINT: return "TSDB_DATA_TYPE_TINYINT"; + case TSDB_DATA_TYPE_SMALLINT: return "TSDB_DATA_TYPE_SMALLINT"; + case TSDB_DATA_TYPE_INT: return "TSDB_DATA_TYPE_INT"; + case TSDB_DATA_TYPE_BIGINT: return "TSDB_DATA_TYPE_BIGINT"; + case TSDB_DATA_TYPE_FLOAT: return "TSDB_DATA_TYPE_FLOAT"; + case TSDB_DATA_TYPE_DOUBLE: return "TSDB_DATA_TYPE_DOUBLE"; + case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_BINARY"; + case TSDB_DATA_TYPE_TIMESTAMP: return "TSDB_DATA_TYPE_TIMESTAMP"; + case TSDB_DATA_TYPE_NCHAR: return "TSDB_DATA_TYPE_NCHAR"; + default: return "UNKNOWN"; + } +} From 602f4509b4791beb1a8475c56a9014a828b3f488 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Dec 2021 12:00:54 +0800 Subject: [PATCH 19/50] TD-10431 process stable meta msg --- include/common/taosmsg.h | 20 +- include/libs/query/query.h | 2 - source/dnode/mgmt/impl/src/dndTransport.c | 2 +- source/dnode/mgmt/impl/test/stb/stb.cpp | 212 ++++++---------------- source/dnode/mnode/impl/src/mndStb.c | 44 +++-- source/libs/query/src/querymsg.c | 2 +- 6 files changed, 87 insertions(+), 195 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index eb2adda394..39107e0b4f 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -785,12 +785,8 @@ typedef struct { } SAuthVnodeMsg; typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; -} SStbInfoMsg; - -typedef struct { - SMsgHead msgHead; - char tableFname[TSDB_TABLE_FNAME_LEN]; + int32_t vgId; + char tableFname[TSDB_TABLE_FNAME_LEN]; } STableInfoMsg; typedef struct { @@ -801,10 +797,6 @@ typedef struct { char tableNames[]; } SMultiTableInfoMsg; -typedef struct SSTableVgroupMsg { - int32_t numOfTables; -} SSTableVgroupMsg, SSTableVgroupRspMsg; - typedef struct SVgroupInfo { int32_t vgId; uint32_t hashBegin; @@ -814,12 +806,6 @@ typedef struct SVgroupInfo { SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; } SVgroupInfo; -typedef struct SVgroupListRspMsg { - int32_t vgroupNum; - int32_t vgroupVersion; - SVgroupInfo vgroupInfo[]; -} SVgroupListRspMsg; - typedef struct { int32_t vgId; int8_t numOfEps; @@ -841,8 +827,8 @@ typedef struct { int8_t update; int32_t sversion; int32_t tversion; - uint64_t tuid; uint64_t suid; + uint64_t tuid; int32_t vgId; SSchema pSchema[]; } STableMetaMsg; diff --git a/include/libs/query/query.h b/include/libs/query/query.h index 8720fd085c..fc0ea1729b 100644 --- a/include/libs/query/query.h +++ b/include/libs/query/query.h @@ -23,8 +23,6 @@ extern "C" { #include "tarray.h" #include "thash.h" -typedef SVgroupListRspMsg SVgroupListInfo; - typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema uint8_t precision; // the number of precision diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 6dc46cefcd..17cf39b019 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -38,7 +38,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TABLE] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TABLE] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_TABLE_META] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_TABLE_META] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_TABLES_META] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_QUERY] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg; diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp index 40af7fbb52..a6addbe5a9 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -179,7 +179,7 @@ SServer* DndTestStb::pServer; SClient* DndTestStb::pClient; int32_t DndTestStb::connId; -TEST_F(DndTestStb, 01_Create_Alter_Drop_Stb) { +TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { { SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); strcpy(pReq->db, "1.d1"); @@ -215,8 +215,8 @@ TEST_F(DndTestStb, 01_Create_Alter_Drop_Stb) { } { - int32_t tags = 2; - int32_t cols = 3; + int32_t cols = 2; + int32_t tags = 3; int32_t size = (tags + cols) * sizeof(SSchema) + sizeof(SCreateStbMsg); SCreateStbMsg* pReq = (SCreateStbMsg*)rpcMallocCont(size); @@ -266,7 +266,7 @@ TEST_F(DndTestStb, 01_Create_Alter_Drop_Stb) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateStbMsg); + rpcMsg.contLen = size; rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_STB; sendMsg(pClient, &rpcMsg); @@ -274,66 +274,70 @@ TEST_F(DndTestStb, 01_Create_Alter_Drop_Stb) { ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); } - taosMsleep(10000); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show stables", 4, NULL); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); - CheckSchema(2, TSDB_DATA_TYPE_INT, 2, "columns"); - CheckSchema(3, TSDB_DATA_TYPE_INT, 2, "tags"); + CheckSchema(2, TSDB_DATA_TYPE_INT, 4, "columns"); + CheckSchema(3, TSDB_DATA_TYPE_INT, 4, "tags"); SendThenCheckShowRetrieveMsg(1); - CheckBinary("stb", TSDB_DB_NAME_LEN - 1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); CheckTimestamp(); - CheckInt16(2); - CheckInt16(3); + CheckInt32(2); + CheckInt32(3); -#if 0 + // ----- meta ------ { - SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); - strcpy(pReq->db, "1.d1"); - pReq->totalBlocks = htonl(12); - pReq->daysToKeep0 = htonl(300); - pReq->daysToKeep1 = htonl(400); - pReq->daysToKeep2 = htonl(500); - pReq->fsyncPeriod = htonl(4000); - pReq->walLevel = 2; - pReq->quorum = 2; - pReq->cacheLastRow = 1; + STableInfoMsg* pReq = (STableInfoMsg*)rpcMallocCont(sizeof(STableInfoMsg)); + strcpy(pReq->tableFname, "1.d1.stb"); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_DB; + rpcMsg.contLen = sizeof(STableInfoMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_TABLE_META; sendMsg(pClient, &rpcMsg); SRpcMsg* pMsg = pClient->pRsp; ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); + + STableMetaMsg* pRsp = (STableMetaMsg*)pMsg->pCont; + pRsp->numOfTags = htonl(pRsp->numOfTags); + pRsp->numOfColumns = htonl(pRsp->numOfColumns); + pRsp->sversion = htonl(pRsp->sversion); + pRsp->tversion = htonl(pRsp->tversion); + pRsp->suid = htobe64(pRsp->suid); + pRsp->tuid = htobe64(pRsp->tuid); + pRsp->vgId = htobe64(pRsp->vgId); + for (int32_t i = 0; i < pRsp->numOfTags + pRsp->numOfColumns; ++i) { + SSchema* pSchema = &pRsp->pSchema[i]; + pSchema->colId = htonl(pSchema->colId); + pSchema->bytes = htonl(pSchema->bytes); + } + + EXPECT_STREQ(pRsp->tbFname, ""); + EXPECT_STREQ(pRsp->stbFname, "1.d1.stb"); + EXPECT_EQ(pRsp->numOfColumns, 2); + EXPECT_EQ(pRsp->numOfTags, 3); + EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRsp->tableType, TSDB_SUPER_TABLE); + EXPECT_EQ(pRsp->update, 0); + EXPECT_EQ(pRsp->sversion, 1); + EXPECT_EQ(pRsp->tversion, 0); + EXPECT_GT(pRsp->suid, 0); + EXPECT_EQ(pRsp->tuid, 0); + EXPECT_EQ(pRsp->vgId, 0); + + { + SSchema* pSchema = &pRsp->pSchema[0]; + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->bytes, 8); + EXPECT_STREQ(pSchema->name, "ts"); + } } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(1); - CheckBinary("d1", TSDB_DB_NAME_LEN - 1); - CheckTimestamp(); - CheckInt16(2); // vgroups - CheckInt16(1); // replica - CheckInt16(2); // quorum - CheckInt16(10); // days - CheckBinary("300,400,500", 24); // days - CheckInt32(16); // cache - CheckInt32(12); // blocks - CheckInt32(100); // minrows - CheckInt32(4096); // maxrows - CheckInt8(2); // wallevel - CheckInt32(4000); // fsync - CheckInt8(2); // comp - CheckInt8(1); // cachelast - CheckBinary("ms", 3); // precision - CheckInt8(0); // update - -#endif - // restart stopServer(pServer); pServer = NULL; @@ -346,19 +350,13 @@ TEST_F(DndTestStb, 01_Create_Alter_Drop_Stb) { uInfo("all server is running"); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show stables", 4, NULL); - CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); - CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); - CheckSchema(2, TSDB_DATA_TYPE_INT, 2, "columns"); - CheckSchema(3, TSDB_DATA_TYPE_INT, 2, "tags"); - + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); SendThenCheckShowRetrieveMsg(1); - CheckBinary("stb", TSDB_DB_NAME_LEN - 1); + CheckBinary("stb", TSDB_TABLE_NAME_LEN); CheckTimestamp(); - CheckInt16(2); - CheckInt16(3); + CheckInt32(2); + CheckInt32(3); -#if 0 { SDropStbMsg* pReq = (SDropStbMsg*)rpcMallocCont(sizeof(SDropStbMsg)); strcpy(pReq->name, "1.d1.stb"); @@ -374,106 +372,6 @@ TEST_F(DndTestStb, 01_Create_Alter_Drop_Stb) { ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show stables", 4, NULL); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); SendThenCheckShowRetrieveMsg(0); -#endif } - -#if 0 -TEST_F(DndTestStb, 03_Create_Use_Restart_Use_Db) { - { - SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); - strcpy(pReq->db, "1.d2"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - } - - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); - SendThenCheckShowRetrieveMsg(1); - CheckBinary("d2", TSDB_DB_NAME_LEN - 1); - - { - SUseDbMsg* pReq = (SUseDbMsg*)rpcMallocCont(sizeof(SUseDbMsg)); - strcpy(pReq->db, "1.d2"); - pReq->vgVersion = htonl(-1); - - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SUseDbMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_USE_DB; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - - SUseDbRsp* pRsp = (SUseDbRsp*)pMsg->pCont; - EXPECT_STREQ(pRsp->db, "1.d2"); - pRsp->vgVersion = htonl(pRsp->vgVersion); - pRsp->vgNum = htonl(pRsp->vgNum); - pRsp->hashMethod = pRsp->hashMethod; - EXPECT_EQ(pRsp->vgVersion, 1); - EXPECT_EQ(pRsp->vgNum, 2); - EXPECT_EQ(pRsp->hashMethod, 1); - - { - SVgroupInfo* pInfo = &pRsp->vgroupInfo[0]; - pInfo->vgId = htonl(pInfo->vgId); - pInfo->hashBegin = htonl(pInfo->hashBegin); - pInfo->hashEnd = htonl(pInfo->hashEnd); - EXPECT_GT(pInfo->vgId, 0); - EXPECT_EQ(pInfo->hashBegin, 0); - EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); - EXPECT_EQ(pInfo->inUse, 0); - EXPECT_EQ(pInfo->numOfEps, 1); - SEpAddrMsg* pAddr = &pInfo->epAddr[0]; - pAddr->port = htons(pAddr->port); - EXPECT_EQ(pAddr->port, 9101); - EXPECT_STREQ(pAddr->fqdn, "localhost"); - } - - { - SVgroupInfo* pInfo = &pRsp->vgroupInfo[1]; - pInfo->vgId = htonl(pInfo->vgId); - pInfo->hashBegin = htonl(pInfo->hashBegin); - pInfo->hashEnd = htonl(pInfo->hashEnd); - EXPECT_GT(pInfo->vgId, 0); - EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2); - EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); - EXPECT_EQ(pInfo->inUse, 0); - EXPECT_EQ(pInfo->numOfEps, 1); - SEpAddrMsg* pAddr = &pInfo->epAddr[0]; - pAddr->port = htons(pAddr->port); - EXPECT_EQ(pAddr->port, 9101); - EXPECT_STREQ(pAddr->fqdn, "localhost"); - } - } -} -#endif \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 0f10e853cd..0ecd42aada 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -80,7 +80,7 @@ static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT64(pRaw, dataPos, pStb->updateTime) SDB_SET_INT64(pRaw, dataPos, pStb->uid) SDB_SET_INT64(pRaw, dataPos, pStb->dbUid) - SDB_SET_INT64(pRaw, dataPos, pStb->version) + SDB_SET_INT32(pRaw, dataPos, pStb->version) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns) SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags) @@ -487,31 +487,37 @@ static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SStbInfoMsg *pInfo = pMsg->rpcMsg.pCont; + SMnode *pMnode = pMsg->pMnode; + STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; - mDebug("stb:%s, start to retrieve meta", pInfo->name); + mDebug("stb:%s, start to retrieve meta", pInfo->tableFname); - SDbObj *pDb = mndAcquireDbByStb(pMnode, pInfo->name); + SDbObj *pDb = mndAcquireDbByStb(pMnode, pInfo->tableFname); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("stb:%s, failed to retrieve meta since %s", pInfo->name, terrstr()); + mError("stb:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr()); return -1; } - SStbObj *pStb = mndAcquireStb(pMnode, pInfo->name); + SStbObj *pStb = mndAcquireStb(pMnode, pInfo->tableFname); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); terrno = TSDB_CODE_MND_INVALID_STB; - mError("stb:%s, failed to get meta since %s", pInfo->name, terrstr()); + mError("stb:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); return -1; } - int32_t contLen = sizeof(STableMetaMsg) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); + taosRLockLatch(&pStb->lock); + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + int32_t contLen = sizeof(STableMetaMsg) + totalCols * sizeof(SSchema); + STableMetaMsg *pMeta = rpcMallocCont(contLen); if (pMeta == NULL) { + taosRUnLockLatch(&pStb->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("stb:%s, failed to get meta since %s", pInfo->name, terrstr()); + mError("stb:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); return -1; } @@ -524,7 +530,7 @@ static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { pMeta->sversion = htonl(pStb->version); pMeta->suid = htonl(pStb->uid); - for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + for (int32_t i = 0; i < totalCols; ++i) { SSchema *pSchema = &pMeta->pSchema[i]; SSchema *pSrcSchema = &pStb->pSchema[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); @@ -532,11 +538,14 @@ static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) { pSchema->colId = htonl(pSrcSchema->colId); pSchema->bytes = htonl(pSrcSchema->bytes); } + taosRUnLockLatch(&pStb->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); pMsg->pCont = pMeta; pMsg->contLen = contLen; - mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", pInfo->name, pStb->numOfColumns, pStb->numOfTags); + mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pStb->numOfColumns, pStb->numOfTags); return 0; } @@ -553,7 +562,7 @@ static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs void *pIter = NULL; while (1) { SStbObj *pStb = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pStb); + pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb); if (pIter == NULL) break; if (strcmp(pStb->db, dbName) == 0) { @@ -610,6 +619,7 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; } + pShow->numOfRows = sdbGetSize(pSdb, SDB_STB); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; strcpy(pMeta->tbFname, mndShowStr(pShow->type)); @@ -653,8 +663,8 @@ static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 cols = 0; - char stbName[TSDB_TABLE_FNAME_LEN] = {0}; - memcpy(stbName, pStb->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen); + char stbName[TSDB_TABLE_NAME_LEN] = {0}; + tstrncpy(stbName, pStb->name + prefixLen, TSDB_TABLE_NAME_LEN); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; STR_TO_VARSTR(pWrite, stbName); cols++; @@ -664,11 +674,11 @@ static int32_t mndRetrieveStb(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pStb->numOfColumns; + *(int32_t *)pWrite = pStb->numOfColumns; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pStb->numOfTags; + *(int32_t *)pWrite = pStb->numOfTags; cols++; numOfRows++; diff --git a/source/libs/query/src/querymsg.c b/source/libs/query/src/querymsg.c index 9d99b568a5..486c9718a2 100644 --- a/source/libs/query/src/querymsg.c +++ b/source/libs/query/src/querymsg.c @@ -39,7 +39,7 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 STableInfoMsg *bMsg = (STableInfoMsg *)*msg; - bMsg->msgHead.vgId = bInput->vgId; + bMsg->vgId = bInput->vgId; strncpy(bMsg->tableFname, bInput->tableFullName, sizeof(bMsg->tableFname)); bMsg->tableFname[sizeof(bMsg->tableFname) - 1] = 0; From d037b24e059f0ae25e6e987cf4c770b4abe30ecd Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 17 Dec 2021 13:40:27 +0800 Subject: [PATCH 20/50] refactor wal --- include/common/taosmsg.h | 5 +- include/dnode/vnode/tq/tq.h | 13 ++--- source/dnode/mgmt/impl/src/dndTransport.c | 3 +- source/dnode/vnode/impl/inc/vnodeStateMgr.h | 8 +-- source/dnode/vnode/impl/src/vnodeWrite.c | 23 +++++++-- source/dnode/vnode/tq/inc/tqInt.h | 20 ++++---- source/dnode/vnode/tq/inc/tqMetaStore.h | 27 ++++------ source/dnode/vnode/tq/src/tq.c | 8 ++- source/libs/wal/inc/walInt.h | 56 ++++++++++----------- source/libs/wal/src/walMeta.c | 14 +++--- source/libs/wal/src/walMgmt.c | 2 +- source/libs/wal/src/walRead.c | 12 ++--- source/libs/wal/src/walSeek.c | 12 ++--- source/libs/wal/src/walWrite.c | 32 ++++++------ 14 files changed, 123 insertions(+), 112 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 8abfe0ffed..1cc6c3b5a2 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -50,10 +50,9 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONSUME, "mq-consume" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_ACK, "mq-ack" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_RESET, "mq-reset" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET, "mq-set" ) // message from client to mnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_ACCT, "alter-acct" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_ACCT, "drop-acct" ) diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index d6f7b46870..7993a8f1ab 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -62,10 +62,10 @@ typedef struct TmqDisconnectRsp { int8_t status; } TmqDisconnectRsp; -typedef struct TmqConsumeReq { +typedef struct STqConsumeReq { TmqMsgHead head; TmqAcks acks; -} TmqConsumeReq; +} STqConsumeReq; typedef struct TmqMsgContent { int64_t topicId; @@ -73,11 +73,11 @@ typedef struct TmqMsgContent { char msg[]; } TmqMsgContent; -typedef struct TmqConsumeRsp { +typedef struct STqConsumeRsp { TmqMsgHead head; int64_t bodySize; TmqMsgContent msgs[]; -} TmqConsumeRsp; +} STqConsumeRsp; typedef struct TmqSubscribeReq { TmqMsgHead head; @@ -261,13 +261,14 @@ typedef struct STQ { // open in each vnode STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac); -void tqDestroy(STQ*); +void tqClose(STQ*); // void* will be replace by a msg type int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); +int tqSetCursor(STQ*, void* msg); -int tqConsume(STQ*, TmqConsumeReq*); +int tqConsume(STQ*, STqConsumeReq*); STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 6dc46cefcd..1db92644ae 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -44,8 +44,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_ACK] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_RESET] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET] = dndProcessVnodeWriteMsg; // msg from client to mnode pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg; diff --git a/source/dnode/vnode/impl/inc/vnodeStateMgr.h b/source/dnode/vnode/impl/inc/vnodeStateMgr.h index 788426e25e..5862b304ed 100644 --- a/source/dnode/vnode/impl/inc/vnodeStateMgr.h +++ b/source/dnode/vnode/impl/inc/vnodeStateMgr.h @@ -21,13 +21,13 @@ extern "C" { #endif typedef struct { - uint64_t processed; - uint64_t committed; - uint64_t applied; + int64_t processed; + int64_t committed; + int64_t applied; } SVState; #ifdef __cplusplus } #endif -#endif /*_TD_VNODE_STATE_MGR_H_*/ \ No newline at end of file +#endif /*_TD_VNODE_STATE_MGR_H_*/ diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index 85460e8d91..85e044266a 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -15,16 +15,31 @@ #include "vnodeDef.h" +int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { + SVnodeReq *pVnodeReq; + + switch (pMsg->msgType) { + case TSDB_MSG_TYPE_MQ_SET: + if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) { + // TODO: handle error + } + break; + } + + void *pBuf = pMsg->pCont; + return 0; +} + int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { - SRpcMsg * pMsg; + SRpcMsg *pMsg; SVnodeReq *pVnodeReq; for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void * pBuf = pMsg->pCont; - uint64_t ver = pVnode->state.processed++; + void *pBuf = pMsg->pCont; + int64_t ver = pVnode->state.processed++; taosEncodeFixedU64(&pBuf, ver); if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { @@ -99,4 +114,4 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { return 0; } -/* ------------------------ STATIC METHODS ------------------------ */ \ No newline at end of file +/* ------------------------ STATIC METHODS ------------------------ */ diff --git a/source/dnode/vnode/tq/inc/tqInt.h b/source/dnode/vnode/tq/inc/tqInt.h index 100149c0ea..022b599816 100644 --- a/source/dnode/vnode/tq/inc/tqInt.h +++ b/source/dnode/vnode/tq/inc/tqInt.h @@ -22,16 +22,16 @@ extern "C" { #endif -//create persistent storage for meta info such as consuming offset -//return value > 0: cgId -//return value <= 0: error code -//int tqCreateTCGroup(STQ*, const char* topic, int cgId, tqBufferHandle** handle); -//create ring buffer in memory and load consuming offset -//int tqOpenTCGroup(STQ*, const char* topic, int cgId); -//destroy ring buffer and persist consuming offset -//int tqCloseTCGroup(STQ*, const char* topic, int cgId); -//delete persistent storage for meta info -//int tqDropTCGroup(STQ*, const char* topic, int cgId); +// create persistent storage for meta info such as consuming offset +// return value > 0: cgId +// return value <= 0: error code +// int tqCreateTCGroup(STQ*, const char* topic, int cgId, tqBufferHandle** handle); +// create ring buffer in memory and load consuming offset +// int tqOpenTCGroup(STQ*, const char* topic, int cgId); +// destroy ring buffer and persist consuming offset +// int tqCloseTCGroup(STQ*, const char* topic, int cgId); +// delete persistent storage for meta info +// int tqDropTCGroup(STQ*, const char* topic, int cgId); #ifdef __cplusplus } diff --git a/source/dnode/vnode/tq/inc/tqMetaStore.h b/source/dnode/vnode/tq/inc/tqMetaStore.h index 253852b00f..5bcedaed74 100644 --- a/source/dnode/vnode/tq/inc/tqMetaStore.h +++ b/source/dnode/vnode/tq/inc/tqMetaStore.h @@ -23,27 +23,22 @@ extern "C" { #endif - -STqMetaStore* tqStoreOpen(const char* path, - FTqSerialize pSerializer, - FTqDeserialize pDeserializer, - FTqDelete pDeleter, - int32_t tqConfigFlag - ); +STqMetaStore* tqStoreOpen(const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, FTqDelete pDeleter, + int32_t tqConfigFlag); int32_t tqStoreClose(STqMetaStore*); -//int32_t tqStoreDelete(TqMetaStore*); -//int32_t tqStoreCommitAll(TqMetaStore*); -int32_t tqStorePersist(STqMetaStore*); -//clean deleted idx and data from persistent file -int32_t tqStoreCompact(STqMetaStore*); +// int32_t tqStoreDelete(TqMetaStore*); +// int32_t tqStoreCommitAll(TqMetaStore*); +int32_t tqStorePersist(STqMetaStore*); +// clean deleted idx and data from persistent file +int32_t tqStoreCompact(STqMetaStore*); -void* tqHandleGet(STqMetaStore*, int64_t key); -//make it unpersist +void* tqHandleGet(STqMetaStore*, int64_t key); +// make it unpersist void* tqHandleTouchGet(STqMetaStore*, int64_t key); int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value); int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize); -//delete committed kv pair -//notice that a delete action still needs to be committed +// delete committed kv pair +// notice that a delete action still needs to be committed int32_t tqHandleDel(STqMetaStore*, int64_t key); int32_t tqHandleCommit(STqMetaStore*, int64_t key); int32_t tqHandleAbort(STqMetaStore*, int64_t key); diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 1326666857..2c07529219 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -214,7 +214,11 @@ int tqCommit(STQ* pTq) { return 0; } -int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) { +int tqSetCursor(STQ* pTq, void* msg) { + return 0; +} + +int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { if (!tqProtoCheck((TmqMsgHead*)pMsg)) { // proto version invalid return -1; @@ -232,7 +236,7 @@ int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) { } } - TmqConsumeRsp* pRsp = (TmqConsumeRsp*)pMsg; + STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) { // fetch error diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 819afcc411..1579cad7b6 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -16,72 +16,70 @@ #ifndef _TD_WAL_INT_H_ #define _TD_WAL_INT_H_ -#include "wal.h" #include "compare.h" #include "tchecksum.h" +#include "wal.h" #ifdef __cplusplus extern "C" { #endif -//meta section begin +// meta section begin typedef struct WalFileInfo { int64_t firstVer; int64_t lastVer; int64_t createTs; int64_t closeTs; int64_t fileSize; -} WalFileInfo; +} SWalFileInfo; typedef struct WalIdxEntry { int64_t ver; int64_t offset; -} WalIdxEntry; +} SWalIdxEntry; static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) { - WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft; - WalFileInfo* pInfoRight = (WalFileInfo*)pRight; + SWalFileInfo* pInfoLeft = (SWalFileInfo*)pLeft; + SWalFileInfo* pInfoRight = (SWalFileInfo*)pRight; return compareInt64Val(&pInfoLeft->firstVer, &pInfoRight->firstVer); } static inline int64_t walGetLastFileSize(SWal* pWal) { - WalFileInfo* pInfo = (WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); return pInfo->fileSize; } static inline int64_t walGetLastFileFirstVer(SWal* pWal) { - WalFileInfo* pInfo = (WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); return pInfo->firstVer; } static inline int64_t walGetCurFileFirstVer(SWal* pWal) { - WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); return pInfo->firstVer; } static inline int64_t walGetCurFileLastVer(SWal* pWal) { - WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); return pInfo->firstVer; } static inline int64_t walGetCurFileOffset(SWal* pWal) { - WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); return pInfo->fileSize; } -static inline bool walCurFileClosed(SWal* pWal) { - return taosArrayGetSize(pWal->fileInfoSet) != pWal->writeCur; +static inline bool walCurFileClosed(SWal* pWal) { return taosArrayGetSize(pWal->fileInfoSet) != pWal->writeCur; } + +static inline SWalFileInfo* walGetCurFileInfo(SWal* pWal) { + return (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); } -static inline WalFileInfo* walGetCurFileInfo(SWal* pWal) { - return (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); -} - -static inline int walBuildLogName(SWal*pWal, int64_t fileFirstVer, char* buf) { +static inline int walBuildLogName(SWal* pWal, int64_t fileFirstVer, char* buf) { return sprintf(buf, "%s/%020" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer); } -static inline int walBuildIdxName(SWal*pWal, int64_t fileFirstVer, char* buf) { +static inline int walBuildIdxName(SWal* pWal, int64_t fileFirstVer, char* buf) { return sprintf(buf, "%s/%020" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); } @@ -93,11 +91,11 @@ static inline int walValidBodyCksum(SWalHead* pHead) { return taosCheckChecksum((uint8_t*)pHead->head.body, pHead->head.len, pHead->cksumBody); } -static inline int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) { +static inline int walValidCksum(SWalHead* pHead, void* body, int64_t bodyLen) { return walValidHeadCksum(pHead) && walValidBodyCksum(pHead); } -static inline uint32_t walCalcHeadCksum(SWalHead *pHead) { +static inline uint32_t walCalcHeadCksum(SWalHead* pHead) { return taosCalcChecksum(0, (uint8_t*)&pHead->head, sizeof(SWalReadHead)); } @@ -106,7 +104,7 @@ static inline uint32_t walCalcBodyCksum(const void* body, uint32_t len) { } static inline int64_t walGetVerIdxOffset(SWal* pWal, int64_t ver) { - return (ver - walGetCurFileFirstVer(pWal)) * sizeof(WalIdxEntry); + return (ver - walGetCurFileFirstVer(pWal)) * sizeof(SWalIdxEntry); } static inline void walResetVer(SWalVer* pVer) { @@ -126,16 +124,16 @@ int walCheckAndRepairMeta(SWal* pWal); int walCheckAndRepairIdx(SWal* pWal); char* walMetaSerialize(SWal* pWal); -int walMetaDeserialize(SWal* pWal, const char* bytes); -//meta section end +int walMetaDeserialize(SWal* pWal, const char* bytes); +// meta section end -//seek section -int walChangeFile(SWal *pWal, int64_t ver); -//seek section end +// seek section +int walChangeFile(SWal* pWal, int64_t ver); +// seek section end int64_t walGetSeq(); -int walSeekVer(SWal *pWal, int64_t ver); -int walRoll(SWal *pWal); +int walSeekVer(SWal* pWal, int64_t ver); +int walRoll(SWal* pWal); #ifdef __cplusplus } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index e86a2b6221..0f155f9553 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -79,13 +79,13 @@ int walRollFileInfo(SWal* pWal) { SArray* pArray = pWal->fileInfoSet; if (taosArrayGetSize(pArray) != 0) { - WalFileInfo* pInfo = taosArrayGetLast(pArray); + SWalFileInfo* pInfo = taosArrayGetLast(pArray); pInfo->lastVer = pWal->vers.lastVer; pInfo->closeTs = ts; } // TODO: change to emplace back - WalFileInfo* pNewInfo = malloc(sizeof(WalFileInfo)); + SWalFileInfo* pNewInfo = malloc(sizeof(SWalFileInfo)); if (pNewInfo == NULL) { return -1; } @@ -122,9 +122,9 @@ char* walMetaSerialize(SWal* pWal) { cJSON_AddStringToObject(pMeta, "lastVer", buf); cJSON_AddItemToObject(pRoot, "files", pFiles); - WalFileInfo* pData = pWal->fileInfoSet->pData; + SWalFileInfo* pData = pWal->fileInfoSet->pData; for (int i = 0; i < sz; i++) { - WalFileInfo* pInfo = &pData[i]; + SWalFileInfo* pInfo = &pData[i]; cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject()); if (pField == NULL) { cJSON_Delete(pRoot); @@ -167,10 +167,10 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { // deserialize SArray* pArray = pWal->fileInfoSet; taosArrayEnsureCap(pArray, sz); - WalFileInfo* pData = pArray->pData; + SWalFileInfo* pData = pArray->pData; for (int i = 0; i < sz; i++) { - cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); - WalFileInfo* pInfo = &pData[i]; + cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + SWalFileInfo* pInfo = &pData[i]; pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); pInfo->firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "lastVer"); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 5b6a8c6b29..189881c86d 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -92,7 +92,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeLogTfd = -1; pWal->writeIdxTfd = -1; pWal->writeCur = -1; - pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo)); + pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); free(pWal); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index b6e232fa5c..48eb84b536 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -52,13 +52,13 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i int64_t logTfd = pRead->readLogTfd; // seek position - int64_t offset = (ver - fileFirstVer) * sizeof(WalIdxEntry); + int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry); code = tfLseek(idxTfd, offset, SEEK_SET); if (code < 0) { return -1; } - WalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { + SWalIdxEntry entry; + if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { return -1; } // TODO:deserialize @@ -105,10 +105,10 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { if (ver < pWal->vers.snapshotVer) { } - WalFileInfo tmpInfo; + SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; // bsearch in fileSet - WalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); ASSERT(pRet != NULL); if (pRead->curFileFirstVer != pRet->firstVer) { code = walReadChangeFile(pRead, pRet->firstVer); @@ -159,9 +159,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { return -1; } - /*code = walValidBodyCksum(pRead->pHead);*/ ASSERT(pRead->pHead->head.version == ver); + code = walValidBodyCksum(pRead->pHead); if (code != 0) { return -1; } diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 82c596d225..1f9b7b6e58 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -32,9 +32,9 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { if (code != 0) { return -1; } - WalIdxEntry entry; + SWalIdxEntry entry; // TODO:deserialize - code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry)); + code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)); if (code != 0) { return -1; } @@ -47,8 +47,8 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { } int walChangeFileToLast(SWal* pWal) { - int64_t idxTfd, logTfd; - WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); + int64_t idxTfd, logTfd; + SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer; @@ -83,10 +83,10 @@ int walChangeFile(SWal* pWal, int64_t ver) { // TODO return -1; } - WalFileInfo tmpInfo; + SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; // bsearch in fileSet - WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + SWalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer; // closed diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index c3a7ca5f4d..c8ffd9d07d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -56,9 +56,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // delete files int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); for (int i = pWal->writeCur; i < fileSetSize; i++) { - walBuildLogName(pWal, ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + walBuildLogName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); remove(fnameStr); - walBuildIdxName(pWal, ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + walBuildIdxName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); remove(fnameStr); } // pop from fileInfoSet @@ -81,8 +81,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } // read idx file and get log file pos // TODO:change to deserialize function - WalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { + SWalIdxEntry entry; + if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { pthread_mutex_unlock(&pWal->mutex); return -1; } @@ -128,8 +128,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { return -1; } pWal->vers.lastVer = ver - 1; - ((WalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; - ((WalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; + ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; + ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; // unlock pthread_mutex_unlock(&pWal->mutex); @@ -153,17 +153,17 @@ int32_t walEndSnapshot(SWal *pWal) { pWal->vers.snapshotVer = ver; int ts = taosGetTimestampSec(); - int deleteCnt = 0; - int64_t newTotSize = pWal->totSize; - WalFileInfo tmp; + int deleteCnt = 0; + int64_t newTotSize = pWal->totSize; + SWalFileInfo tmp; tmp.firstVer = ver; // find files safe to delete - WalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); + SWalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); if (ver >= pInfo->lastVer) { pInfo++; } // iterate files, until the searched result - for (WalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { + for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { if (pWal->totSize > pWal->cfg.retentionSize || iter->closeTs + pWal->cfg.retentionPeriod > ts) { // delete according to file size or close time deleteCnt++; @@ -173,7 +173,7 @@ int32_t walEndSnapshot(SWal *pWal) { char fnameStr[WAL_FILE_LEN]; // remove file for (int i = 0; i < deleteCnt; i++) { - WalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i); + SWalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i); walBuildLogName(pWal, pInfo->firstVer, fnameStr); remove(fnameStr); walBuildIdxName(pWal, pInfo->firstVer, fnameStr); @@ -186,7 +186,7 @@ int32_t walEndSnapshot(SWal *pWal) { pWal->writeCur = -1; pWal->vers.firstVer = -1; } else { - pWal->vers.firstVer = ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + pWal->vers.firstVer = ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; } pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; ; @@ -248,9 +248,9 @@ int walRoll(SWal *pWal) { } static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { - WalIdxEntry entry = {.ver = ver, .offset = offset}; - int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry)); - if (size != sizeof(WalIdxEntry)) { + SWalIdxEntry entry = {.ver = ver, .offset = offset}; + int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry)); + if (size != sizeof(SWalIdxEntry)) { // TODO truncate return -1; } From c604393e5d84cbdd4247256c380d7725ac1bf8a3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Dec 2021 13:44:11 +0800 Subject: [PATCH 21/50] change file directory --- CMakeLists.txt | 1 + {src/kit => tools}/CMakeLists.txt | 0 {src/kit => tools}/shell/CMakeLists.txt | 0 {src/kit => tools}/shell/inc/shell.h | 0 {src/kit => tools}/shell/inc/shellCommand.h | 0 {src/kit => tools}/shell/inc/tnettest.h | 0 {src/kit => tools}/shell/src/shellCheck.c | 0 {src/kit => tools}/shell/src/shellCommand.c | 0 {src/kit => tools}/shell/src/shellDarwin.c | 0 {src/kit => tools}/shell/src/shellEngine.c | 0 {src/kit => tools}/shell/src/shellImport.c | 0 {src/kit => tools}/shell/src/shellLinux.c | 0 {src/kit => tools}/shell/src/shellMain.c | 0 {src/kit => tools}/shell/src/shellWindows.c | 0 {src/kit => tools}/shell/src/tnettest.c | 0 15 files changed, 1 insertion(+) rename {src/kit => tools}/CMakeLists.txt (100%) rename {src/kit => tools}/shell/CMakeLists.txt (100%) rename {src/kit => tools}/shell/inc/shell.h (100%) rename {src/kit => tools}/shell/inc/shellCommand.h (100%) rename {src/kit => tools}/shell/inc/tnettest.h (100%) rename {src/kit => tools}/shell/src/shellCheck.c (100%) rename {src/kit => tools}/shell/src/shellCommand.c (100%) rename {src/kit => tools}/shell/src/shellDarwin.c (100%) rename {src/kit => tools}/shell/src/shellEngine.c (100%) rename {src/kit => tools}/shell/src/shellImport.c (100%) rename {src/kit => tools}/shell/src/shellLinux.c (100%) rename {src/kit => tools}/shell/src/shellMain.c (100%) rename {src/kit => tools}/shell/src/shellWindows.c (100%) rename {src/kit => tools}/shell/src/tnettest.c (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a8cb1d710..681559a37b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,7 @@ if(${BUILD_TEST}) enable_testing() endif(${BUILD_TEST}) add_subdirectory(source) +add_subdirectory(tools) # docs add_subdirectory(docs) diff --git a/src/kit/CMakeLists.txt b/tools/CMakeLists.txt similarity index 100% rename from src/kit/CMakeLists.txt rename to tools/CMakeLists.txt diff --git a/src/kit/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt similarity index 100% rename from src/kit/shell/CMakeLists.txt rename to tools/shell/CMakeLists.txt diff --git a/src/kit/shell/inc/shell.h b/tools/shell/inc/shell.h similarity index 100% rename from src/kit/shell/inc/shell.h rename to tools/shell/inc/shell.h diff --git a/src/kit/shell/inc/shellCommand.h b/tools/shell/inc/shellCommand.h similarity index 100% rename from src/kit/shell/inc/shellCommand.h rename to tools/shell/inc/shellCommand.h diff --git a/src/kit/shell/inc/tnettest.h b/tools/shell/inc/tnettest.h similarity index 100% rename from src/kit/shell/inc/tnettest.h rename to tools/shell/inc/tnettest.h diff --git a/src/kit/shell/src/shellCheck.c b/tools/shell/src/shellCheck.c similarity index 100% rename from src/kit/shell/src/shellCheck.c rename to tools/shell/src/shellCheck.c diff --git a/src/kit/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c similarity index 100% rename from src/kit/shell/src/shellCommand.c rename to tools/shell/src/shellCommand.c diff --git a/src/kit/shell/src/shellDarwin.c b/tools/shell/src/shellDarwin.c similarity index 100% rename from src/kit/shell/src/shellDarwin.c rename to tools/shell/src/shellDarwin.c diff --git a/src/kit/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c similarity index 100% rename from src/kit/shell/src/shellEngine.c rename to tools/shell/src/shellEngine.c diff --git a/src/kit/shell/src/shellImport.c b/tools/shell/src/shellImport.c similarity index 100% rename from src/kit/shell/src/shellImport.c rename to tools/shell/src/shellImport.c diff --git a/src/kit/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c similarity index 100% rename from src/kit/shell/src/shellLinux.c rename to tools/shell/src/shellLinux.c diff --git a/src/kit/shell/src/shellMain.c b/tools/shell/src/shellMain.c similarity index 100% rename from src/kit/shell/src/shellMain.c rename to tools/shell/src/shellMain.c diff --git a/src/kit/shell/src/shellWindows.c b/tools/shell/src/shellWindows.c similarity index 100% rename from src/kit/shell/src/shellWindows.c rename to tools/shell/src/shellWindows.c diff --git a/src/kit/shell/src/tnettest.c b/tools/shell/src/tnettest.c similarity index 100% rename from src/kit/shell/src/tnettest.c rename to tools/shell/src/tnettest.c From f57407f3029a205246857b46347a40becf6312ea Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 17 Dec 2021 13:47:58 +0800 Subject: [PATCH 22/50] make it compile --- include/util/tmacro.h | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 include/util/tmacro.h diff --git a/include/util/tmacro.h b/include/util/tmacro.h new file mode 100644 index 0000000000..5cca8a1062 --- /dev/null +++ b/include/util/tmacro.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_MACRO_H_ +#define _TD_UTIL_MACRO_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Module init/clear MACRO definitions +#define TD_MOD_UNINITIALIZED 0 +#define TD_MOD_INITIALIZED 1 + +#define TD_MOD_UNCLEARD 0 +#define TD_MOD_CLEARD 1 + +typedef int8_t td_mode_flag_t; + +#define TD_CHECK_AND_SET_MODE_INIT(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED) + +#define TD_CHECK_AND_SET_MOD_CLEAR(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNCLEARD, TD_MOD_CLEARD) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_MACRO_H_*/ \ No newline at end of file From f991a732d925d1c3cc505da2c619198926bb1df8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Dec 2021 13:50:32 +0800 Subject: [PATCH 23/50] minor changes --- tools/CMakeLists.txt | 8 +---- tools/shell/CMakeLists.txt | 63 +++++++------------------------------- 2 files changed, 12 insertions(+), 59 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index fdf58d5ae1..629677c9a5 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,7 +1 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -ADD_SUBDIRECTORY(shell) -ADD_SUBDIRECTORY(taosdemo) -ADD_SUBDIRECTORY(taosdump) -ADD_SUBDIRECTORY(taospack) +add_subdirectory(shell) \ No newline at end of file diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index bf2bbca14d..9216bfcb6c 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -1,54 +1,13 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) +aux_source_directory(src SHELL_SRC) +list(REMOVE_ITEM SHELL_SRC ./src/shellWindows.c) +list(REMOVE_ITEM SHELL_SRC ./src/shellDarwin.c) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) -INCLUDE_DIRECTORIES(inc) - -IF (TD_LINUX) - AUX_SOURCE_DIRECTORY(./src SRC) - LIST(REMOVE_ITEM SRC ./src/shellWindows.c) - LIST(REMOVE_ITEM SRC ./src/shellDarwin.c) - ADD_EXECUTABLE(shell ${SRC}) - -IF (TD_LINUX_64 AND JEMALLOC_ENABLED) - ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) - SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") -ELSE () - SET(LINK_JEMALLOC "") -ENDIF () - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(shell taos_static lua ${LINK_JEMALLOC}) - ELSE () - TARGET_LINK_LIBRARIES(shell taos lua ${LINK_JEMALLOC}) - ENDIF () - - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) -ELSEIF (TD_WINDOWS) - LIST(APPEND SRC ./src/shellEngine.c) - LIST(APPEND SRC ./src/shellMain.c) - LIST(APPEND SRC ./src/shellWindows.c) - ADD_EXECUTABLE(shell ${SRC}) - TARGET_LINK_LIBRARIES(shell taos_static) - - IF (TD_POWER) - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME power) - ELSE () - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) - ENDIF () -ELSEIF (TD_DARWIN) - LIST(APPEND SRC ./src/shellEngine.c) - LIST(APPEND SRC ./src/shellMain.c) - LIST(APPEND SRC ./src/shellDarwin.c) - LIST(APPEND SRC ./src/shellCommand.c) - LIST(APPEND SRC ./src/shellImport.c) - LIST(APPEND SRC ./src/shellCheck.c) - ADD_EXECUTABLE(shell ${SRC}) - # linking with dylib - TARGET_LINK_LIBRARIES(shell taos) - # linking taos statically - # TARGET_LINK_LIBRARIES(shell taos_static) - SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) -ENDIF () +add_executable(shell ${SHELL_SRC}) +target_link_libraries( + shell + PUBLIC taos + PUBLIC util + PUBLIC os +) +SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) From d7ed95530b5bbe2df42157c3bffea29d6269a822 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 17 Dec 2021 01:06:50 -0500 Subject: [PATCH 24/50] TD-12193 physical plan to json --- include/libs/planner/planner.h | 30 +- include/libs/planner/plannerOp.h | 10 +- source/libs/parser/src/insertParser.c | 16 +- source/libs/planner/CMakeLists.txt | 2 +- source/libs/planner/inc/plannerInt.h | 8 +- source/libs/planner/src/physicalPlan.c | 4 - source/libs/planner/src/physicalPlanJson.c | 410 +++++++++++++++++++++ source/libs/planner/src/planner.c | 8 + 8 files changed, 460 insertions(+), 28 deletions(-) create mode 100644 source/libs/planner/src/physicalPlanJson.c diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index a7d418d45e..28d9a04513 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -34,6 +34,13 @@ enum OPERATOR_TYPE_E { OP_TotalNum }; +enum DATASINK_TYPE_E { + DSINK_Unknown, + DSINK_Dispatch, + DSINK_Insert, + DSINK_TotalNum +}; + struct SEpSet; struct SQueryStmtInfo; @@ -49,6 +56,22 @@ typedef struct SQueryNodeBasicInfo { const char *name; // operator name } SQueryNodeBasicInfo; +typedef struct SDataSink { + SQueryNodeBasicInfo info; + SDataBlockSchema schema; +} SDataSink; + +typedef struct SDataDispatcher { + SDataSink sink; + // todo +} SDataDispatcher; + +typedef struct SDataInserter { + SDataSink sink; + uint64_t uid; // unique id of the table + // todo data field +} SDataInserter; + typedef struct SPhyNode { SQueryNodeBasicInfo info; SArray *pTargets; // target list to be computed or scanned at this node @@ -113,15 +136,16 @@ typedef struct SQueryDag { */ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); -int32_t qSetSuplanExecutionNode(SArray* subplans, SArray* nodes); +int32_t qSetSuplanExecutionNode(SSubplan* subplan, SArray* nodes); int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); - /** * Convert to subplan to string for the scheduler to send to the executor */ -int32_t qSubPlanToString(struct SSubplan *pPhyNode, char** str); +int32_t qSubPlanToString(const SSubplan* subplan, char** str); + +int32_t qStringToSubplan(const char* str, SSubplan** subplan); /** * Destroy the physical plan. diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 9f51969dc1..41d6e028cf 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -14,13 +14,13 @@ */ #if defined(INCLUDE_AS_ENUM) // enum define mode - #undef OP_ENUM_MACRO - #define OP_ENUM_MACRO(op) OP_##op, + #undef OP_ENUM_MACRO + #define OP_ENUM_MACRO(op) OP_##op, #elif defined(INCLUDE_AS_NAME) // comment define mode - #undef OP_ENUM_MACRO - #define OP_ENUM_MACRO(op) #op, + #undef OP_ENUM_MACRO + #define OP_ENUM_MACRO(op) #op, #else - #error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME + #error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME #endif OP_ENUM_MACRO(TableScan) diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index f63f13d8ac..877ecb2c85 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -77,6 +77,14 @@ typedef struct SInsertParseContext { SInsertStmtInfo* pOutput; } SInsertParseContext; +typedef int32_t (*FRowAppend)(const void *value, int32_t len, void *param); + +typedef struct SKvParam { + char buf[TSDB_MAX_TAGS_LEN]; + SKVRowBuilder* builder; + SSchema* schema; +} SKvParam; + static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; @@ -300,14 +308,6 @@ static int parseTime(SInsertParseContext* pCxt, SToken *pToken, int16_t timePrec return TSDB_CODE_SUCCESS; } -typedef int32_t (*FRowAppend)(const void *value, int32_t len, void *param); - -typedef struct SKvParam { - char buf[TSDB_MAX_TAGS_LEN]; - SKVRowBuilder* builder; - SSchema* schema; -} SKvParam; - static FORCE_INLINE int32_t KvRowAppend(const void *value, int32_t len, void *param) { SKvParam* pa = (SKvParam*)param; if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index 4e0d03d07a..8a309af526 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( planner - PRIVATE os util common catalog parser transport function query + PRIVATE os util common cjson catalog parser transport function query ) ADD_SUBDIRECTORY(test) diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 38b399fb0b..c5f948b722 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -101,13 +101,7 @@ int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag); -/** - * Convert to physical plan to string to enable to print it out in the shell. - * @param pPhyNode - * @param str - * @return - */ -int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str); +int32_t subPlanToString(const SSubplan *pPhyNode, char** str); /** * Destroy the query plan object. diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 277703f5c0..67e5770b75 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -215,7 +215,3 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD *pDag = context.pDag; return TSDB_CODE_SUCCESS; } - -int32_t subPlanToString(struct SSubplan *pPhyNode, char** str) { - return TSDB_CODE_SUCCESS; -} diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c new file mode 100644 index 0000000000..943c6b8dc0 --- /dev/null +++ b/source/libs/planner/src/physicalPlanJson.c @@ -0,0 +1,410 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "plannerInt.h" +#include "parser.h" +#include "cJSON.h" + +typedef cJSON* (*FToObj)(const void* obj); + +static bool addObject(cJSON* json, const char* name, FToObj func, const void* obj) { + if (NULL == obj) { + return true; + } + + cJSON* jObj = func(obj); + if (NULL == jObj) { + return false; + } + return cJSON_AddItemToObject(json, name, jObj); +} + +static bool addItem(cJSON* json, FToObj func, const void* item) { + cJSON* jItem = func(item); + if (NULL == jItem) { + return false; + } + return cJSON_AddItemToArray(json, jItem); +} + +static bool addArray(cJSON* json, const char* name, FToObj func, const SArray* array) { + size_t size = (NULL == array) ? 0 : taosArrayGetSize(array); + if (size > 0) { + cJSON* jArray = cJSON_AddArrayToObject(json, name); + if (NULL == jArray) { + return false; + } + for (size_t i = 0; i < size; ++i) { + if (!addItem(jArray, func, taosArrayGetP(array, i))) { + return false; + } + } + } + return true; +} + +static bool addRawArray(cJSON* json, const char* name, FToObj func, const void* array, int32_t itemSize, int32_t size) { + if (size > 0) { + cJSON* jArray = cJSON_AddArrayToObject(json, name); + if (NULL == jArray) { + return false; + } + for (size_t i = 0; i < size; ++i) { + if (!addItem(jArray, func, (const char*)array + itemSize * i)) { + return false; + } + } + } + return true; +} + +static cJSON* schemaToJson(const void* obj) { + const SSlotSchema* schema = (const SSlotSchema*)obj; + cJSON* jSchema = cJSON_CreateObject(); + if (NULL == jSchema) { + return NULL; + } + + // The 'name' field do not need to be serialized. + + bool res = cJSON_AddNumberToObject(jSchema, "Type", schema->type); + if (res) { + res = cJSON_AddNumberToObject(jSchema, "ColId", schema->colId); + } + if (res) { + res = cJSON_AddNumberToObject(jSchema, "Bytes", schema->bytes); + } + + if (!res) { + cJSON_Delete(jSchema); + return NULL; + } + return jSchema; +} + +static cJSON* columnFilterInfoToJson(const void* obj) { + const SColumnFilterInfo* filter = (const SColumnFilterInfo*)obj; + cJSON* jFilter = cJSON_CreateObject(); + if (NULL == jFilter) { + return NULL; + } + + bool res = cJSON_AddNumberToObject(jFilter, "LowerRelOptr", filter->lowerRelOptr); + if (res) { + res = cJSON_AddNumberToObject(jFilter, "UpperRelOptr", filter->upperRelOptr); + } + if (res) { + res = cJSON_AddNumberToObject(jFilter, "Filterstr", filter->filterstr); + } + if (res) { + res = cJSON_AddNumberToObject(jFilter, "LowerBnd", filter->lowerBndd); + } + if (res) { + res = cJSON_AddNumberToObject(jFilter, "UpperBnd", filter->upperBndd); + } + + if (!res) { + cJSON_Delete(jFilter); + return NULL; + } + return jFilter; +} + +static cJSON* columnInfoToJson(const void* obj) { + const SColumnInfo* col = (const SColumnInfo*)obj; + cJSON* jCol = cJSON_CreateObject(); + if (NULL == jCol) { + return NULL; + } + + bool res = cJSON_AddNumberToObject(jCol, "ColId", col->colId); + if (res) { + res = cJSON_AddNumberToObject(jCol, "Type", col->type); + } + if (res) { + res = cJSON_AddNumberToObject(jCol, "Bytes", col->bytes); + } + if (res) { + res = addRawArray(jCol, "FilterList", columnFilterInfoToJson, col->flist.filterInfo, sizeof(SColumnFilterInfo), col->flist.numOfFilters); + } + + if (!res) { + cJSON_Delete(jCol); + return NULL; + } + return jCol; +} + +static cJSON* columnToJson(const void* obj) { + const SColumn* col = (const SColumn*)obj; + cJSON* jCol = cJSON_CreateObject(); + if (NULL == jCol) { + return NULL; + } + + bool res = cJSON_AddNumberToObject(jCol, "TableId", col->uid); + if (res) { + res = cJSON_AddNumberToObject(jCol, "Flag", col->flag); + } + if (res) { + res = addObject(jCol, "Info", columnInfoToJson, &col->info); + } + + if (!res) { + cJSON_Delete(jCol); + return NULL; + } + return jCol; +} + +static cJSON* exprNodeToJson(const void* obj); + +static cJSON* operatorToJson(const void* obj) { + const tExprNode* exprInfo = (const tExprNode*)obj; + cJSON* jOper = cJSON_CreateObject(); + if (NULL == jOper) { + return NULL; + } + + bool res = cJSON_AddNumberToObject(jOper, "Oper", exprInfo->_node.optr); + if (res) { + res = addObject(jOper, "Left", exprNodeToJson, exprInfo->_node.pLeft); + } + if (res) { + res = addObject(jOper, "Right", exprNodeToJson, exprInfo->_node.pRight); + } + + if (!res) { + cJSON_Delete(jOper); + return NULL; + } + return jOper; +} + +static cJSON* functionToJson(const void* obj) { + const tExprNode* exprInfo = (const tExprNode*)obj; + cJSON* jFunc = cJSON_CreateObject(); + if (NULL == jFunc) { + return NULL; + } + + bool res = cJSON_AddStringToObject(jFunc, "Name", exprInfo->_function.functionName); + if (res) { + res = addRawArray(jFunc, "Child", exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), exprInfo->_function.num); + } + + if (!res) { + cJSON_Delete(jFunc); + return NULL; + } + return jFunc; +} + +static cJSON* variantToJson(const void* obj) { + const SVariant* var = (const SVariant*)obj; + cJSON* jVar = cJSON_CreateObject(); + if (NULL == jVar) { + return NULL; + } + + bool res = cJSON_AddNumberToObject(jVar, "Type", var->nType); + if (res) { + res = cJSON_AddNumberToObject(jVar, "Len", var->nLen); + } + if (res) { + if (0/* in */) { + res = addArray(jVar, "values", variantToJson, var->arr); + } else if (IS_NUMERIC_TYPE(var->nType)) { + res = cJSON_AddNumberToObject(jVar, "Value", var->d); + } else { + res = cJSON_AddStringToObject(jVar, "Value", var->pz); + } + } + + if (!res) { + cJSON_Delete(jVar); + return NULL; + } + return jVar; +} + +static cJSON* exprNodeToJson(const void* obj) { + const tExprNode* exprInfo = (const tExprNode*)obj; + cJSON* jExprInfo = cJSON_CreateObject(); + if (NULL == jExprInfo) { + return NULL; + } + + bool res = cJSON_AddNumberToObject(jExprInfo, "Type", exprInfo->nodeType); + if (res) { + switch (exprInfo->nodeType) { + case TEXPR_BINARYEXPR_NODE: + case TEXPR_UNARYEXPR_NODE: + res = addObject(jExprInfo, "Operator", operatorToJson, exprInfo); + break; + case TEXPR_FUNCTION_NODE: + res = addObject(jExprInfo, "Function", functionToJson, exprInfo); + break; + case TEXPR_COL_NODE: + res = addObject(jExprInfo, "Column", schemaToJson, exprInfo->pSchema); + break; + case TEXPR_VALUE_NODE: + res = addObject(jExprInfo, "Value", variantToJson, exprInfo->pVal); + break; + default: + res = false; + break; + } + } + + if (!res) { + cJSON_Delete(jExprInfo); + return NULL; + } + return jExprInfo; +} + +static cJSON* sqlExprToJson(const void* obj) { + const SSqlExpr* expr = (const SSqlExpr*)obj; + cJSON* jExpr = cJSON_CreateObject(); + if (NULL == jExpr) { + return NULL; + } + + // token does not need to be serialized. + + bool res = addObject(jExpr, "Schema", schemaToJson, &expr->resSchema); + if (res) { + res = addRawArray(jExpr, "Columns", columnToJson, expr->pColumns, sizeof(SColumn), expr->numOfCols); + } + if (res) { + res = cJSON_AddNumberToObject(jExpr, "InterBytes", expr->interBytes); + } + if (res) { + res = addRawArray(jExpr, "Params", variantToJson, expr->param, sizeof(SVariant), expr->numOfParams); + } + + if (!res) { + cJSON_Delete(jExpr); + return NULL; + } + return jExpr; +} + +static cJSON* exprInfoToJson(const void* obj) { + const SExprInfo* exprInfo = (const SExprInfo*)obj; + cJSON* jExprInfo = cJSON_CreateObject(); + if (NULL == jExprInfo) { + return NULL; + } + + bool res = addObject(jExprInfo, "Base", sqlExprToJson, &exprInfo->base); + if (res) { + res = addObject(jExprInfo, "Expr", exprNodeToJson, exprInfo->pExpr); + } + + if (!res) { + cJSON_Delete(jExprInfo); + return NULL; + } + return jExprInfo; +} + +static cJSON* phyNodeToJson(const void* obj) { + const SPhyNode* phyNode = (const SPhyNode*)obj; + cJSON* jNode = cJSON_CreateObject(); + if (NULL == jNode) { + return NULL; + } + + // The 'pParent' field do not need to be serialized. + + bool res = cJSON_AddStringToObject(jNode, "Name", phyNode->info.name); + if (res) { + res = addArray(jNode, "Targets", exprInfoToJson, phyNode->pTargets); + } + if (res) { + res = addArray(jNode, "Conditions", exprInfoToJson, phyNode->pConditions); + } + if (res) { + res = addRawArray(jNode, "Schema", schemaToJson, phyNode->targetSchema.pSchema, sizeof(SSlotSchema), phyNode->targetSchema.numOfCols); + } + if (res) { + res = addArray(jNode, "Children", phyNodeToJson, phyNode->pChildren); + } + + if (!res) { + cJSON_Delete(jNode); + return NULL; + } + return jNode; +} + +static cJSON* subplanIdToJson(const void* obj) { + const SSubplanId* id = (const SSubplanId*)obj; + cJSON* jId = cJSON_CreateObject(); + if (NULL == jId) { + return NULL; + } + + bool res = cJSON_AddNumberToObject(jId, "QueryId", id->queryId); + if (res) { + res = cJSON_AddNumberToObject(jId, "TemplateId", id->templateId); + } + if (res) { + res = cJSON_AddNumberToObject(jId, "SubplanId", id->subplanId); + } + + if (!res) { + cJSON_Delete(jId); + return NULL; + } + return jId; +} + +static cJSON* subplanToJson(const SSubplan* subplan) { + cJSON* jSubplan = cJSON_CreateObject(); + if (NULL == jSubplan) { + return NULL; + } + + // The 'type', 'level', 'execEpSet', 'pChildern' and 'pParents' fields do not need to be serialized. + + bool res = addObject(jSubplan, "Id", subplanIdToJson, &subplan->id); + if (res) { + res = addObject(jSubplan, "Node", phyNodeToJson, subplan->pNode); + } + + if (!res) { + cJSON_Delete(jSubplan); + return NULL; + } + return jSubplan; +} + +int32_t subPlanToString(const SSubplan* subplan, char** str) { + cJSON* json = subplanToJson(subplan); + if (NULL == json) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + *str = cJSON_Print(json); + return TSDB_CODE_SUCCESS; +} + +int32_t stringToSubplan(const char* str, SSubplan** subplan) { + // todo + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 744a849e2d..ee989234d5 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -41,3 +41,11 @@ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* destroyQueryPlan(logicPlan); return TSDB_CODE_SUCCESS; } + +int32_t qSubPlanToString(const SSubplan *subplan, char** str) { + return subPlanToString(subplan, str); +} + +int32_t qStringToSubplan(const char* str, SSubplan** subplan) { + return stringToSubplan(str, subplan); +} From f65a1339d65fb5733bb693e0ada66804afd65914 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 17 Dec 2021 14:12:01 +0800 Subject: [PATCH 25/50] merge tlist.h and tdlist.h --- include/util/tdlist.h | 169 ------------------------- include/util/tlist.h | 107 ++++++++++++++++ source/dnode/vnode/impl/inc/vnodeDef.h | 2 +- source/dnode/vnode/tsdb/inc/tsdbDef.h | 2 +- 4 files changed, 109 insertions(+), 171 deletions(-) delete mode 100644 include/util/tdlist.h diff --git a/include/util/tdlist.h b/include/util/tdlist.h deleted file mode 100644 index d047a57770..0000000000 --- a/include/util/tdlist.h +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_TDLIST_H_ -#define _TD_UTIL_TDLIST_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -// Single linked list -#define TD_SLIST_NODE(TYPE) \ - struct { \ - struct TYPE *sl_next_; \ - } - -#define TD_SLIST(TYPE) \ - struct { \ - struct TYPE *sl_head_; \ - int sl_neles_; \ - } - -#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) -#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) -#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) - -#define tSListInit(sl) \ - do { \ - (sl)->sl_head_ = NULL; \ - (sl)->sl_neles_ = 0; \ - } while (0) - -#define tSListPush(sl, sln) \ - do { \ - TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ - TD_SLIST_HEAD(sl) = (sln); \ - TD_SLIST_NELES(sl) += 1; \ - } while (0) - -#define tSListPop(sl) \ - do { \ - TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ - TD_SLIST_NELES(sl) -= 1; \ - } while (0) - -// Double linked list -#define TD_DLIST_NODE(TYPE) \ - struct { \ - struct TYPE *dl_prev_; \ - struct TYPE *dl_next_; \ - } - -#define TD_DLIST(TYPE) \ - struct { \ - struct TYPE *dl_head_; \ - struct TYPE *dl_tail_; \ - int dl_neles_; \ - } - -#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) -#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) -#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) -#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) -#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) - -#define tDListInit(dl) \ - do { \ - TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \ - TD_DLIST_NELES(dl) = 0; \ - } while (0) - -#define tDListAppend(dl, dln) \ - do { \ - if (TD_DLIST_HEAD(dl) == NULL) { \ - TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ - TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ - } else { \ - TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl); \ - TD_DLIST_NODE_NEXT(dln) = NULL; \ - TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln); \ - TD_DLIST_TAIL(dl) = (dln); \ - } \ - TD_DLIST_NELES(dl) += 1; \ - } while (0) - -#define tDListPrepend(dl, dln) \ - do { \ - if (TD_DLIST_HEAD(dl) == NULL) { \ - TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ - TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ - } else { \ - TD_DLIST_NODE_PREV(dln) = NULL; \ - TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl); \ - TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln); \ - TD_DLIST_HEAD(dl) = (dln); \ - } \ - TD_DLIST_NELES(dl) += 1; \ - } while (0) - -#define tDListPop(dl, dln) \ - do { \ - if (TD_DLIST_HEAD(dl) == (dln)) { \ - TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln); \ - } \ - if (TD_DLIST_TAIL(dl) == (dln)) { \ - TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln); \ - } \ - if (TD_DLIST_NODE_PREV(dln) != NULL) { \ - TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \ - } \ - if (TD_DLIST_NODE_NEXT(dln) != NULL) { \ - TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \ - } \ - TD_DLIST_NELES(dl) -= 1; \ - TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ - } while (0) - -#if 0 -// List iterator -#define TD_LIST_FITER 0 -#define TD_LIST_BITER 1 -#define TD_LIST_ITER(S) \ - struct { \ - int it_dir_; \ - S * it_next_; \ - S * it_ptr_; \ - TD_DLIST(S) * it_list_; \ - } - -#define tlistIterInit(it, l, dir) \ - (it)->it_dir_ = (dir); \ - (it)->it_list_ = l; \ - if ((dir) == TD_LIST_FITER) { \ - (it)->it_next_ = (l)->dl_head_; \ - } else { \ - (it)->it_next_ = (l)->dl_tail_; \ - } - -#define tlistIterNext(it) \ - ({ \ - (it)->it_ptr_ = (it)->it_next_; \ - if ((it)->it_next_ != NULL) { \ - if ((it)->it_dir_ == TD_LIST_FITER) { \ - (it)->it_next_ = (it)->it_next_->next_; \ - } else { \ - (it)->it_next_ = (it)->it_next_->prev_; \ - } \ - } \ - (it)->it_ptr_; \ - }) -#endif - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_TDLIST_H_*/ \ No newline at end of file diff --git a/include/util/tlist.h b/include/util/tlist.h index e803e96605..497f88b29e 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -19,6 +19,113 @@ extern "C" { #endif +// Single linked list +#define TD_SLIST_NODE(TYPE) \ + struct { \ + struct TYPE *sl_next_; \ + } + +#define TD_SLIST(TYPE) \ + struct { \ + struct TYPE *sl_head_; \ + int sl_neles_; \ + } + +#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) +#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) + +#define tSListInit(sl) \ + do { \ + (sl)->sl_head_ = NULL; \ + (sl)->sl_neles_ = 0; \ + } while (0) + +#define tSListPush(sl, sln) \ + do { \ + TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ + TD_SLIST_HEAD(sl) = (sln); \ + TD_SLIST_NELES(sl) += 1; \ + } while (0) + +#define tSListPop(sl) \ + do { \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ + TD_SLIST_NELES(sl) -= 1; \ + } while (0) + +// Double linked list +#define TD_DLIST_NODE(TYPE) \ + struct { \ + struct TYPE *dl_prev_; \ + struct TYPE *dl_next_; \ + } + +#define TD_DLIST(TYPE) \ + struct { \ + struct TYPE *dl_head_; \ + struct TYPE *dl_tail_; \ + int dl_neles_; \ + } + +#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) +#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) +#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) +#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) + +#define tDListInit(dl) \ + do { \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \ + TD_DLIST_NELES(dl) = 0; \ + } while (0) + +#define tDListAppend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_NODE_NEXT(TD_DLIST_TAIL(dl)) = (dln); \ + TD_DLIST_TAIL(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + +#define tDListPrepend(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV(dln) = NULL; \ + TD_DLIST_NODE_NEXT(dln) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV(TD_DLIST_HEAD(dl)) = (dln); \ + TD_DLIST_HEAD(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + +#define tDListPop(dl, dln) \ + do { \ + if (TD_DLIST_HEAD(dl) == (dln)) { \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_TAIL(dl) == (dln)) { \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV(dln); \ + } \ + if (TD_DLIST_NODE_PREV(dln) != NULL) { \ + TD_DLIST_NODE_NEXT(TD_DLIST_NODE_PREV(dln)) = TD_DLIST_NODE_NEXT(dln); \ + } \ + if (TD_DLIST_NODE_NEXT(dln) != NULL) { \ + TD_DLIST_NODE_PREV(TD_DLIST_NODE_NEXT(dln)) = TD_DLIST_NODE_PREV(dln); \ + } \ + TD_DLIST_NELES(dl) -= 1; \ + TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ + } while (0) + typedef enum { TD_LIST_FORWARD, TD_LIST_BACKWARD } TD_LIST_DIRECTION_T; typedef struct _list_node { diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index e6a88c7629..56e07aca10 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -19,7 +19,7 @@ #include "mallocator.h" #include "sync.h" #include "tcoding.h" -#include "tdlist.h" +#include "tlist.h" #include "tlockfree.h" #include "tmacro.h" #include "wal.h" diff --git a/source/dnode/vnode/tsdb/inc/tsdbDef.h b/source/dnode/vnode/tsdb/inc/tsdbDef.h index 7c593cb4c7..b1375c9477 100644 --- a/source/dnode/vnode/tsdb/inc/tsdbDef.h +++ b/source/dnode/vnode/tsdb/inc/tsdbDef.h @@ -18,7 +18,7 @@ #include "mallocator.h" #include "taosmsg.h" -#include "tdlist.h" +#include "tlist.h" #include "thash.h" #include "tskiplist.h" From 0d0f33ab5e4ee6eb85f4b951f25cda97a9993525 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Dec 2021 14:20:32 +0800 Subject: [PATCH 26/50] fix compile error and ci problems --- .../dnode/mgmt/impl/test/cluster/cluster.cpp | 7 ++-- source/dnode/mgmt/impl/test/db/db.cpp | 9 ++--- source/dnode/mgmt/impl/test/dnode/dnode.cpp | 13 +++---- .../dnode/mgmt/impl/test/profile/profile.cpp | 5 +-- source/dnode/mgmt/impl/test/show/show.cpp | 14 ++++---- source/dnode/mgmt/impl/test/stb/stb.cpp | 7 ++-- source/dnode/mgmt/impl/test/user/user.cpp | 7 ++-- source/dnode/mnode/impl/src/mndCluster.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 2 +- source/dnode/mnode/impl/src/mndSync.c | 36 +++++++++++++++++++ source/dnode/mnode/impl/src/mndUser.c | 2 +- source/dnode/vnode/impl/CMakeLists.txt | 2 +- tools/CMakeLists.txt | 2 +- 13 files changed, 63 insertions(+), 45 deletions(-) create mode 100644 source/dnode/mnode/impl/src/mndSync.c diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index 0822be67f7..0aca2fde4b 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -88,7 +88,7 @@ class DndTestCluster : public ::testing::Test { void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, type); EXPECT_EQ(pSchema->bytes, bytes); @@ -114,17 +114,14 @@ class DndTestCluster : public ::testing::Test { pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; ASSERT_NE(pRetrieveRsp, nullptr); pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->offset, 0); EXPECT_EQ(pRetrieveRsp->useconds, 0); // EXPECT_EQ(pRetrieveRsp->completed, completed); EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->reserved, 0); EXPECT_EQ(pRetrieveRsp->compLen, 0); pData = pRetrieveRsp->data; @@ -164,7 +161,7 @@ TEST_F(DndTestCluster, 01_ShowCluster) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_CLUSTER, "show cluster", 3, NULL); CheckSchema(0, TSDB_DATA_TYPE_INT, 4, "id"); CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, "name"); - CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); SendThenCheckShowRetrieveMsg(1); CheckInt32(); diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index c568b1152d..de1a606c86 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -88,7 +88,7 @@ class DndTestDb : public ::testing::Test { void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, type); EXPECT_EQ(pSchema->bytes, bytes); @@ -114,17 +114,14 @@ class DndTestDb : public ::testing::Test { pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; ASSERT_NE(pRetrieveRsp, nullptr); pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->offset, 0); EXPECT_EQ(pRetrieveRsp->useconds, 0); // EXPECT_EQ(pRetrieveRsp->completed, completed); EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->reserved, 0); EXPECT_EQ(pRetrieveRsp->compLen, 0); pData = pRetrieveRsp->data; @@ -182,13 +179,13 @@ int32_t DndTestDb::connId; TEST_F(DndTestDb, 01_ShowDb) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); - CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vgroups"); CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); CheckSchema(5, TSDB_DATA_TYPE_SMALLINT, 2, "days"); CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); - CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "cache(MB)"); + CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "cache"); CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "blocks"); CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "minrows"); CheckSchema(10, TSDB_DATA_TYPE_INT, 4, "maxrows"); diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 0cadb3a4c4..1fa91f26cb 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -102,7 +102,7 @@ class DndTestDnode : public ::testing::Test { void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, type); EXPECT_EQ(pSchema->bytes, bytes); @@ -128,17 +128,14 @@ class DndTestDnode : public ::testing::Test { pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; ASSERT_NE(pRetrieveRsp, nullptr); pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->offset, 0); EXPECT_EQ(pRetrieveRsp->useconds, 0); // EXPECT_EQ(pRetrieveRsp->completed, completed); EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->reserved, 0); EXPECT_EQ(pRetrieveRsp->compLen, 0); pData = pRetrieveRsp->data; @@ -187,12 +184,12 @@ SClient* DndTestDnode::pClient; TEST_F(DndTestDnode, 01_ShowDnode) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); CheckSchema(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); - CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "end point"); + CheckSchema(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vnodes"); - CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "max vnodes"); + CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "max_vnodes"); CheckSchema(4, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "status"); - CheckSchema(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); - CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline reason"); + CheckSchema(5, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "offline_reason"); SendThenCheckShowRetrieveMsg(1); CheckInt16(1); diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index f5970b61c5..f1e4704777 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -216,17 +216,14 @@ TEST_F(DndTestProfile, SConnectMsg_03) { SRetrieveTableRsp* pRsp = (SRetrieveTableRsp*)pMsg->pCont; ASSERT_NE(pRsp, nullptr); pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->offset = htobe64(pRsp->offset); pRsp->useconds = htobe64(pRsp->useconds); pRsp->compLen = htonl(pRsp->compLen); EXPECT_EQ(pRsp->numOfRows, 1); - EXPECT_EQ(pRsp->offset, 0); EXPECT_EQ(pRsp->useconds, 0); EXPECT_EQ(pRsp->completed, 1); EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRsp->compressed, 0); - EXPECT_EQ(pRsp->reserved, 0); EXPECT_EQ(pRsp->compLen, 0); } } @@ -497,7 +494,7 @@ TEST_F(DndTestProfile, SKillQueryMsg_03) { EXPECT_STREQ(pSchema->name, "queryId"); pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index eb925d407b..f0c20f71e5 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -151,49 +151,49 @@ TEST_F(DndTestShow, SShowMsg_04) { SSchema* pSchema = NULL; pSchema = &pMeta->pSchema[0]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "connId"); pSchema = &pMeta->pSchema[1]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "user"); pSchema = &pMeta->pSchema[2]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "program"); pSchema = &pMeta->pSchema[3]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "pid"); pSchema = &pMeta->pSchema[4]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "ip:port"); pSchema = &pMeta->pSchema[5]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); EXPECT_EQ(pSchema->bytes, 8); EXPECT_STREQ(pSchema->name, "login_time"); pSchema = &pMeta->pSchema[6]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); EXPECT_EQ(pSchema->bytes, 8); diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp index a6addbe5a9..ee8c86bcea 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -88,7 +88,7 @@ class DndTestStb : public ::testing::Test { void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, type); EXPECT_EQ(pSchema->bytes, bytes); @@ -114,17 +114,14 @@ class DndTestStb : public ::testing::Test { pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; ASSERT_NE(pRetrieveRsp, nullptr); pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->offset, 0); EXPECT_EQ(pRetrieveRsp->useconds, 0); // EXPECT_EQ(pRetrieveRsp->completed, completed); EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->reserved, 0); EXPECT_EQ(pRetrieveRsp->compLen, 0); pData = pRetrieveRsp->data; @@ -277,7 +274,7 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_STB, "show stables", 4, "1.d1"); CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); - CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); CheckSchema(2, TSDB_DATA_TYPE_INT, 4, "columns"); CheckSchema(3, TSDB_DATA_TYPE_INT, 4, "tags"); diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 2b7f70df71..32309b106e 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -87,7 +87,7 @@ class DndTestUser : public ::testing::Test { void CheckSchema(int32_t index, int8_t type, int32_t bytes, const char* name) { SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htons(pSchema->bytes); + pSchema->bytes = htonl(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, type); EXPECT_EQ(pSchema->bytes, bytes); @@ -113,17 +113,14 @@ class DndTestUser : public ::testing::Test { pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; ASSERT_NE(pRetrieveRsp, nullptr); pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); - pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); EXPECT_EQ(pRetrieveRsp->numOfRows, rows); - EXPECT_EQ(pRetrieveRsp->offset, 0); EXPECT_EQ(pRetrieveRsp->useconds, 0); // EXPECT_EQ(pRetrieveRsp->completed, completed); EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); EXPECT_EQ(pRetrieveRsp->compressed, 0); - EXPECT_EQ(pRetrieveRsp->reserved, 0); EXPECT_EQ(pRetrieveRsp->compLen, 0); pData = pRetrieveRsp->data; @@ -170,7 +167,7 @@ TEST_F(DndTestUser, 01_ShowUser) { SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "name"); CheckSchema(1, TSDB_DATA_TYPE_BINARY, 10 + VARSTR_HEADER_SIZE, "privilege"); - CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); + CheckSchema(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); CheckSchema(3, TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN + VARSTR_HEADER_SIZE, "account"); SendThenCheckShowRetrieveMsg(1); diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 4fa1dfecf5..7c53a4ebd0 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -154,7 +154,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); + strcpy(pSchema[cols].name, "create_time"); pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 258a96affe..188ca7963c 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -747,7 +747,7 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "vgroups"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c new file mode 100644 index 0000000000..59161b32f2 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "mndInt.h" +#include "mndTrans.h" + +int32_t mndInitSync(SMnode *pMnode) { return 0; } +void mndCleanupSync(SMnode *pMnode) {} + +int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) { + int32_t code = 0; + + // int32_t len = sdbGetRawTotalSize(pRaw); + // SSdbRaw *pReceived = calloc(1, len); + // memcpy(pReceived, pRaw, len); + // mDebug("trans:%d, data:%p recv from sync, code:0x%x pMsg:%p", pMsg->id, pReceived, code & 0xFFFF, pMsg); + + // mndTransApply(pMnode, pReceived, code); + return code; +} + +bool mndIsMaster(SMnode *pMnode) { return true; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index c97e1ff7d6..58d687a148 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -421,7 +421,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); + strcpy(pSchema[cols].name, "create_time"); pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 6972605afd..3623516624 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -19,5 +19,5 @@ target_link_libraries( # test if(${BUILD_TEST}) - add_subdirectory(test) + #add_subdirectory(test) endif(${BUILD_TEST}) \ No newline at end of file diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 629677c9a5..fdf6373810 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1 +1 @@ -add_subdirectory(shell) \ No newline at end of file +#add_subdirectory(shell) \ No newline at end of file From 10a7547b61f7c7d8b6aa41d6dae1b7667c6ac486 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Dec 2021 14:26:27 +0800 Subject: [PATCH 27/50] update index cache --- source/libs/index/inc/indexInt.h | 9 ++-- source/libs/index/inc/index_cache.h | 8 ++-- source/libs/index/src/index.c | 40 ++++++++++++++--- source/libs/index/src/index_cache.c | 70 ++++++++++++++++++----------- 4 files changed, 88 insertions(+), 39 deletions(-) diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index f6ff9bc139..cc740826e9 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -37,9 +37,11 @@ struct SIndex { #endif void *cache; void *tindex; - SHashObj *fieldObj; // - uint64_t suid; - int fieldId; + SHashObj *fieldObj;// < field name, field id> + + int64_t suid; // current super table id, -1 is normal table + int fieldId; // field id allocated to cache + int32_t cVersion; // current version allocated to cache pthread_mutex_t mtx; }; @@ -58,6 +60,7 @@ struct SIndexMultiTermQuery { // field and key; typedef struct SIndexTerm { + uint8_t type; // term data type, str/interger/json char *key; int32_t nKey; char *val; diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 27e095ff31..ff915a3930 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -20,8 +20,8 @@ // ----------------- row structure in skiplist --------------------- /* A data row, the format is like below: - * |<--totalLen-->|<-- fieldId-->|<-- value len--->|<-- value-->|<--version--->|<-- itermType -->| - * + * content: |<--totalLen-->|<-- fieldid-->|<--field type -->|<-- value len--->|<-- value -->|<-- uid -->|<--version--->|<-- itermType -->| + * len : |<--int32_t -->|<-- int16_t-->|<-- int16_t --->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<-- int8_t --->| */ #ifdef __cplusplus @@ -30,7 +30,6 @@ extern "C" { typedef struct IndexCache { T_REF_DECLARE() - int cVersion; // } IndexCache; @@ -39,7 +38,8 @@ IndexCache *indexCacheCreate(); void indexCacheDestroy(IndexCache *cache); -int indexCachePut(IndexCache *cache, int32_t fieldId, const char *fieldVale, int32_t fvlen, uint64_t uid, int8_t operaType); +int indexCachePut(IndexCache *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen, + uint32_t version, uint64_t uid, int8_t operType); int indexCacheGet(IndexCache *cache, uint64_t *rst); int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index c011411189..6a2697491d 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -23,7 +23,8 @@ typedef struct SIdxFieldInfo { - int id; // generated by index internal + int fieldId; // generated by index internal + int cVersion; int type; // field type } SIdxFieldInfo; @@ -39,7 +40,7 @@ static int indexMergeCacheIntoTindex(struct SIndex *sIdx) { } SIndex *indexOpen(SIndexOpts *opts, const char *path) { pthread_once(&isInit, indexInit); - SIndex *sIdx = malloc(sizeof(SIndex)); + SIndex *sIdx = calloc(1, sizeof(SIndex)); #ifdef USE_LUCENE index_t *index = index_open(path); @@ -49,6 +50,8 @@ SIndex *indexOpen(SIndexOpts *opts, const char *path) { sIdx->cache = (void*)indexCacheCreate(); sIdx->tindex = NULL; sIdx->fieldObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + sIdx->fieldId = 1; + sIdx->cVersion = 1; pthread_mutex_init(&sIdx->mtx, NULL); return sIdx; } @@ -65,7 +68,7 @@ void indexClose(SIndex *sIdx) { return; } -int indexPut(SIndex *index, SArray* field_vals, int uid) { +int indexPut(SIndex *index, SArray* fVals, int uid) { #ifdef USE_LUCENE index_document_t *doc = index_document_create(); @@ -73,8 +76,8 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) { char buf[16] = {0}; sprintf(buf, "%d", uid); - for (int i = 0; i < taosArrayGetSize(field_vals); i++) { - SIndexTerm *p = taosArrayGetP(field_vals, i); + for (int i = 0; i < taosArrayGetSize(fVals); i++) { + SIndexTerm *p = taosArrayGetP(fVals, i); index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1); } index_document_add(doc, NULL, 0, buf, strlen(buf), 0); @@ -82,10 +85,33 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) { index_put(index->index, doc); index_document_destroy(doc); #endif + + //TODO(yihao): reduce the lock range pthread_mutex_lock(&index->mtx); + for (int i = 0; i < taosArrayGetSize(fVals); i++) { + SIndexTerm *p = taosArrayGetP(fVals, i); + SIdxFieldInfo *fi = taosHashGet(index->fieldObj, p->key, p->nKey); + if (fi == NULL) { + SIdxFieldInfo tfi = {.fieldId = index->fieldId, .type = p->type}; + index->cVersion++; + index->fieldId++; + taosHashPut(index->fieldObj, p->key, p->nKey, &tfi, sizeof(tfi)); + } else { + //TODO, del + } + } + + for (int i = 0; i < taosArrayGetSize(fVals); i++) { + SIndexTerm *p = taosArrayGetP(fVals, i); + SIdxFieldInfo *fi = taosHashGet(index->fieldObj, p->key, p->nKey); + assert(fi != NULL); + int32_t fieldId = fi->fieldId; + int32_t colType = fi->type; + int32_t version = index->cVersion; + + } pthread_mutex_unlock(&index->mtx); return 1; - } int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result) { #ifdef USE_LUCENE @@ -152,7 +178,7 @@ SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery)); if (p == NULL) { return NULL; } p->opera = opera; - p->query = taosArrayInit(1, sizeof(SIndexTermQuery)); + p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; } void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) { diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 7c355b0f0a..2ecc823ef9 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -19,7 +19,7 @@ static int32_t compareKey(const void *l, const void *r) { char *lp = (char *)l; char *rp = (char *)r; - // skip total len + // skip total len, not compare int32_t ll, rl; // len memcpy(&ll, lp, sizeof(int32_t)); memcpy(&rl, rp, sizeof(int32_t)); @@ -27,7 +27,7 @@ static int32_t compareKey(const void *l, const void *r) { rp += sizeof(int32_t); // compare field id - int32_t lf, rf; // field id + int16_t lf, rf; // field id memcpy(&lf, lp, sizeof(lf)); memcpy(&rf, rp, sizeof(rf)); if (lf != rf) { @@ -36,14 +36,22 @@ static int32_t compareKey(const void *l, const void *r) { lp += sizeof(lf); rp += sizeof(rf); - // compare field value + // compare field type + int16_t lft, rft; + memcpy(&lft, lp, sizeof(lft)); + memcpy(&rft, rp, sizeof(rft)); + lp += sizeof(lft); + rp += sizeof(rft); + assert(rft == rft); + + // skip value len int32_t lfl, rfl; memcpy(&lfl, lp, sizeof(lfl)); memcpy(&rfl, rp, sizeof(rfl)); lp += sizeof(lfl); rp += sizeof(rfl); - //refator later + // compare value int32_t i, j; for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) { if (lp[i] == rp[j]) { continue; } @@ -54,17 +62,24 @@ static int32_t compareKey(const void *l, const void *r) { lp += lfl; rp += rfl; - // compare version + // skip uid + uint64_t lu, ru; + memcpy(&lu, lp, sizeof(lu)); + memcpy(&ru, rp, sizeof(ru)); + lp += sizeof(lu); + rp += sizeof(ru); + + // compare version, desc order int32_t lv, rv; memcpy(&lv, lp, sizeof(lv)); memcpy(&rv, rp, sizeof(rv)); if (lv != rv) { return lv > rv ? -1 : 1; - } + } lp += sizeof(lv); rp += sizeof(rv); + // not care item type - return 0; } @@ -77,35 +92,40 @@ void indexCacheDestroy(IndexCache *cache) { free(cache); } -int indexCachePut(IndexCache *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { +int indexCachePut(IndexCache *cache, int16_t fieldId, int16_t fieldType, const char *fieldValue, int32_t fvLen, + uint32_t version, uint64_t uid, int8_t operType) { if (cache == NULL) { return -1;} - int32_t version = T_REF_INC(cache); - int32_t total = sizeof(int32_t) + sizeof(fieldId) + 4 + fvlen + sizeof(version) + sizeof(uid) + sizeof(operType); + int32_t total = sizeof(int32_t) + sizeof(fieldId) + sizeof(fieldType) + sizeof(fvLen) + fvLen + sizeof(version) + sizeof(uid) + sizeof(operType); char *buf = calloc(1, total); char *p = buf; - memcpy(buf, &total, sizeof(total)); - total += total; + memcpy(p, &total, sizeof(total)); + p += sizeof(total); - memcpy(buf, &fieldId, sizeof(fieldId)); - buf += sizeof(fieldId); + memcpy(p, &fieldId, sizeof(fieldId)); + p += sizeof(fieldId); - memcpy(buf, &fvlen, sizeof(fvlen)); - buf += sizeof(fvlen); - memcpy(buf, fieldValue, fvlen); - buf += fvlen; + memcpy(p, &fieldType, sizeof(fieldType)); + p += sizeof(fieldType); + + memcpy(p, &fvLen, sizeof(fvLen)); + p += sizeof(fvLen); + memcpy(p, fieldValue, fvLen); + p += fvLen; - memcpy(buf, &version, sizeof(version)); - buf += sizeof(version); + memcpy(p, &version, sizeof(version)); + p += sizeof(version); - memcpy(buf, &uid, sizeof(uid)); - buf += sizeof(uid); + memcpy(p, &uid, sizeof(uid)); + p += sizeof(uid); - memcpy(buf, &operType, sizeof(operType)); - buf += sizeof(operType); - + memcpy(p, &operType, sizeof(operType)); + p += sizeof(operType); + +} +int indexCacheDel(IndexCache *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { } int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result) { From 522d25c6273b44ca339e970a866dd9fab575a46c Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 17 Dec 2021 14:35:19 +0800 Subject: [PATCH 28/50] build TDengine 3.0 --- Jenkinsfile | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c19a720d49..a237a0f76f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -76,36 +76,7 @@ def pre_test(){ git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD git clean -dfx - ''' - script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - cd ${WK} - git checkout master - ''' - } - else if(env.CHANGE_TARGET == '2.0'){ - sh ''' - cd ${WK} - git checkout 2.0 - ''' - } - else if(env.CHANGE_TARGET == '3.0'){ - sh ''' - cd ${WK} - git checkout 3.0 - ''' - } - else{ - sh ''' - cd ${WK} - git checkout develop - ''' - } - } - sh ''' - cd ${WK} - git pull >/dev/null + git clean -dfx export TZ=Asia/Harbin date @@ -124,7 +95,7 @@ pipeline { options { skipDefaultCheckout() } environment{ WK = '/var/lib/jenkins/workspace/TDinternal' - WKC= '/var/lib/jenkins/workspace/TDinternal/community' + WKC= '/var/lib/jenkins/workspace/TDengine' } stages { stage('pre_build'){ From 9d5e0d53fbcdbf6d145bcf450871592e23f6b8c6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 17 Dec 2021 14:49:34 +0800 Subject: [PATCH 29/50] refactor list --- include/util/tlist.h | 46 +++---- .../dnode/vnode/impl/src/vnodeArenaMAImpl.c | 10 +- source/dnode/vnode/impl/src/vnodeBufferPool.c | 22 +-- source/dnode/vnode/impl/src/vnodeMgr.c | 6 +- source/dnode/vnode/tsdb/src/tsdbMemTable.c | 4 +- source/util/src/tlist.c | 125 +++++------------- 6 files changed, 75 insertions(+), 138 deletions(-) diff --git a/include/util/tlist.h b/include/util/tlist.h index 497f88b29e..06d0abc797 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -19,7 +19,7 @@ extern "C" { #endif -// Single linked list +// Single linked list ================ #define TD_SLIST_NODE(TYPE) \ struct { \ struct TYPE *sl_next_; \ @@ -35,30 +35,30 @@ extern "C" { #define TD_SLIST_NELES(sl) ((sl)->sl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) -#define tSListInit(sl) \ +#define TD_SLIST_INIT(sl) \ do { \ (sl)->sl_head_ = NULL; \ (sl)->sl_neles_ = 0; \ } while (0) -#define tSListPush(sl, sln) \ +#define TD_SLIST_PUSH(sl, sln) \ do { \ TD_SLIST_NODE_NEXT(sln) = TD_SLIST_HEAD(sl); \ TD_SLIST_HEAD(sl) = (sln); \ TD_SLIST_NELES(sl) += 1; \ } while (0) -#define tSListPop(sl) \ +#define TD_SLIST_POP(sl) \ do { \ TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ TD_SLIST_NELES(sl) -= 1; \ } while (0) -// Double linked list +// Double linked list ================ #define TD_DLIST_NODE(TYPE) \ struct { \ - struct TYPE *dl_prev_; \ - struct TYPE *dl_next_; \ + struct TYPE *dl_prev_; \ + struct TYPE *dl_next_; \ } #define TD_DLIST(TYPE) \ @@ -74,13 +74,13 @@ extern "C" { #define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) #define TD_DLIST_NELES(dl) ((dl)->dl_neles_) -#define tDListInit(dl) \ +#define TD_DLIST_INIT(dl) \ do { \ TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = NULL; \ TD_DLIST_NELES(dl) = 0; \ } while (0) -#define tDListAppend(dl, dln) \ +#define TD_DLIST_APPEND(dl, dln) \ do { \ if (TD_DLIST_HEAD(dl) == NULL) { \ TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ @@ -94,7 +94,7 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) -#define tDListPrepend(dl, dln) \ +#define TD_DLIST_PREPEND(dl, dln) \ do { \ if (TD_DLIST_HEAD(dl) == NULL) { \ TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ @@ -108,7 +108,7 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) -#define tDListPop(dl, dln) \ +#define TD_DLIST_POP(dl, dln) \ do { \ if (TD_DLIST_HEAD(dl) == (dln)) { \ TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT(dln); \ @@ -126,19 +126,17 @@ extern "C" { TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ } while (0) +// General double linked list typedef enum { TD_LIST_FORWARD, TD_LIST_BACKWARD } TD_LIST_DIRECTION_T; -typedef struct _list_node { - struct _list_node *next; - struct _list_node *prev; - char data[]; +typedef struct SListNode { + TD_DLIST_NODE(SListNode); + char data[]; } SListNode; typedef struct { - struct _list_node *head; - struct _list_node *tail; - int numOfEles; - int eleSize; + TD_DLIST(SListNode); + int eleSize; } SList; typedef struct { @@ -146,11 +144,11 @@ typedef struct { TD_LIST_DIRECTION_T direction; } SListIter; -#define listHead(l) (l)->head -#define listTail(l) (l)->tail -#define listNEles(l) (l)->numOfEles -#define listEleSize(l) (l)->eleSize -#define isListEmpty(l) ((l)->numOfEles == 0) +#define listHead(l) TD_DLIST_HEAD(l) +#define listTail(l) TD_DLIST_TAIL(l) +#define listNEles(l) TD_DLIST_NELES(l) +#define listEleSize(l) ((l)->eleSize) +#define isListEmpty(l) (TD_DLIST_NELES(l) == 0) #define listNodeFree(n) free(n) void tdListInit(SList *list, int eleSize); diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c index 99d4781df9..5999b08a7d 100644 --- a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c @@ -27,7 +27,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { pVMA->capacity = capacity; pVMA->ssize = ssize; pVMA->lsize = lsize; - tSListInit(&(pVMA->nlist)); + TD_SLIST_INIT(&(pVMA->nlist)); pVMA->pNode = vArenaNodeNew(capacity); if (pVMA->pNode == NULL) { @@ -35,7 +35,7 @@ SVMemAllocator *vmaCreate(uint64_t capacity, uint64_t ssize, uint64_t lsize) { return NULL; } - tSListPush(&(pVMA->nlist), pVMA->pNode); + TD_SLIST_PUSH(&(pVMA->nlist), pVMA->pNode); return pVMA; } @@ -44,7 +44,7 @@ void vmaDestroy(SVMemAllocator *pVMA) { if (pVMA) { while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); - tSListPop(&(pVMA->nlist)); + TD_SLIST_POP(&(pVMA->nlist)); vArenaNodeFree(pNode); } @@ -55,7 +55,7 @@ void vmaDestroy(SVMemAllocator *pVMA) { void vmaReset(SVMemAllocator *pVMA) { while (TD_SLIST_NELES(&(pVMA->nlist)) > 1) { SVArenaNode *pNode = TD_SLIST_HEAD(&(pVMA->nlist)); - tSListPop(&(pVMA->nlist)); + TD_SLIST_POP(&(pVMA->nlist)); vArenaNodeFree(pNode); } @@ -75,7 +75,7 @@ void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) { return NULL; } - tSListPush(&(pVMA->nlist), pNode); + TD_SLIST_PUSH(&(pVMA->nlist), pNode); } ptr = pNode->ptr; diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 6c1ededfc9..763cf3e3b3 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -39,8 +39,8 @@ int vnodeOpenBufPool(SVnode *pVnode) { return -1; } - tDListInit(&(pVnode->pBufPool->free)); - tDListInit(&(pVnode->pBufPool->incycle)); + TD_DLIST_INIT(&(pVnode->pBufPool->free)); + TD_DLIST_INIT(&(pVnode->pBufPool->incycle)); pVnode->pBufPool->inuse = NULL; @@ -54,7 +54,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { return -1; } - tDListAppend(&(pVnode->pBufPool->free), pVMA); + TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA); } pVnode->pBufPool->pMAF = (SMemAllocatorFactory *)malloc(sizeof(SMemAllocatorFactory)); @@ -76,14 +76,14 @@ void vnodeCloseBufPool(SVnode *pVnode) { while (true) { SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->incycle)); if (pVMA == NULL) break; - tDListPop(&(pVnode->pBufPool->incycle), pVMA); + TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA); vmaDestroy(pVMA); } while (true) { SVMemAllocator *pVMA = TD_DLIST_HEAD(&(pVnode->pBufPool->free)); if (pVMA == NULL) break; - tDListPop(&(pVnode->pBufPool->free), pVMA); + TD_DLIST_POP(&(pVnode->pBufPool->free), pVMA); vmaDestroy(pVMA); } @@ -97,7 +97,7 @@ int vnodeBufPoolSwitch(SVnode *pVnode) { pVnode->pBufPool->inuse = NULL; - tDListAppend(&(pVnode->pBufPool->incycle), pvma); + TD_DLIST_APPEND(&(pVnode->pBufPool->incycle), pvma); return 0; } @@ -106,9 +106,9 @@ int vnodeBufPoolRecycle(SVnode *pVnode) { SVMemAllocator *pvma = TD_DLIST_HEAD(&(pBufPool->incycle)); ASSERT(pvma != NULL); - tDListPop(&(pBufPool->incycle), pvma); + TD_DLIST_POP(&(pBufPool->incycle), pvma); vmaReset(pvma); - tDListAppend(&(pBufPool->free), pvma); + TD_DLIST_APPEND(&(pBufPool->free), pvma); return 0; } @@ -121,7 +121,7 @@ void *vnodeMalloc(SVnode *pVnode, uint64_t size) { // TODO: add sem_wait and sem_post pBufPool->inuse = TD_DLIST_HEAD(&(pBufPool->free)); if (pBufPool->inuse) { - tDListPop(&(pBufPool->free), pBufPool->inuse); + TD_DLIST_POP(&(pBufPool->free), pBufPool->inuse); break; } else { // tsem_wait(&(pBufPool->hasFree)); @@ -184,7 +184,7 @@ static void vBufPoolDestroyMA(SMemAllocatorFactory *pMAF, SMemAllocator *pMA) { free(pMA); if (--pVMA->_ref.val == 0) { - tDListPop(&(pVnode->pBufPool->incycle), pVMA); - tDListAppend(&(pVnode->pBufPool->free), pVMA); + TD_DLIST_POP(&(pVnode->pBufPool->incycle), pVMA); + TD_DLIST_APPEND(&(pVnode->pBufPool->free), pVMA); } } \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c index 964cbe77da..fae96ae22c 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -34,7 +34,7 @@ int vnodeInit(uint16_t nthreads) { pthread_mutex_init(&(vnodeMgr.mutex), NULL); pthread_cond_init(&(vnodeMgr.hasTask), NULL); - tDListInit(&(vnodeMgr.queue)); + TD_DLIST_INIT(&(vnodeMgr.queue)); for (uint16_t i = 0; i < nthreads; i++) { pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); @@ -77,7 +77,7 @@ void vnodeClear() { int vnodeScheduleTask(SVnodeTask* pTask) { pthread_mutex_lock(&(vnodeMgr.mutex)); - tDListAppend(&(vnodeMgr.queue), pTask); + TD_DLIST_APPEND(&(vnodeMgr.queue), pTask); pthread_cond_signal(&(vnodeMgr.hasTask)); @@ -101,7 +101,7 @@ static void* loop(void* arg) { pthread_cond_wait(&(vnodeMgr.hasTask), &(vnodeMgr.mutex)); } } else { - tDListPop(&(vnodeMgr.queue), pTask); + TD_DLIST_POP(&(vnodeMgr.queue), pTask); break; } } diff --git a/source/dnode/vnode/tsdb/src/tsdbMemTable.c b/source/dnode/vnode/tsdb/src/tsdbMemTable.c index e3d1f8673e..8a320129b0 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMemTable.c +++ b/source/dnode/vnode/tsdb/src/tsdbMemTable.c @@ -62,7 +62,7 @@ STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) { pMemTable->keyMax = TSKEY_MIN; pMemTable->nRow = 0; pMemTable->pMA = pMA; - tSListInit(&(pMemTable->list)); + TD_SLIST_INIT(&(pMemTable->list)); // TODO return pMemTable; @@ -86,7 +86,7 @@ int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) { // TODO } - tSListPush(&(pMemTable->list), pTbData); + TD_SLIST_PUSH(&(pMemTable->list), pTbData); return 0; } diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index 6756af226f..f79bca1e4b 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -17,9 +17,8 @@ #include "os.h" void tdListInit(SList *list, int eleSize) { - list->eleSize = eleSize; - list->numOfEles = 0; - list->head = list->tail = NULL; + TD_DLIST_INIT(list); + listEleSize(list) = eleSize; } SList *tdListNew(int eleSize) { @@ -31,14 +30,11 @@ SList *tdListNew(int eleSize) { } void tdListEmpty(SList *list) { - SListNode *node = list->head; - while (node) { - list->head = node->next; + SListNode *node; + while ((node = TD_DLIST_HEAD(list)) != NULL) { + TD_DLIST_POP(list, node); free(node); - node = list->head; } - list->head = list->tail = 0; - list->numOfEles = 0; } void *tdListFree(SList *list) { @@ -50,40 +46,16 @@ void *tdListFree(SList *list) { return NULL; } -void tdListPrependNode(SList *list, SListNode *node) { - if (list->head == NULL) { - list->head = node; - list->tail = node; - } else { - node->next = list->head; - node->prev = NULL; - list->head->prev = node; - list->head = node; - } - list->numOfEles++; -} +void tdListPrependNode(SList *list, SListNode *node) { TD_DLIST_PREPEND(list, node); } -void tdListAppendNode(SList *list, SListNode *node) { - if (list->head == NULL) { - list->head = node; - list->tail = node; - } else { - node->prev = list->tail; - node->next = NULL; - list->tail->next = node; - list->tail = node; - } - - list->numOfEles++; -} +void tdListAppendNode(SList *list, SListNode *node) { TD_DLIST_APPEND(list, node); } int tdListPrepend(SList *list, void *data) { SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; - node->next = node->prev = NULL; memcpy((void *)(node->data), data, list->eleSize); - tdListPrependNode(list, node); + TD_DLIST_PREPEND(list, node); return 0; } @@ -93,73 +65,40 @@ int tdListAppend(SList *list, void *data) { if (node == NULL) return -1; memcpy((void *)(node->data), data, list->eleSize); - tdListAppendNode(list, node); + TD_DLIST_APPEND(list, node); return 0; } SListNode *tdListPopHead(SList *list) { - if (list->head == NULL) return NULL; - SListNode *node = list->head; - if (node->next == NULL) { - list->head = NULL; - list->tail = NULL; - } else { - list->head = node->next; + SListNode *node; + + node = TD_DLIST_HEAD(list); + + if (node) { + TD_DLIST_POP(list, node); } - list->numOfEles--; - node->next = NULL; - node->prev = NULL; + return node; } SListNode *tdListPopTail(SList *list) { - if (list->tail == NULL) return NULL; - SListNode *node = list->tail; - if (node->prev == NULL) { - list->head = NULL; - list->tail = NULL; - } else { - list->tail = node->prev; + SListNode *node; + + node = TD_DLIST_TAIL(list); + if (node) { + TD_DLIST_POP(list, node); } - list->numOfEles--; - node->next = node->prev = NULL; + return node; } -SListNode *tdListGetHead(SList *list) { - if (list == NULL || list->numOfEles == 0) { - return NULL; - } +SListNode *tdListGetHead(SList *list) { return TD_DLIST_HEAD(list); } - return list->head; -} - -SListNode *tsListGetTail(SList *list) { - if (list == NULL || list->numOfEles == 0) { - return NULL; - } - - return list->tail; -} +SListNode *tsListGetTail(SList *list) { return TD_DLIST_TAIL(list); } SListNode *tdListPopNode(SList *list, SListNode *node) { - if (list->head == node) { - list->head = node->next; - } - if (list->tail == node) { - list->tail = node->prev; - } - - if (node->prev != NULL) { - node->prev->next = node->next; - } - if (node->next != NULL) { - node->next->prev = node->prev; - } - list->numOfEles--; - node->next = node->prev = NULL; - + TD_DLIST_POP(list, node); return node; } @@ -174,19 +113,19 @@ void tdListMove(SList *src, SList *dst) { void tdListDiscard(SList *list) { if (list) { - list->head = list->tail = NULL; - list->numOfEles = 0; + listHead(list) = listTail(list) = NULL; + listNEles(list) = 0; } } -void tdListNodeGetData(SList *list, SListNode *node, void *target) { memcpy(target, node->data, list->eleSize); } +void tdListNodeGetData(SList *list, SListNode *node, void *target) { memcpy(target, node->data, listEleSize(list)); } void tdListInitIter(SList *list, SListIter *pIter, TD_LIST_DIRECTION_T direction) { pIter->direction = direction; if (direction == TD_LIST_FORWARD) { - pIter->next = list->head; + pIter->next = listHead(list); } else { - pIter->next = list->tail; + pIter->next = listTail(list); } } @@ -194,9 +133,9 @@ SListNode *tdListNext(SListIter *pIter) { SListNode *node = pIter->next; if (node == NULL) return NULL; if (pIter->direction == TD_LIST_FORWARD) { - pIter->next = node->next; + pIter->next = TD_DLIST_NODE_NEXT(node); } else { - pIter->next = node->prev; + pIter->next = TD_DLIST_NODE_PREV(node); } return node; From 53cbb261ae00f5a2549eddc6173d03b5db788578 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 17 Dec 2021 15:05:19 +0800 Subject: [PATCH 30/50] configure build --- source/util/CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 7c42afcc51..6fa6e5f82f 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -1,3 +1,4 @@ +configure_file(src/version.c.in src/version.c) aux_source_directory(src UTIL_SRC) add_library(util STATIC ${UTIL_SRC}) target_include_directories( @@ -11,6 +12,4 @@ target_link_libraries( PUBLIC zlib PUBLIC lz4_static PUBLIC api -) - -CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/src/version.c.in" "${CMAKE_CURRENT_SOURCE_DIR}/src/version.c") +) \ No newline at end of file From 2e4f2f266eae2f8af1c1230c9fa4313a68ef622c Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 17 Dec 2021 15:16:25 +0800 Subject: [PATCH 31/50] add a new server --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a237a0f76f..398ce0f869 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -99,7 +99,7 @@ pipeline { } stages { stage('pre_build'){ - agent{label 'master'} + agent{label 'slave3_0'} options { skipDefaultCheckout() } when { changeRequest() From 7f7de09d4e7a08219be2005bcac5577fdfe786c8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 17 Dec 2021 15:30:11 +0800 Subject: [PATCH 32/50] make it compile --- source/util/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 6fa6e5f82f..d343945a80 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -1,4 +1,4 @@ -configure_file(src/version.c.in src/version.c) +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/src/version.c.in" "${CMAKE_CURRENT_SOURCE_DIR}/src/version.c") aux_source_directory(src UTIL_SRC) add_library(util STATIC ${UTIL_SRC}) target_include_directories( From 91a36a51cf20672a28d236984a20a150f23dfc12 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 17 Dec 2021 15:49:15 +0800 Subject: [PATCH 33/50] make compile faster --- Jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 398ce0f869..2558df777b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -83,8 +83,7 @@ def pre_test(){ mkdir debug cd debug cmake .. > /dev/null - make > /dev/null - make install > /dev/null + make -j4> /dev/null ''' return 1 From 00833f73907dd42f923f6b06709ff9ad5c629219 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Dec 2021 15:49:27 +0800 Subject: [PATCH 34/50] TD-10431 shell module --- source/client/src/clientImpl.c | 8 +- tools/CMakeLists.txt | 2 +- tools/shell/CMakeLists.txt | 9 +- tools/shell/inc/shell.h | 10 +- tools/shell/src/{ => backup}/shellCheck.c | 0 tools/shell/src/{ => backup}/shellDarwin.c | 0 tools/shell/src/{ => backup}/shellImport.c | 0 tools/shell/src/{ => backup}/shellWindows.c | 0 tools/shell/src/{ => backup}/tnettest.c | 0 tools/shell/{inc => src/backup}/tnettest.h | 0 tools/shell/src/shellCommand.c | 2 + tools/shell/src/shellEngine.c | 183 +++++++++----------- tools/shell/src/shellLinux.c | 7 +- tools/shell/src/shellMain.c | 50 +++--- 14 files changed, 140 insertions(+), 131 deletions(-) rename tools/shell/src/{ => backup}/shellCheck.c (100%) rename tools/shell/src/{ => backup}/shellDarwin.c (100%) rename tools/shell/src/{ => backup}/shellImport.c (100%) rename tools/shell/src/{ => backup}/shellWindows.c (100%) rename tools/shell/src/{ => backup}/tnettest.c (100%) rename tools/shell/{inc => src/backup}/tnettest.h (100%) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 4815e862e8..c038799bb4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -434,4 +434,10 @@ void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 pResultInfo->pCol[i] = pResultInfo->row[i]; offset += pResultInfo->fields[i].bytes; } -} \ No newline at end of file +} + +const char *taos_get_client_info() { return version; } + +int taos_affected_rows(TAOS_RES *res) { return 1; } + +int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index fdf6373810..629677c9a5 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1 +1 @@ -#add_subdirectory(shell) \ No newline at end of file +add_subdirectory(shell) \ No newline at end of file diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 9216bfcb6c..def421c212 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -1,13 +1,18 @@ aux_source_directory(src SHELL_SRC) -list(REMOVE_ITEM SHELL_SRC ./src/shellWindows.c) -list(REMOVE_ITEM SHELL_SRC ./src/shellDarwin.c) +list(REMOVE_ITEM SHELL_SRC src/shellWindows.c) +list(REMOVE_ITEM SHELL_SRC src/shellDarwin.c) add_executable(shell ${SHELL_SRC}) target_link_libraries( shell PUBLIC taos PUBLIC util + PUBLIC common PUBLIC os ) +target_include_directories( + shell + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) diff --git a/tools/shell/inc/shell.h b/tools/shell/inc/shell.h index 10221a409a..cf31e9d9d9 100644 --- a/tools/shell/inc/shell.h +++ b/tools/shell/inc/shell.h @@ -13,13 +13,13 @@ * along with this program. If not, see . */ -#ifndef __SHELL__ -#define __SHELL__ +#ifndef _TD_SHELL_H_ +#define _TD_SHELL_H_ -#include "../../../../include/client/taos.h" -#include "stdbool.h" +#include "os.h" + +#include "taos.h" #include "taosdef.h" -#include "tsclient.h" #define MAX_USERNAME_SIZE 64 #define MAX_DBNAME_SIZE 64 diff --git a/tools/shell/src/shellCheck.c b/tools/shell/src/backup/shellCheck.c similarity index 100% rename from tools/shell/src/shellCheck.c rename to tools/shell/src/backup/shellCheck.c diff --git a/tools/shell/src/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c similarity index 100% rename from tools/shell/src/shellDarwin.c rename to tools/shell/src/backup/shellDarwin.c diff --git a/tools/shell/src/shellImport.c b/tools/shell/src/backup/shellImport.c similarity index 100% rename from tools/shell/src/shellImport.c rename to tools/shell/src/backup/shellImport.c diff --git a/tools/shell/src/shellWindows.c b/tools/shell/src/backup/shellWindows.c similarity index 100% rename from tools/shell/src/shellWindows.c rename to tools/shell/src/backup/shellWindows.c diff --git a/tools/shell/src/tnettest.c b/tools/shell/src/backup/tnettest.c similarity index 100% rename from tools/shell/src/tnettest.c rename to tools/shell/src/backup/tnettest.c diff --git a/tools/shell/inc/tnettest.h b/tools/shell/src/backup/tnettest.h similarity index 100% rename from tools/shell/inc/tnettest.h rename to tools/shell/src/backup/tnettest.h diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 67e0c94989..cf0ceded38 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -19,6 +19,8 @@ #include "shell.h" #include "shellCommand.h" +#include + extern int wcwidth(wchar_t c); extern int wcswidth(const wchar_t *s, size_t n); typedef struct { diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index efc37403b4..3a6cbd259e 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -21,13 +21,14 @@ #include "os.h" #include "shell.h" #include "shellCommand.h" -#include "tutil.h" #include "taosdef.h" #include "taoserror.h" #include "tglobal.h" -#include "tsclient.h" +#include "ttypes.h" +#include "tutil.h" #include +#include /**************** Global variables ****************/ #ifdef _TD_POWER_ @@ -58,7 +59,7 @@ SShellHistory history; #define DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30 extern int32_t tsMaxBinaryDisplayWidth; -extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); +extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); /* * FUNCTION: Initialize the shell. @@ -89,12 +90,6 @@ TAOS *shellInit(SShellArguments *_args) { _args->user = TSDB_DEFAULT_USER; } - if (taos_init()) { - printf("failed to init taos\n"); - fflush(stdout); - return NULL; - } - // Connect to the database. TAOS *con = NULL; if (_args->auth == NULL) { @@ -127,6 +122,7 @@ TAOS *shellInit(SShellArguments *_args) { exit(EXIT_SUCCESS); } +#if 0 #ifndef WINDOWS if (_args->dir[0] != 0) { source_dir(con, _args); @@ -139,12 +135,13 @@ TAOS *shellInit(SShellArguments *_args) { taos_close(con); exit(EXIT_SUCCESS); } +#endif #endif return con; } -static bool isEmptyCommand(const char* cmd) { +static bool isEmptyCommand(const char *cmd) { for (char c = *cmd++; c != 0; c = *cmd++) { if (c != ' ' && c != '\t' && c != ';') { return false; @@ -153,7 +150,6 @@ static bool isEmptyCommand(const char* cmd) { return true; } - static int32_t shellRunSingleCommand(TAOS *con, char *command) { /* If command is empty just return */ if (isEmptyCommand(command)) { @@ -176,10 +172,11 @@ static int32_t shellRunSingleCommand(TAOS *con, char *command) { return 0; } - if (regex_match(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$", REG_EXTENDED | REG_ICASE)) { + if (regex_match(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$", + REG_EXTENDED | REG_ICASE)) { strtok(command, " \t"); strtok(NULL, " \t"); - char* p = strtok(NULL, " \t"); + char *p = strtok(NULL, " \t"); if (strcasecmp(p, "default") == 0) { tsMaxBinaryDisplayWidth = DEFAULT_MAX_BINARY_DISPLAY_WIDTH; } else { @@ -202,8 +199,7 @@ static int32_t shellRunSingleCommand(TAOS *con, char *command) { return 0; } - -int32_t shellRunCommand(TAOS* con, char* command) { +int32_t shellRunCommand(TAOS *con, char *command) { /* If command is empty just return */ if (isEmptyCommand(command)) { return 0; @@ -255,7 +251,7 @@ int32_t shellRunCommand(TAOS* con, char* command) { if (c == '\\') { if (quote != 0 && (*command == '_' || *command == '\\')) { - //DO nothing + // DO nothing } else { esc = true; continue; @@ -284,21 +280,22 @@ int32_t shellRunCommand(TAOS* con, char* command) { return shellRunSingleCommand(con, cmd); } - void freeResultWithRid(int64_t rid) { +#if 0 SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); if(pSql){ taos_free_result(pSql); taosReleaseRef(tscObjRef, rid); } +#endif } void shellRunCommandOnServer(TAOS *con, char command[]) { int64_t st, et; wordexp_t full_path; - char * sptr = NULL; - char * cptr = NULL; - char * fname = NULL; + char *sptr = NULL; + char *cptr = NULL; + char *fname = NULL; bool printMode = false; if ((sptr = strstr(command, ">>")) != NULL) { @@ -327,7 +324,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { st = taosGetTimestampUs(); - TAOS_RES* pSql = taos_query_h(con, command, &result); + TAOS_RES *pSql = taos_query(con, command); if (taos_errno(pSql)) { taos_error(pSql, st); return; @@ -344,7 +341,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { return; } - if (!tscIsUpdateQuery(pSql)) { // select and show kinds of commands + if (true /*!tscIsUpdateQuery(pSql)*/) { // select and show kinds of commands int error_no = 0; int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); @@ -405,14 +402,13 @@ int regex_match(const char *s, const char *reg, int cflags) { return 0; } - -static char* formatTimestamp(char* buf, int64_t val, int precision) { +static char *formatTimestamp(char *buf, int64_t val, int precision) { if (args.is_raw_time) { sprintf(buf, "%" PRId64, val); return buf; } - time_t tt; + time_t tt; int32_t ms = 0; if (precision == TSDB_TIME_PRECISION_NANO) { tt = (time_t)(val / 1000000000); @@ -425,13 +421,13 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { ms = val % 1000; } -/* comment out as it make testcases like select_with_tags.sim fail. - but in windows, this may cause the call to localtime crash if tt < 0, - need to find a better solution. - if (tt < 0) { - tt = 0; - } - */ + /* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ #ifdef WINDOWS if (tt < 0) tt = 0; @@ -447,8 +443,8 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } } - struct tm* ptm = localtime(&tt); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + struct tm *ptm = localtime(&tt); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); @@ -461,8 +457,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } - -static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_t length, int precision) { +static void dumpFieldToFile(FILE *fp, const char *val, TAOS_FIELD *field, int32_t length, int precision) { if (val == NULL) { fprintf(fp, "%s", TSDB_DATA_NULL_STR); return; @@ -498,7 +493,7 @@ static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_ fprintf(fp, "\'%s\'", buf); break; case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(buf, *(int64_t*)val, precision); + formatTimestamp(buf, *(int64_t *)val, precision); fprintf(fp, "'%s'", buf); break; default: @@ -506,7 +501,7 @@ static void dumpFieldToFile(FILE* fp, const char* val, TAOS_FIELD* field, int32_ } } -static int dumpResultToFile(const char* fname, TAOS_RES* tres) { +static int dumpResultToFile(const char *fname, TAOS_RES *tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; @@ -519,7 +514,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { return -1; } - FILE* fp = fopen(full_path.we_wordv[0], "w"); + FILE *fp = fopen(full_path.we_wordv[0], "w"); if (fp == NULL) { fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]); wordfree(&full_path); @@ -528,9 +523,9 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { wordfree(&full_path); - int num_fields = taos_num_fields(tres); + int num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int precision = taos_result_precision(tres); for (int col = 0; col < num_fields; col++) { if (col > 0) { @@ -542,18 +537,18 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { int numOfRows = 0; do { - int32_t* length = taos_fetch_lengths(tres); + int32_t *length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { if (i > 0) { fputc(',', fp); } - dumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision); + dumpFieldToFile(fp, (const char *)row[i], fields + i, length[i], precision); } fputc('\n', fp); numOfRows++; row = taos_fetch_row(tres); - } while( row != NULL); + } while (row != NULL); result = 0; fclose(fp); @@ -561,14 +556,13 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { return numOfRows; } - static void shellPrintNChar(const char *str, int length, int width) { wchar_t tail[3]; - int pos = 0, cols = 0, totalCols = 0, tailLen = 0; + int pos = 0, cols = 0, totalCols = 0, tailLen = 0; while (pos < length) { wchar_t wc; - int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX); + int bytes = mbtowc(&wc, str + pos, MB_CUR_MAX); if (bytes == 0) { break; } @@ -625,8 +619,7 @@ static void shellPrintNChar(const char *str, int length, int width) { } } - -static void printField(const char* val, TAOS_FIELD* field, int width, int32_t length, int precision) { +static void printField(const char *val, TAOS_FIELD *field, int width, int32_t length, int precision) { if (val == NULL) { int w = width; if (field->type < TSDB_DATA_TYPE_TINYINT || field->type > TSDB_DATA_TYPE_DOUBLE) { @@ -679,7 +672,7 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le shellPrintNChar(val, length, width); break; case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(buf, *(int64_t*)val, precision); + formatTimestamp(buf, *(int64_t *)val, precision); printf("%s", buf); break; default: @@ -687,27 +680,26 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le } } - -bool isSelectQuery(TAOS_RES* tres) { +bool isSelectQuery(TAOS_RES *tres) { +#if 0 char *sql = tscGetSqlStr(tres); if (regex_match(sql, "^[\t ]*select[ \t]*", REG_EXTENDED | REG_ICASE)) { return true; } - +#endif return false; } - -static int verticalPrintResult(TAOS_RES* tres) { +static int verticalPrintResult(TAOS_RES *tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - int num_fields = taos_num_fields(tres); + int num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int precision = taos_result_precision(tres); int maxColNameLen = 0; for (int col = 0; col < num_fields; col++) { @@ -719,7 +711,7 @@ static int verticalPrintResult(TAOS_RES* tres) { uint64_t resShowMaxNum = UINT64_MAX; - if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) && !tscIsQueryWithLimit(tres)) { + if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) /*&& !tscIsQueryWithLimit(tres)*/) { resShowMaxNum = DEFAULT_RES_SHOW_NUM; } @@ -729,52 +721,52 @@ static int verticalPrintResult(TAOS_RES* tres) { if (numOfRows < resShowMaxNum) { printf("*************************** %d.row ***************************\n", numOfRows + 1); - int32_t* length = taos_fetch_lengths(tres); + int32_t *length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { - TAOS_FIELD* field = fields + i; + TAOS_FIELD *field = fields + i; int padding = (int)(maxColNameLen - strlen(field->name)); printf("%*.s%s: ", padding, " ", field->name); - printField((const char*)row[i], field, 0, length[i], precision); + printField((const char *)row[i], field, 0, length[i], precision); putchar('\n'); } } else if (showMore) { - printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); - printf("[You can add limit statement to get more or redirect results to specific file to get all.]\n"); - showMore = 0; + printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); + printf("[You can add limit statement to get more or redirect results to specific file to get all.]\n"); + showMore = 0; } numOfRows++; row = taos_fetch_row(tres); - } while(row != NULL); + } while (row != NULL); return numOfRows; } -static int calcColWidth(TAOS_FIELD* field, int precision) { +static int calcColWidth(TAOS_FIELD *field, int precision) { int width = (int)strlen(field->name); switch (field->type) { case TSDB_DATA_TYPE_BOOL: - return MAX(5, width); // 'false' + return MAX(5, width); // 'false' case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: - return MAX(4, width); // '-127' + return MAX(4, width); // '-127' case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: - return MAX(6, width); // '-32767' + return MAX(6, width); // '-32767' case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: - return MAX(11, width); // '-2147483648' + return MAX(11, width); // '-2147483648' case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: - return MAX(21, width); // '-9223372036854775807' + return MAX(21, width); // '-9223372036854775807' case TSDB_DATA_TYPE_FLOAT: return MAX(20, width); @@ -801,12 +793,13 @@ static int calcColWidth(TAOS_FIELD* field, int precision) { case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { return MAX(14, width); - } if (precision == TSDB_TIME_PRECISION_NANO) { + } + if (precision == TSDB_TIME_PRECISION_NANO) { return MAX(29, width); } else if (precision == TSDB_TIME_PRECISION_MICRO) { - return MAX(26, width); // '2020-01-01 00:00:00.000000' + return MAX(26, width); // '2020-01-01 00:00:00.000000' } else { - return MAX(23, width); // '2020-01-01 00:00:00.000' + return MAX(23, width); // '2020-01-01 00:00:00.000' } default: @@ -816,13 +809,12 @@ static int calcColWidth(TAOS_FIELD* field, int precision) { return 0; } - -static void printHeader(TAOS_FIELD* fields, int* width, int num_fields) { +static void printHeader(TAOS_FIELD *fields, int *width, int num_fields) { int rowWidth = 0; for (int col = 0; col < num_fields; col++) { - TAOS_FIELD* field = fields + col; - int padding = (int)(width[col] - strlen(field->name)); - int left = padding / 2; + TAOS_FIELD *field = fields + col; + int padding = (int)(width[col] - strlen(field->name)); + int left = padding / 2; printf(" %*.s%s%*.s |", left, " ", field->name, padding - left, " "); rowWidth += width[col] + 3; } @@ -834,16 +826,15 @@ static void printHeader(TAOS_FIELD* fields, int* width, int num_fields) { putchar('\n'); } - -static int horizontalPrintResult(TAOS_RES* tres) { +static int horizontalPrintResult(TAOS_RES *tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - int num_fields = taos_num_fields(tres); + int num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int precision = taos_result_precision(tres); int width[TSDB_MAX_COLUMNS]; for (int col = 0; col < num_fields; col++) { @@ -854,7 +845,7 @@ static int horizontalPrintResult(TAOS_RES* tres) { uint64_t resShowMaxNum = UINT64_MAX; - if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) && !tscIsQueryWithLimit(tres)) { + if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) /* && !tscIsQueryWithLimit(tres)*/) { resShowMaxNum = DEFAULT_RES_SHOW_NUM; } @@ -862,34 +853,33 @@ static int horizontalPrintResult(TAOS_RES* tres) { int showMore = 1; do { - int32_t* length = taos_fetch_lengths(tres); + int32_t *length = taos_fetch_lengths(tres); if (numOfRows < resShowMaxNum) { for (int i = 0; i < num_fields; i++) { putchar(' '); - printField((const char*)row[i], fields + i, width[i], length[i], precision); + printField((const char *)row[i], fields + i, width[i], length[i], precision); putchar(' '); putchar('|'); } putchar('\n'); } else if (showMore) { - printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); - printf("[You can add limit statement to show more or redirect results to specific file to get all.]\n"); - showMore = 0; + printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); + printf("[You can add limit statement to show more or redirect results to specific file to get all.]\n"); + showMore = 0; } numOfRows++; row = taos_fetch_row(tres); - } while(row != NULL); + } while (row != NULL); return numOfRows; } - int shellDumpResult(TAOS_RES *tres, char *fname, int *error_no, bool vertical) { int numOfRows = 0; if (fname != NULL) { numOfRows = dumpResultToFile(fname, tres); - } else if(vertical) { + } else if (vertical) { numOfRows = verticalPrintResult(tres); } else { numOfRows = horizontalPrintResult(tres); @@ -899,13 +889,12 @@ int shellDumpResult(TAOS_RES *tres, char *fname, int *error_no, bool vertical) { return numOfRows; } - void read_history() { // Initialize history memset(history.hist, 0, sizeof(char *) * MAX_HISTORY_SIZE); history.hstart = 0; history.hend = 0; - char * line = NULL; + char *line = NULL; size_t line_size = 0; int read_size = 0; @@ -975,9 +964,9 @@ int isCommentLine(char *line) { void source_file(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char * cmd = calloc(1, tsMaxSQLStringLen+1); + char *cmd = calloc(1, tsMaxSQLStringLen + 1); size_t cmd_len = 0; - char * line = NULL; + char *line = NULL; size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { @@ -1087,5 +1076,5 @@ void shellGetGrantInfo(void *con) { } fprintf(stdout, "\n"); - #endif +#endif } diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index 93783b2055..766a57c968 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -19,7 +19,11 @@ #include "shell.h" #include "shellCommand.h" #include "tkey.h" -#include "tulog.h" +#include "ulog.h" + +#include +#include +#include #define OPT_ABORT 1 /* �Cabort */ @@ -68,7 +72,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { break; case 'P': if (arg) { - tsDnodeShellPort = atoi(arg); arguments->port = atoi(arg); } else { fprintf(stderr, "Invalid port\n"); diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 29713686ea..607dc79257 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -16,7 +16,7 @@ #include "os.h" #include "shell.h" #include "tconfig.h" -#include "tnettest.h" +#include "tglobal.h" pthread_t pid; static tsem_t cancelSem; @@ -28,23 +28,27 @@ void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { void *cancelHandler(void *arg) { setThreadName("cancelHandler"); - while(1) { + while (1) { if (tsem_wait(&cancelSem) != 0) { taosMsleep(10); continue; } #ifdef LINUX +#if 0 int64_t rid = atomic_val_compare_exchange_64(&result, result, 0); SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); taos_stop_query(pSql); taosReleaseRef(tscObjRef, rid); +#endif #else printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); #endif + printf("\nReceive ctrl+c or other signal, quit shell.\n"); + exit(0); } - + return NULL; } @@ -69,31 +73,29 @@ int checkVersion() { } // Global configurations -SShellArguments args = { - .host = NULL, +SShellArguments args = {.host = NULL, #ifndef TD_WINDOWS - .password = NULL, + .password = NULL, #endif - .user = NULL, - .database = NULL, - .timezone = NULL, - .is_raw_time = false, - .is_use_passwd = false, - .dump_config = false, - .file = "\0", - .dir = "\0", - .threadNum = 5, - .commands = NULL, - .pktLen = 1000, - .pktNum = 100, - .pktType = "TCP", - .netTestRole = NULL -}; + .user = NULL, + .database = NULL, + .timezone = NULL, + .is_raw_time = false, + .is_use_passwd = false, + .dump_config = false, + .file = "\0", + .dir = "\0", + .threadNum = 5, + .commands = NULL, + .pktLen = 1000, + .pktNum = 100, + .pktType = "TCP", + .netTestRole = NULL}; /* * Main function. */ -int main(int argc, char* argv[]) { +int main(int argc, char *argv[]) { /*setlocale(LC_ALL, "en_US.UTF-8"); */ if (!checkVersion()) { @@ -102,6 +104,7 @@ int main(int argc, char* argv[]) { shellParseArgument(argc, argv, &args); +#if 0 if (args.dump_config) { taosInitGlobalCfg(); taosReadGlobalLogCfg(); @@ -123,9 +126,10 @@ int main(int argc, char* argv[]) { taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType); exit(0); } +#endif /* Initialize the shell */ - TAOS* con = shellInit(&args); + TAOS *con = shellInit(&args); if (con == NULL) { exit(EXIT_FAILURE); } From a3360ea0942d9f9a894867818d0ba22052107c37 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 17 Dec 2021 16:12:49 +0800 Subject: [PATCH 35/50] refact --- include/util/mallocator.h | 33 +++++++++++-------- source/dnode/vnode/impl/src/vnodeBufferPool.c | 6 +--- source/dnode/vnode/tsdb/src/tsdbMemTable.c | 6 ++-- source/util/src/mallocator.c | 4 ++- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/include/util/mallocator.h b/include/util/mallocator.h index ffe242017e..49a9327353 100644 --- a/include/util/mallocator.h +++ b/include/util/mallocator.h @@ -22,23 +22,28 @@ extern "C" { #endif -typedef struct SMemAllocator SMemAllocator; -typedef struct SMemAllocatorFactory SMemAllocatorFactory; +// Memory allocator +#define TD_MEM_ALCT(TYPE) \ + struct { \ + void *(*malloc_)(struct TYPE *, uint64_t size); \ + void (*free_)(struct TYPE *, void *ptr); \ + } +#define TD_MA_MALLOC_FUNC(TMA) (TMA)->malloc_ +#define TD_MA_FREE_FUNC(TMA) (TMA)->free_ -struct SMemAllocator { - void *impl; - void *(*malloc)(SMemAllocator *, uint64_t size); - void *(*calloc)(SMemAllocator *, uint64_t nmemb, uint64_t size); - void *(*realloc)(SMemAllocator *, void *ptr, uint64_t size); - void (*free)(SMemAllocator *, void *ptr); - uint64_t (*usage)(SMemAllocator *); -}; +#define TD_MA_MALLOC(TMA, SIZE) (*((TMA)->malloc_))(TMA, (SIZE)) +#define TD_MA_FREE(TMA, PTR) (*((TMA)->free_))(TMA, (PTR)) -struct SMemAllocatorFactory { +typedef struct SMemAllocator { void *impl; - SMemAllocator *(*create)(SMemAllocatorFactory *); - void (*destroy)(SMemAllocatorFactory *, SMemAllocator *); -}; + TD_MEM_ALCT(SMemAllocator); +} SMemAllocator; + +typedef struct SMemAllocatorFactory { + void *impl; + SMemAllocator *(*create)(struct SMemAllocatorFactory *); + void (*destroy)(struct SMemAllocatorFactory *, SMemAllocator *); +} SMemAllocatorFactory; #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/impl/src/vnodeBufferPool.c index 763cf3e3b3..228df6c0a4 100644 --- a/source/dnode/vnode/impl/src/vnodeBufferPool.c +++ b/source/dnode/vnode/impl/src/vnodeBufferPool.c @@ -168,11 +168,7 @@ static SMemAllocator *vBufPoolCreateMA(SMemAllocatorFactory *pMAF) { pWrapper->pVMA = pVnode->pBufPool->inuse; pMA->impl = pWrapper; - pMA->malloc = vmaMaloocCb; - pMA->calloc = NULL; - pMA->realloc = NULL; - pMA->free = NULL; - pMA->usage = NULL; + TD_MA_MALLOC_FUNC(pMA) = vmaMaloocCb; return pMA; } diff --git a/source/dnode/vnode/tsdb/src/tsdbMemTable.c b/source/dnode/vnode/tsdb/src/tsdbMemTable.c index 8a320129b0..7b0df18f5a 100644 --- a/source/dnode/vnode/tsdb/src/tsdbMemTable.c +++ b/source/dnode/vnode/tsdb/src/tsdbMemTable.c @@ -50,7 +50,7 @@ STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) { pMA = (*pMAF->create)(pMAF); ASSERT(pMA != NULL); - pMemTable = (STsdbMemTable *)((*pMA->malloc)(pMA, sizeof(*pMemTable))); + pMemTable = (STsdbMemTable *)TD_MA_MALLOC(pMA, sizeof(*pMemTable)); if (pMemTable == NULL) { (*pMAF->destroy)(pMAF, pMA); return NULL; @@ -71,7 +71,7 @@ STsdbMemTable *tsdbNewMemTable(SMemAllocatorFactory *pMAF) { void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable) { SMemAllocator *pMA = pMemTable->pMA; - if (pMA->free) { + if (TD_MA_FREE_FUNC(pMA) != NULL) { // TODO ASSERT(0); } @@ -81,7 +81,7 @@ void tsdbFreeMemTable(SMemAllocatorFactory *pMAF, STsdbMemTable *pMemTable) { int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) { SMemAllocator *pMA = pMemTable->pMA; - STbData * pTbData = (STbData *)((*pMA->malloc)(pMA, sizeof(*pTbData))); + STbData * pTbData = (STbData *)TD_MA_MALLOC(pMA, sizeof(*pTbData)); if (pTbData == NULL) { // TODO } diff --git a/source/util/src/mallocator.c b/source/util/src/mallocator.c index 1819396ccd..a56fbfa597 100644 --- a/source/util/src/mallocator.c +++ b/source/util/src/mallocator.c @@ -16,6 +16,7 @@ #include "mallocator.h" /* ------------------------ HEAP ALLOCATOR ------------------------ */ +#if 0 typedef struct { size_t tusage; } SHeapAllocator; @@ -104,4 +105,5 @@ static size_t haUsage(SMemAllocator *pma) { return ((SHeapAllocator *)(pma->impl /* ------------------------ ARENA ALLOCATOR ------------------------ */ typedef struct { size_t usage; -} SArenaAllocator; \ No newline at end of file +} SArenaAllocator; +#endif \ No newline at end of file From 4192f1add984337ed01c43192527a3b3a71838a6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Dec 2021 16:42:26 +0800 Subject: [PATCH 36/50] [td-11818] enable create database. --- include/common/tname.h | 16 +- include/libs/{query => qcom}/query.h | 2 +- source/client/CMakeLists.txt | 7 +- source/client/inc/clientInt.h | 3 +- source/client/src/clientImpl.c | 2 +- source/client/src/clientMsgHandler.c | 50 ++++-- source/client/test/clientTests.cpp | 50 ++++-- source/common/src/tname.c | 128 +++------------- source/libs/CMakeLists.txt | 2 +- source/libs/catalog/CMakeLists.txt | 2 +- source/libs/parser/CMakeLists.txt | 2 +- source/libs/parser/inc/astGenerator.h | 4 +- source/libs/parser/inc/astToMsg.h | 2 + source/libs/parser/inc/parserUtil.h | 1 + source/libs/parser/src/astGenerator.c | 19 +-- source/libs/parser/src/astToMsg.c | 146 +++++++++++++++++- source/libs/parser/src/astValidate.c | 169 +++++++++++++-------- source/libs/parser/src/parser.c | 2 +- source/libs/parser/src/parserUtil.c | 19 +++ source/libs/parser/src/sql.c | 8 +- source/libs/planner/CMakeLists.txt | 2 +- source/libs/qcom/CMakeLists.txt | 12 ++ source/libs/{query => qcom}/inc/queryInt.h | 0 source/libs/{query => qcom}/src/querymsg.c | 0 source/libs/query/CMakeLists.txt | 12 -- source/libs/scheduler/CMakeLists.txt | 2 +- 26 files changed, 410 insertions(+), 252 deletions(-) rename include/libs/{query => qcom}/query.h (99%) create mode 100644 source/libs/qcom/CMakeLists.txt rename source/libs/{query => qcom}/inc/queryInt.h (100%) rename source/libs/{query => qcom}/src/querymsg.c (100%) delete mode 100644 source/libs/query/CMakeLists.txt diff --git a/include/common/tname.h b/include/common/tname.h index 7578a7804c..e31bfd38a6 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -16,7 +16,7 @@ #ifndef TDENGINE_TNAME_H #define TDENGINE_TNAME_H -#include "taosmsg.h" +//#include "taosmsg.h" #define TSDB_DB_NAME_T 1 #define TSDB_TABLE_NAME_T 2 @@ -27,7 +27,7 @@ typedef struct SName { uint8_t type; //db_name_t, table_name_t - char acctId[TSDB_ACCT_ID_LEN]; + int32_t acctId; char dbname[TSDB_DB_NAME_LEN]; char tname[TSDB_TABLE_NAME_LEN]; } SName; @@ -38,7 +38,7 @@ int32_t tNameLen(const SName* name); SName* tNameDup(const SName* name); -bool tIsValidName(const SName* name); +bool tNameIsValid(const SName* name); const char* tNameGetTableName(const SName* name); @@ -50,14 +50,10 @@ bool tNameIsEmpty(const SName* name); void tNameAssign(SName* dst, const SName* src); +int32_t tNameSetDbName(SName* dst, int32_t acctId, const char* dbName, size_t nameLen); + int32_t tNameFromString(SName* dst, const char* str, uint32_t type); -int32_t tNameSetAcctId(SName* dst, const char* acct); - -SSchema* tGetTbnameColumnSchema(); - -#if 0 -int32_t tNameSetDbName(SName* dst, const char* acct, SToken* dbToken); -#endif +int32_t tNameSetAcctId(SName* dst, int32_t acctId); #endif // TDENGINE_TNAME_H diff --git a/include/libs/query/query.h b/include/libs/qcom/query.h similarity index 99% rename from include/libs/query/query.h rename to include/libs/qcom/query.h index f8c7d787ba..fbfa19e5a5 100644 --- a/include/libs/query/query.h +++ b/include/libs/qcom/query.h @@ -84,9 +84,9 @@ typedef struct STableMetaOutput { extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); +SSchema* tGetTbnameColumnSchema(); extern void msgInit(); - extern int32_t qDebugFlag; #define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 61b60e5ba6..c78bf02cbd 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -2,14 +2,13 @@ aux_source_directory(src CLIENT_SRC) add_library(taos ${CLIENT_SRC}) target_include_directories( taos - PUBLIC "${CMAKE_SOURCE_DIR}/include/client" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_SOURCE_DIR}/include/client" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( taos - PRIVATE common INTERFACE api - PRIVATE os util common transport parser catalog function query + PRIVATE os util common transport parser catalog function qcom ) ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 9ef1d67e74..4e7fff06b6 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -76,11 +76,10 @@ typedef struct SAppInfo { typedef struct STscObj { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; - char acctId[TSDB_ACCT_ID_LEN]; char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; + int32_t acctId; uint32_t connId; uint64_t id; // ref ID returned by taosAddRef -// struct SSqlObj *sqlList; void *pTransporter; pthread_mutex_t mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj from this tscObj diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c038799bb4..d0c6d99c27 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -153,7 +153,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { void* output = NULL; int32_t outputLen = 0; code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); - if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW) { + if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_CREATE_DB) { pRequest->type = type; pRequest->body.param = output; pRequest->body.paramLen = outputLen; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 646964e319..36fa013f27 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -14,6 +14,7 @@ */ #include +#include #include "clientInt.h" #include "clientLog.h" #include "os.h" @@ -2885,7 +2886,7 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg } int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool autocreate, bool onlyLocal) { - assert(tIsValidName(&pTableMetaInfo->name)); + assert(tNameIsValid(&pTableMetaInfo->name)); char name[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&pTableMetaInfo->name, name); @@ -3138,7 +3139,7 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { // TODO refactor pthread_mutex_lock(&pTscObj->mutex); char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; - int32_t len = sprintf(temp, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db); + int32_t len = sprintf(temp, "%d%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pTscObj->db); assert(len <= sizeof(pTscObj->db)); tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db)); @@ -3153,6 +3154,7 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { } pTscObj->connId = pConnect->connId; + pTscObj->acctId = pConnect->acctId; // update the appInstInfo pTscObj->pAppInfo->clusterId = pConnect->clusterId; @@ -3165,19 +3167,33 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { return 0; } -int32_t buildCreateUserMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { - pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER; - pMsgBody->msgLen = sizeof(SCreateUserMsg); +int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { pMsgBody->requestObjRefId = pRequest->self; - pMsgBody->pData = pRequest->body.param; - return 0; -} - -int32_t buildShowMsg(SRequestObj* pRequest, SRequestMsgBody* pMsgBody) { - pMsgBody->msgType = TSDB_MSG_TYPE_SHOW; pMsgBody->msgLen = pRequest->body.paramLen; - pMsgBody->requestObjRefId = pRequest->self; pMsgBody->pData = pRequest->body.param; + + switch(pRequest->type) { + case TSDB_SQL_CREATE_USER: + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER; + break; + case TSDB_SQL_CREATE_DB: { + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_DB; + + SCreateDbMsg* pCreateMsg = pRequest->body.param; + SName name = {0}; + int32_t ret = tNameSetDbName(&name, pRequest->pTscObj->acctId, pCreateMsg->db, strnlen(pCreateMsg->db, tListLen(pCreateMsg->db))); + if (ret != TSDB_CODE_SUCCESS) { + return -1; + } + + tNameGetFullDbName(&name, pCreateMsg->db); + + break; + } + case TSDB_SQL_SHOW: + pMsgBody->msgType = TSDB_MSG_TYPE_SHOW; + break; + } } STableMeta* createTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { @@ -3283,6 +3299,9 @@ int32_t processRetrieveMnodeRsp(SRequestObj *pRequest, const char* pMsg, int32_t return 0; } +int32_t processCreateDbRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + // todo rsp with the vnode id list +} void initMsgHandleFp() { #if 0 @@ -3363,11 +3382,14 @@ void initMsgHandleFp() { buildRequestMsgFp[TSDB_SQL_CONNECT] = buildConnectMsg; handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; - buildRequestMsgFp[TSDB_SQL_CREATE_USER] = buildCreateUserMsg; + buildRequestMsgFp[TSDB_SQL_CREATE_USER] = doBuildMsgSupp; - buildRequestMsgFp[TSDB_SQL_SHOW] = buildShowMsg; + buildRequestMsgFp[TSDB_SQL_SHOW] = doBuildMsgSupp; handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp; buildRequestMsgFp[TSDB_SQL_RETRIEVE_MNODE] = buildRetrieveMnodeMsg; handleRequestRspFp[TSDB_SQL_RETRIEVE_MNODE]= processRetrieveMnodeRsp; + + buildRequestMsgFp[TSDB_SQL_CREATE_DB] = doBuildMsgSupp; + handleRequestRspFp[TSDB_SQL_CREATE_DB] = processCreateDbRsp; } \ No newline at end of file diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 46fd76234e..944247e88e 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -57,25 +57,38 @@ TEST(testCase, create_user_Test) { taos_close(pConn); } -//TEST(testCase, show_user_Test) { +//TEST(testCase, drop_user_Test) { // TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); // assert(pConn != NULL); // -// TAOS_RES* pRes = taos_query(pConn, "show users"); -// TAOS_ROW pRow = NULL; -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); +// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); // } // +// taos_free_result(pRes); // taos_close(pConn); //} +TEST(testCase, show_user_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show users"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_close(pConn); +} + TEST(testCase, show_db_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -94,3 +107,18 @@ TEST(testCase, show_db_Test) { taos_close(pConn); } + +TEST(testCase, create_db_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database abc"); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_close(pConn); +} diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 28f920a6a9..fa303fe4e9 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -4,9 +4,6 @@ #include "tname.h" #include "taosmsg.h" -#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) -#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) - #define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) char* extractDBName(const char* tableId, char* name) { @@ -120,84 +117,11 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in #endif -static struct SSchema _s = { - .colId = TSDB_TBNAME_COLUMN_INDEX, - .type = TSDB_DATA_TYPE_BINARY, - .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, - .name = "tbname", -}; - -SSchema* tGetTbnameColumnSchema() { - return &_s; -} - -static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) { - int32_t rowLen = 0; - - for (int32_t i = 0; i < numOfCols; ++i) { - // 1. valid types - if (!isValidDataType(pSchema[i].type)) { - return false; - } - - // 2. valid length for each type - if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) { - if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) { - return false; - } - } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) { - return false; - } - } else { - if (pSchema[i].bytes != tDataTypes[pSchema[i].type].bytes) { - return false; - } - } - - // 3. valid column names - for (int32_t j = i + 1; j < numOfCols; ++j) { - if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) { - return false; - } - } - - rowLen += pSchema[i].bytes; - } - - return rowLen <= maxLen; -} - -bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) { - if (!VALIDNUMOFCOLS(numOfCols)) { - return false; - } - - if (!VALIDNUMOFTAGS(numOfTags)) { - return false; - } - - /* first column must be the timestamp, which is a primary key */ - if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) { - return false; - } - - if (!doValidateSchema(pSchema, numOfCols, TSDB_MAX_BYTES_PER_ROW)) { - return false; - } - - if (!doValidateSchema(&pSchema[numOfCols], numOfTags, TSDB_MAX_TAGS_LEN)) { - return false; - } - - return true; -} - int32_t tNameExtractFullName(const SName* name, char* dst) { assert(name != NULL && dst != NULL); // invalid full name format, abort - if (!tIsValidName(name)) { + if (!tNameIsValid(name)) { return -1; } @@ -230,7 +154,7 @@ int32_t tNameLen(const SName* name) { } } -bool tIsValidName(const SName* name) { +bool tNameIsValid(const SName* name) { assert(name != NULL); if (!VALID_NAME_TYPE(name->type)) { @@ -265,13 +189,13 @@ int32_t tNameGetDbName(const SName* name, char* dst) { int32_t tNameGetFullDbName(const SName* name, char* dst) { assert(name != NULL && dst != NULL); snprintf(dst, TSDB_ACCT_ID_LEN + TS_PATH_DELIMITER_LEN + TSDB_DB_NAME_LEN, // there is a over write risk - "%s.%s", name->acctId, name->dbname); + "%d.%s", name->acctId, name->dbname); return 0; } bool tNameIsEmpty(const SName* name) { assert(name != NULL); - return name->type == 0 || strlen(name->acctId) <= 0; + return name->type == 0 || name->acctId == 0; } const char* tNameGetTableName(const SName* name) { @@ -283,32 +207,23 @@ void tNameAssign(SName* dst, const SName* src) { memcpy(dst, src, sizeof(SName)); } -//int32_t tNameSetDbName(SName* dst, const char* acct, SStrToken* dbToken) { -// assert(dst != NULL && dbToken != NULL && acct != NULL); -// -// // too long account id or too long db name -// if (strlen(acct) >= tListLen(dst->acctId) || dbToken->n >= tListLen(dst->dbname)) { -// return -1; -// } -// -// dst->type = TSDB_DB_NAME_T; -// tstrncpy(dst->acctId, acct, tListLen(dst->acctId)); -// tstrncpy(dst->dbname, dbToken->z, dbToken->n + 1); -// return 0; -//} - -int32_t tNameSetAcctId(SName* dst, const char* acct) { - assert(dst != NULL && acct != NULL); +int32_t tNameSetDbName(SName* dst, int32_t acct, const char* dbName, size_t nameLen) { + assert(dst != NULL && dbName != NULL && nameLen > 0); // too long account id or too long db name - if (strlen(acct) >= tListLen(dst->acctId)) { + if (nameLen >= tListLen(dst->dbname)) { return -1; } - tstrncpy(dst->acctId, acct, tListLen(dst->acctId)); + dst->type = TSDB_DB_NAME_T; + dst->acctId = acct; + tstrncpy(dst->dbname, dbName, nameLen + 1); + return 0; +} - assert(strlen(dst->acctId) > 0); - +int32_t tNameSetAcctId(SName* dst, int32_t acctId) { + assert(dst != NULL && acct != NULL); + dst->acctId = acctId; return 0; } @@ -325,14 +240,11 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { int32_t len = (int32_t)(p - str); // too long account id or too long db name - if ((len >= tListLen(dst->acctId)) || (len <= 0)) { - return -1; - } - - memcpy (dst->acctId, str, len); - dst->acctId[len] = 0; - - assert(strlen(dst->acctId) > 0); +// if ((len >= tListLen(dst->acctId)) || (len <= 0)) { +// return -1; +// } +// memcpy (dst->acctId, str, len); + dst->acctId = strtoll(str, NULL, 10); } if ((type & T_NAME_DB) == T_NAME_DB) { diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 57a5023807..636cf0a9a8 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -10,4 +10,4 @@ add_subdirectory(catalog) add_subdirectory(executor) add_subdirectory(planner) add_subdirectory(function) -add_subdirectory(query) +add_subdirectory(qcom) diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index e6311152d6..f47e105b8a 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( catalog - PRIVATE os util common transport query + PRIVATE os util transport qcom ) ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 5e635aa6a1..6ab3020935 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util common catalog function transport query + PRIVATE os util catalog function transport qcom ) ADD_SUBDIRECTORY(test) diff --git a/source/libs/parser/inc/astGenerator.h b/source/libs/parser/inc/astGenerator.h index 954bc29e62..6ae40b0d71 100644 --- a/source/libs/parser/inc/astGenerator.h +++ b/source/libs/parser/inc/astGenerator.h @@ -171,8 +171,8 @@ typedef struct SCreateDbInfo { int8_t update; int8_t cachelast; SArray *keep; - int8_t dbType; - int16_t partitions; +// int8_t dbType; +// int16_t partitions; } SCreateDbInfo; typedef struct SCreateFuncInfo { diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 223d5a5768..de7cdd58b8 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -5,5 +5,7 @@ #include "taosmsg.h" SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen); +SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen); +SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen); #endif // TDENGINE_ASTTOMSG_H diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index b402621903..c588a34a40 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -46,6 +46,7 @@ SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index); int32_t parserValidateIdToken(SToken* pToken); int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf); +int32_t parserValidateNameToken(SToken* pToken); int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg); int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr); diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index d12278632b..dda30c56fc 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -948,27 +948,24 @@ void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam) { void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) { pDBInfo->compressionLevel = -1; - pDBInfo->walLevel = -1; + pDBInfo->walLevel = -1; pDBInfo->fsyncPeriod = -1; - pDBInfo->commitTime = -1; + pDBInfo->commitTime = -1; pDBInfo->maxTablesPerVnode = -1; - pDBInfo->cacheBlockSize = -1; - pDBInfo->numOfBlocks = -1; + pDBInfo->cacheBlockSize = -1; + pDBInfo->numOfBlocks = -1; pDBInfo->maxRowsPerBlock = -1; pDBInfo->minRowsPerBlock = -1; - pDBInfo->daysPerFile = -1; + pDBInfo->daysPerFile = -1; pDBInfo->replica = -1; - pDBInfo->quorum = -1; - pDBInfo->keep = NULL; + pDBInfo->quorum = -1; + pDBInfo->keep = NULL; - pDBInfo->update = -1; + pDBInfo->update = -1; pDBInfo->cachelast = -1; - pDBInfo->dbType = -1; - pDBInfo->partitions = -1; - memset(&pDBInfo->precision, 0, sizeof(SToken)); } diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 6bfbd5ebed..9d57fa87f7 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -1,23 +1,159 @@ #include "parserInt.h" +#include "parserUtil.h" SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) { - SCreateUserMsg *pMsg = (SCreateUserMsg *)calloc(1, sizeof(SCreateUserMsg)); + SCreateUserMsg* pMsg = (SCreateUserMsg*)calloc(1, sizeof(SCreateUserMsg)); if (pMsg == NULL) { -// tscError("0x%" PRIx64 " failed to malloc for query msg", id); + // tscError("0x%" PRIx64 " failed to malloc for query msg", id); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } - SUserInfo *pUser = &pInfo->pMiscInfo->user; + SUserInfo* pUser = &pInfo->pMiscInfo->user; strncpy(pMsg->user, pUser->user.z, pUser->user.n); pMsg->type = pUser->type; pMsg->superUser = (int8_t)pUser->type; if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { -// pMsg->privilege = (char)pCmd->count; + // pMsg->privilege = (char)pCmd->count; } else { strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n); } return pMsg; -} \ No newline at end of file +} + +SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen) { + SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg)); + + pShowMsg->type = pShowInfo->showType; + + if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { + SToken* pPattern = &pShowInfo->pattern; + if (pPattern->type > 0) { // only show tables support wildcard query + strncpy(pShowMsg->payload, pPattern->z, pPattern->n); + pShowMsg->payloadLen = htons(pPattern->n); + } + } else { + SToken* pEpAddr = &pShowInfo->prefix; + assert(pEpAddr->n > 0 && pEpAddr->type > 0); + + strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); + pShowMsg->payloadLen = htons(pEpAddr->n); + } + + return pShowMsg; +} + +static int32_t setKeepOption(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb, SMsgBuf* pMsgBuf) { + const char* msg1 = "invalid number of keep options"; + const char* msg2 = "invalid keep value"; + const char* msg3 = "invalid keep value, should be keep0 <= keep1 <= keep2"; + + pMsg->daysToKeep0 = htonl(-1); + pMsg->daysToKeep1 = htonl(-1); + pMsg->daysToKeep2 = htonl(-1); + + SArray* pKeep = pCreateDb->keep; + if (pKeep != NULL) { + size_t s = taosArrayGetSize(pKeep); +#ifdef _STORAGE + if (s >= 4 ||s <= 0) { +#else + if (s != 1) { +#endif + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + +// tListI* p0 = taosArrayGet(pKeep, 0); +// tVariantListItem* p1 = (s > 1) ? taosArrayGet(pKeep, 1) : p0; +// tVariantListItem* p2 = (s > 2) ? taosArrayGet(pKeep, 2) : p1; +// +// if ((int32_t)p0->pVar.i64 <= 0 || (int32_t)p1->pVar.i64 <= 0 || (int32_t)p2->pVar.i64 <= 0) { +// return buildInvalidOperationMsg(pMsgBuf, msg2); +// } +// if (!(((int32_t)p0->pVar.i64 <= (int32_t)p1->pVar.i64) && ((int32_t)p1->pVar.i64 <= (int32_t)p2->pVar.i64))) { +// return buildInvalidOperationMsg(pMsgBuf, msg3); +// } +// +// pMsg->daysToKeep0 = htonl((int32_t)p0->pVar.i64); +// pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64); +// pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64); + + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t setTimePrecision(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDbInfo, SMsgBuf* pMsgBuf) { + const char* msg = "invalid time precision"; + + pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default + + SToken* pToken = &pCreateDbInfo->precision; + if (pToken->n > 0) { + pToken->n = strdequote(pToken->z); + + if (strncmp(pToken->z, TSDB_TIME_PRECISION_MILLI_STR, pToken->n) == 0 && + strlen(TSDB_TIME_PRECISION_MILLI_STR) == pToken->n) { + // time precision for this db: million second + pMsg->precision = TSDB_TIME_PRECISION_MILLI; + } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_MICRO_STR, pToken->n) == 0 && + strlen(TSDB_TIME_PRECISION_MICRO_STR) == pToken->n) { + pMsg->precision = TSDB_TIME_PRECISION_MICRO; + } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_NANO_STR, pToken->n) == 0 && + strlen(TSDB_TIME_PRECISION_NANO_STR) == pToken->n) { + pMsg->precision = TSDB_TIME_PRECISION_NANO; + } else { + return buildInvalidOperationMsg(pMsgBuf, msg); + } + } + + return TSDB_CODE_SUCCESS; +} + +static void doSetDbOptions(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb) { + pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); + pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); + pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); + pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime); + pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock); + pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock); + pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); + pMsg->compression = pCreateDb->compressionLevel; + pMsg->walLevel = (char)pCreateDb->walLevel; + pMsg->replications = pCreateDb->replica; + pMsg->quorum = pCreateDb->quorum; + pMsg->ignoreExist = pCreateDb->ignoreExists; + pMsg->update = pCreateDb->update; + pMsg->cacheLastRow = pCreateDb->cachelast; +} + +int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbSql, SMsgBuf* pMsgBuf) { + doSetDbOptions(pCreateDbMsg, pCreateDbSql); + + if (setKeepOption(pCreateDbMsg, pCreateDbSql, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + if (setTimePrecision(pCreateDbMsg, pCreateDbSql, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + // todo configurable + pCreateDbMsg->numOfVgroups = htonl(2); + + return TSDB_CODE_SUCCESS; +} + +SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen) { + SCreateDbMsg* pCreateMsg = calloc(1, sizeof(SCreateDbMsg)); + + SMsgBuf msg = {.buf = msgBuf, .len = msgLen}; + if (setDbOptions(pCreateMsg, pCreateDbInfo, &msg) != TSDB_CODE_SUCCESS) { + tfree(pCreateMsg); + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + return pCreateMsg; +} diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 2ee95732d0..099932a479 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -759,11 +759,6 @@ int32_t validateIntervalNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMs // It is a time window query pQueryInfo->info.timewindow = true; return TSDB_CODE_SUCCESS; - // disable it temporarily -// bool interpQuery = tscIsPointInterpQuery(pQueryInfo); -// if ((pSqlNode->interval.token == TK_EVERY && (!interpQuery)) || (pSqlNode->interval.token == TK_INTERVAL && interpQuery)) { -// return buildInvalidOperationMsg(pMsgBuf, msg4); -// } } int32_t validateSessionNode(SQueryStmtInfo *pQueryInfo, SSessionWindowVal* pSession, int32_t precision, SMsgBuf* pMsgBuf) { @@ -3707,14 +3702,6 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer return TSDB_CODE_SUCCESS; } - case TSDB_SQL_SHOW: { - if (setShowInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - break; - } - case TSDB_SQL_CREATE_FUNCTION: case TSDB_SQL_DROP_FUNCTION: { code = handleUserDefinedFunc(pSql, pInfo); @@ -3725,35 +3712,6 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer break; } - case TSDB_SQL_ALTER_DB: - case TSDB_SQL_CREATE_DB: { - const char* msg1 = "invalid db name"; - const char* msg2 = "name too long"; - - SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); - if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) { - return buildInvalidOperationMsg(pMsgBuf, msg2); - } - - char buf[TSDB_DB_NAME_LEN] = {0}; - SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf)); - - if (tscValidateName(&token) != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); - } - - int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &token); - if (ret != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg2); - } - - if (parseCreateDBOptions(pCmd, pCreateDB) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - break; - } - case TSDB_SQL_CREATE_DNODE: { const char* msg = "invalid host name (ip address)"; @@ -4133,28 +4091,87 @@ static int32_t setShowInfo(struct SSqlInfo* pInfo, void** output, int32_t* msgLe } } - SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg)); - pShowMsg->type = pShowInfo->showType; - - if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { - SToken* pPattern = &pShowInfo->pattern; - if (pPattern->type > 0) { // only show tables support wildcard query - strncpy(pShowMsg->payload, pPattern->z, pPattern->n); - pShowMsg->payloadLen = htons(pPattern->n); - } - } else { - SToken* pEpAddr = &pShowInfo->prefix; - assert(pEpAddr->n > 0 && pEpAddr->type > 0); - - strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); - pShowMsg->payloadLen = htons(pEpAddr->n); - } - - *output = pShowMsg; - *msgLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); + *output = buildShowMsg(pShowInfo, 0, pMsgBuf->buf, pMsgBuf->len); + *msgLen = sizeof(SShowMsg)/* + htons(pShowMsg->payloadLen)*/; return TSDB_CODE_SUCCESS; } +// can only perform the parameters based on the macro definitation +static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) { + char msg[512] = {0}; + + if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { + snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->replications != -1 && + (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { + snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, + TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + int32_t blocks = ntohl(pCreate->totalBlocks); + if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) { + snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks, + TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->quorum != -1 && + (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { + snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, + TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + int32_t val = htonl(pCreate->daysPerFile); + if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { + snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, + TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = htonl(pCreate->cacheBlockSize); + if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { + snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, + TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO && + pCreate->precision != TSDB_TIME_PRECISION_NANO) { + snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision, + TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = htonl(pCreate->commitTime); + if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { + snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, + TSDB_MIN_COMMIT_TIME, TSDB_MAX_COMMIT_TIME); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = htonl(pCreate->fsyncPeriod); + if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { + snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, + TSDB_MIN_FSYNC_PERIOD, TSDB_MAX_FSYNC_PERIOD); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->compression != -1 && + (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { + snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, + TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) { int32_t code = 0; @@ -4216,6 +4233,36 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in code = setShowInfo(pInfo, output, outputLen, pMsgBuf); break; } + + case TSDB_SQL_ALTER_DB: + case TSDB_SQL_CREATE_DB: { + const char* msg1 = "invalid db name"; + const char* msg2 = "name too long"; + + SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); + if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + char buf[TSDB_DB_NAME_LEN] = {0}; + SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf)); + + if (parserValidateNameToken(&token) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + SCreateDbMsg* pCreateMsg = buildCreateDbMsg(pCreateDB, pMsgBuf->buf, pMsgBuf->len); + if (doCheckDbOptions(pCreateMsg, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + strncpy(pCreateMsg->db, token.z, token.n); + + *output = pCreateMsg; + *outputLen = sizeof(SCreateDbMsg); + break; + } + default: break; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 5e08859a66..e0ac7c295b 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -77,7 +77,7 @@ static int32_t tnameComparFn(const void* p1, const void* p2) { SName* pn1 = (SName*)p1; SName* pn2 = (SName*)p2; - int32_t ret = strncmp(pn1->acctId, pn2->acctId, tListLen(pn1->acctId)); + int32_t ret = pn1->acctId - pn2->acctId; if (ret != 0) { return ret > 0? 1:-1; } else { diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 3a61f5912b..b72bc06324 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -122,6 +122,25 @@ int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf) { return TSDB_CODE_SUCCESS; } +int32_t parserValidateNameToken(SToken* pToken) { + if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + // it is a token quoted with escape char '`' + if (pToken->z[0] == TS_ESCAPE_CHAR && pToken->z[pToken->n - 1] == TS_ESCAPE_CHAR) { + return TSDB_CODE_SUCCESS; + } + + char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); + if (sep != NULL) { // It is a complex type, not allow + return TSDB_CODE_TSC_INVALID_OPERATION; + } + + strntolower(pToken->z, pToken->z, pToken->n); + return TSDB_CODE_SUCCESS; +} + int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) { strncpy(pBuf->buf, msg, pBuf->len); return TSDB_CODE_TSC_INVALID_OPERATION; diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 53a7ba48f8..d674462fc0 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -2519,7 +2519,7 @@ static void yy_reduce( { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; case 105: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy256); yymsp[1].minor.yy256.dbType = TSDB_DB_TYPE_DEFAULT;} +{setDefaultCreateDbOption(&yymsp[1].minor.yy256);} break; case 106: /* db_optr ::= db_optr cache */ { yylhsminor.yy256 = yymsp[-1].minor.yy256; yylhsminor.yy256.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } @@ -2590,16 +2590,16 @@ static void yy_reduce( break; case 121: /* topic_optr ::= db_optr */ case 131: /* alter_topic_optr ::= alter_db_optr */ yytestcase(yyruleno==131); -{ yylhsminor.yy256 = yymsp[0].minor.yy256; yylhsminor.yy256.dbType = TSDB_DB_TYPE_TOPIC; } +{ yylhsminor.yy256 = yymsp[0].minor.yy256;} yymsp[0].minor.yy256 = yylhsminor.yy256; break; case 122: /* topic_optr ::= topic_optr partitions */ case 132: /* alter_topic_optr ::= alter_topic_optr partitions */ yytestcase(yyruleno==132); -{ yylhsminor.yy256 = yymsp[-1].minor.yy256; yylhsminor.yy256.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); } +{ yylhsminor.yy256 = yymsp[-1].minor.yy256; } yymsp[-1].minor.yy256 = yylhsminor.yy256; break; case 123: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy256); yymsp[1].minor.yy256.dbType = TSDB_DB_TYPE_DEFAULT;} +{ setDefaultCreateDbOption(&yymsp[1].minor.yy256); } break; case 133: /* typename ::= ids */ { diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index 8a309af526..c42019e23d 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( planner - PRIVATE os util common cjson catalog parser transport function query + PRIVATE os util catalog cjson parser transport function query ) ADD_SUBDIRECTORY(test) diff --git a/source/libs/qcom/CMakeLists.txt b/source/libs/qcom/CMakeLists.txt new file mode 100644 index 0000000000..41cf1826bc --- /dev/null +++ b/source/libs/qcom/CMakeLists.txt @@ -0,0 +1,12 @@ +aux_source_directory(src QUERY_SRC) +add_library(qcom ${QUERY_SRC}) +target_include_directories( + qcom + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qcom" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + qcom + PRIVATE os util transport +) diff --git a/source/libs/query/inc/queryInt.h b/source/libs/qcom/inc/queryInt.h similarity index 100% rename from source/libs/query/inc/queryInt.h rename to source/libs/qcom/inc/queryInt.h diff --git a/source/libs/query/src/querymsg.c b/source/libs/qcom/src/querymsg.c similarity index 100% rename from source/libs/query/src/querymsg.c rename to source/libs/qcom/src/querymsg.c diff --git a/source/libs/query/CMakeLists.txt b/source/libs/query/CMakeLists.txt deleted file mode 100644 index 579a4b279c..0000000000 --- a/source/libs/query/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -aux_source_directory(src QUERY_SRC) -add_library(query ${QUERY_SRC}) -target_include_directories( - query - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/query" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -target_link_libraries( - query - PRIVATE os util common transport -) diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 6675b7f5ec..9df723c29e 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,5 +9,5 @@ target_include_directories( target_link_libraries( scheduler - PRIVATE os util planner common query + PRIVATE os util planner qcom ) \ No newline at end of file From 2e68133ef3676145cdaf89f22468e8d2de97e1fc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Dec 2021 16:44:32 +0800 Subject: [PATCH 37/50] [td-11818] refactor. --- source/libs/parser/src/astToMsg.c | 5 +++-- source/libs/parser/src/astValidate.c | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 9d57fa87f7..1b46faecec 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -79,7 +79,6 @@ static int32_t setKeepOption(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb, // pMsg->daysToKeep0 = htonl((int32_t)p0->pVar.i64); // pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64); // pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64); - } return TSDB_CODE_SUCCESS; @@ -152,7 +151,9 @@ SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32 SMsgBuf msg = {.buf = msgBuf, .len = msgLen}; if (setDbOptions(pCreateMsg, pCreateDbInfo, &msg) != TSDB_CODE_SUCCESS) { tfree(pCreateMsg); - return TSDB_CODE_TSC_INVALID_OPERATION; + terrno = TSDB_CODE_TSC_INVALID_OPERATION; + + return NULL; } return pCreateMsg; diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 099932a479..135774cd3b 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4171,7 +4171,6 @@ static int32_t doCheckDbOptions(SCreateDbMsg* pCreate, SMsgBuf* pMsgBuf) { return TSDB_CODE_SUCCESS; } - int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) { int32_t code = 0; From 89f4a7adc628d976fd4e3e9fc5e85236db31b8c8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Dec 2021 16:44:09 +0800 Subject: [PATCH 38/50] TD-10431 change numOfVgroups as config of db --- include/util/tdef.h | 1 + source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndDb.c | 23 +++++++++++---------- source/dnode/mnode/impl/src/mndVgroup.c | 27 ++++--------------------- tests/CMakeLists.txt | 17 +++------------- tests/tsim/CMakeLists.txt | 23 ++++++++++++--------- tools/shell/src/shellLinux.c | 2 ++ 7 files changed, 37 insertions(+), 58 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index dfb53de58f..02b5a1e620 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -235,6 +235,7 @@ do { \ #define TSDB_MAX_VNODES 512 #define TSDB_MIN_VNODES_PER_DB 1 #define TSDB_MAX_VNODES_PER_DB 4096 +#define TSDB_DEFAULT_VN_PER_DB 2 #define TSDB_DNODE_ROLE_ANY 0 #define TSDB_DNODE_ROLE_MGMT 1 diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 74138500bd..36b6725737 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -182,6 +182,7 @@ typedef struct { } SUserObj; typedef struct { + int32_t numOfVgroups; int32_t cacheBlockSize; int32_t totalBlocks; int32_t daysPerFile; @@ -209,7 +210,6 @@ typedef struct { int64_t uid; int32_t cfgVersion; int32_t vgVersion; - int32_t numOfVgroups; int8_t hashMethod; // default is 1 SDbCfg cfg; } SDbObj; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 188ca7963c..fbbae13b63 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -77,8 +77,8 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT64(pRaw, dataPos, pDb->uid) SDB_SET_INT32(pRaw, dataPos, pDb->cfgVersion) SDB_SET_INT32(pRaw, dataPos, pDb->vgVersion) - SDB_SET_INT32(pRaw, dataPos, pDb->numOfVgroups) SDB_SET_INT8(pRaw, dataPos, pDb->hashMethod) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfVgroups) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile) @@ -124,8 +124,8 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfgVersion) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->vgVersion) - SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->numOfVgroups) SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->hashMethod) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.numOfVgroups) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile) @@ -163,7 +163,6 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { pOldDb->updateTime = pNewDb->createdTime; pOldDb->cfgVersion = pNewDb->cfgVersion; pOldDb->vgVersion = pNewDb->vgVersion; - pOldDb->numOfVgroups = pNewDb->numOfVgroups; memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg)); return 0; } @@ -195,6 +194,7 @@ static int32_t mndCheckDbName(char *dbName, SUserObj *pUser) { } static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { + if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1; if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1; if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1; if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1; @@ -222,6 +222,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { } static void mndSetDefaultDbCfg(SDbCfg *pCfg) { + if (pCfg->numOfVgroups < 0) pCfg->numOfVgroups = TSDB_DEFAULT_VN_PER_DB; if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; if (pCfg->totalBlocks < 0) pCfg->totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; @@ -246,7 +247,7 @@ static int32_t mndSetRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb if (pDbRaw == NULL || mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1; sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING); - for (int v = 0; v < pDb->numOfVgroups; ++v) { + for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); if (pVgRaw == NULL || mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING); @@ -260,7 +261,7 @@ static int32_t mndSetUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb if (pDbRaw == NULL || mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1; sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED); - for (int v = 0; v < pDb->numOfVgroups; ++v) { + for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); if (pVgRaw == NULL || mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1; sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED); @@ -274,7 +275,7 @@ static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVg if (pDbRaw == NULL || mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; sdbSetRawStatus(pDbRaw, SDB_STATUS_READY); - for (int v = 0; v < pDb->numOfVgroups; ++v) { + for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); @@ -298,11 +299,11 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat dbObj.createdTime = taosGetTimestampMs(); dbObj.updateTime = dbObj.createdTime; dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN); - dbObj.numOfVgroups = pCreate->numOfVgroups; dbObj.hashMethod = 1; dbObj.cfgVersion = 1; dbObj.vgVersion = 1; - dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize, + dbObj.cfg = (SDbCfg){.numOfVgroups = pCreate->numOfVgroups, + .cacheBlockSize = pCreate->cacheBlockSize, .totalBlocks = pCreate->totalBlocks, .daysPerFile = pCreate->daysPerFile, .daysToKeep0 = pCreate->daysToKeep0, @@ -643,7 +644,7 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { return -1; } - int32_t contLen = sizeof(SUseDbRsp) + pDb->numOfVgroups * sizeof(SVgroupInfo); + int32_t contLen = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo); SUseDbRsp *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -654,7 +655,7 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { if (pUse->vgVersion < pDb->vgVersion) { void *pIter = NULL; - while (vindex < pDb->numOfVgroups) { + while (vindex < pDb->cfg.numOfVgroups) { SVgObj *pVgroup = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; @@ -888,7 +889,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDb->numOfVgroups; + *(int16_t *)pWrite = pDb->cfg.numOfVgroups; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index b880434bf6..c2be7fa39a 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -156,11 +156,6 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { sdbRelease(pSdb, pVgroup); } -static int32_t mndGetDefaultVgroupSize(SMnode *pMnode) { - // todo - return 2; -} - static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup) { SSdb *pSdb = pMnode->pSdb; int32_t allocedVnodes = 0; @@ -193,21 +188,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup) { } int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { - if (pDb->numOfVgroups != -1 && - (pDb->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pDb->numOfVgroups > TSDB_MAX_VNODES_PER_DB)) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - return -1; - } - - if (pDb->numOfVgroups == -1) { - pDb->numOfVgroups = mndGetDefaultVgroupSize(pMnode); - if (pDb->numOfVgroups < 0) { - terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - return -1; - } - } - - SVgObj *pVgroups = calloc(pDb->numOfVgroups, sizeof(SVgObj)); + SVgObj *pVgroups = calloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); if (pVgroups == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -217,9 +198,9 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); uint32_t hashMin = 0; uint32_t hashMax = UINT32_MAX; - uint32_t hashInterval = (hashMax - hashMin) / pDb->numOfVgroups; + uint32_t hashInterval = (hashMax - hashMin) / pDb->cfg.numOfVgroups; - for (uint32_t v = 0; v < pDb->numOfVgroups; v++) { + for (uint32_t v = 0; v < pDb->cfg.numOfVgroups; v++) { SVgObj *pVgroup = &pVgroups[v]; pVgroup->vgId = maxVgId++; pVgroup->createdTime = taosGetTimestampMs(); @@ -227,7 +208,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { pVgroup->version = 1; pVgroup->dbUid = pDb->uid; pVgroup->hashBegin = hashMin + hashInterval * v; - if (v == pDb->numOfVgroups - 1) { + if (v == pDb->cfg.numOfVgroups - 1) { pVgroup->hashEnd = hashMax; } else { pVgroup->hashEnd = hashMin + hashInterval * (v + 1) - 1; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e21905af3b..966eb94354 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,15 +1,4 @@ -# generate debug version: -# mkdir debug; cd debug; cmake -DCMAKE_BUILD_TYPE=Debug .. -# generate release version: -# mkdir release; cd release; cmake -DCMAKE_BUILD_TYPE=Release .. - -CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) -PROJECT(TDengine) - -SET(CMAKE_C_STANDARD 11) -SET(CMAKE_VERBOSE_MAKEFILE ON) - -ADD_SUBDIRECTORY(examples/c) +#ADD_SUBDIRECTORY(examples/c) ADD_SUBDIRECTORY(tsim) -ADD_SUBDIRECTORY(test/c) -ADD_SUBDIRECTORY(comparisonTest/tdengine) +#ADD_SUBDIRECTORY(test/c) +#ADD_SUBDIRECTORY(comparisonTest/tdengine) diff --git a/tests/tsim/CMakeLists.txt b/tests/tsim/CMakeLists.txt index 50b42941af..9d44738e3e 100644 --- a/tests/tsim/CMakeLists.txt +++ b/tests/tsim/CMakeLists.txt @@ -1,9 +1,14 @@ -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) -INCLUDE_DIRECTORIES(inc) - -AUX_SOURCE_DIRECTORY(src SRC) -ADD_EXECUTABLE(tsim ${SRC}) -TARGET_LINK_LIBRARIES(tsim taos_static trpc tutil pthread cJson) +aux_source_directory(src TSIM_SRC) +add_executable(tsim ${TSIM_SRC}) +target_link_libraries( + tsim + PUBLIC taos + PUBLIC util + PUBLIC common + PUBLIC os + PUBLIC cJson +) +target_include_directories( + tsim + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index 766a57c968..81d8c0a35b 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -219,7 +219,9 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { argp_parse(&argp, argc, argv, 0, 0, arguments); if (arguments->abort) { #ifndef _ALPINE + #if 0 error(10, 0, "ABORTED"); + #endif #else abort(); #endif From 5d38440cda8af3a542bd4a02086f29c226abfb97 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Dec 2021 16:53:14 +0800 Subject: [PATCH 39/50] [td-11818] refactor. --- tools/shell/src/shellEngine.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 3a6cbd259e..1ad61ee2b0 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -341,7 +341,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { return; } - if (true /*!tscIsUpdateQuery(pSql)*/) { // select and show kinds of commands + TAOS_FIELD* pFields = taos_fetch_fields(pSql); + if (pFields != NULL) { // select and show kinds of commands int error_no = 0; int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); From 8e68fa6e4d13e533992cdbef4f07ce9eca96c509 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Dec 2021 17:07:36 +0800 Subject: [PATCH 40/50] TD-10431 tsim compile --- CMakeLists.txt | 1 + tests/tsim/CMakeLists.txt | 2 +- tests/tsim/inc/sim.h | 7 +- tests/tsim/inc/simParse.h | 6 +- tests/tsim/src/simExe.c | 247 +++------------------------------ tests/tsim/src/simMain.c | 4 - tests/tsim/src/simParse.c | 1 - tests/tsim/src/simSystem.c | 7 +- tools/shell/inc/shellCommand.h | 4 +- 9 files changed, 32 insertions(+), 247 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 681559a37b..1b8a3f9a1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,7 @@ if(${BUILD_TEST}) endif(${BUILD_TEST}) add_subdirectory(source) add_subdirectory(tools) +add_subdirectory(tests) # docs add_subdirectory(docs) diff --git a/tests/tsim/CMakeLists.txt b/tests/tsim/CMakeLists.txt index 9d44738e3e..81737809d9 100644 --- a/tests/tsim/CMakeLists.txt +++ b/tests/tsim/CMakeLists.txt @@ -6,7 +6,7 @@ target_link_libraries( PUBLIC util PUBLIC common PUBLIC os - PUBLIC cJson + PUBLIC cjson ) target_include_directories( tsim diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 39c6f6ac36..4cb08d6401 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef __SIM_H__ -#define __SIM_H__ +#ifndef _TD_SIM_H_ +#define _TD_SIM_H_ #include #include @@ -150,7 +150,6 @@ extern int32_t simScriptPos; extern int32_t simScriptSucced; extern int32_t simDebugFlag; extern char tsScriptDir[]; -extern bool simAsyncQuery; extern bool abortExecution; SScript *simParseScript(char *fileName); @@ -178,4 +177,4 @@ bool simExecuteLineInsertCmd(SScript *script, char *option); bool simExecuteLineInsertErrorCmd(SScript *script, char *option); void simVisuallizeOption(SScript *script, char *src, char *dst); -#endif \ No newline at end of file +#endif /*_TD_SIM_H_*/ \ No newline at end of file diff --git a/tests/tsim/inc/simParse.h b/tests/tsim/inc/simParse.h index ef7d8e5ce7..addd74bf80 100644 --- a/tests/tsim/inc/simParse.h +++ b/tests/tsim/inc/simParse.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef __SIM_PARSE_H__ -#define __SIM_PARSE_H__ +#ifndef _TD_SIM_PARSE_H_ +#define _TD_SIM_PARSE_H_ #define MAX_NUM_CMD 64 #define MAX_NUM_LABLES 100 @@ -52,4 +52,4 @@ typedef struct { bool simParseExpression(char *token, int32_t lineNum); -#endif \ No newline at end of file +#endif /*_TD_SIM_PARSE_H_*/ \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 7a75dd7d85..4f64564404 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -14,14 +14,14 @@ */ #define _DEFAULT_SOURCE -#include "../../../include/client/taos.h" +#include "taos.h" #include "cJSON.h" #include "os.h" #include "sim.h" #include "taoserror.h" #include "tglobal.h" #include "tutil.h" -#undef TAOS_MEM_CHECK +#include "ttypes.h" void simLogSql(char *sql, bool useSharp) { static FILE *fp = NULL; @@ -466,10 +466,6 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) { strcpy(dst + dstLen, src); } -void simCloseRestFulConnect(SScript *script) { - memset(script->auth, 0, sizeof(script->auth)); -} - void simCloseNativeConnect(SScript *script) { if (script->taos == NULL) return; @@ -480,166 +476,7 @@ void simCloseNativeConnect(SScript *script) { } void simCloseTaosdConnect(SScript *script) { - if (simAsyncQuery) { - simCloseRestFulConnect(script); - } else { - simCloseNativeConnect(script); - } -} -// {"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"} -// {"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1} -// {"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10} -int32_t simParseHttpCommandResult(SScript *script, char *command) { - cJSON* root = cJSON_Parse(command); - if (root == NULL) { - simError("script:%s, failed to parse json, response:%s", script->fileName, command); - return -1; - } - - cJSON *status = cJSON_GetObjectItem(root, "status"); - if (status == NULL) { - simError("script:%s, failed to parse json, status is null, response:%s", script->fileName, command); - cJSON_Delete(root); - return -1; - } - - if (status->valuestring == NULL || strlen(status->valuestring) == 0) { - simError("script:%s, failed to parse json, status value is null, response:%s", script->fileName, command); - cJSON_Delete(root); - return -1; - } - - if (strcmp(status->valuestring, "succ") != 0) { - cJSON *code = cJSON_GetObjectItem(root, "code"); - if (code == NULL) { - simError("script:%s, failed to parse json, code is null, response:%s", script->fileName, command); - cJSON_Delete(root); - return -1; - } - int32_t retcode = (int32_t)code->valueint; - if (retcode != 1017) { - simError("script:%s, json:status:%s not equal to succ, response:%s", script->fileName, status->valuestring, - command); - cJSON_Delete(root); - return retcode; - } else { - simDebug("script:%s, json:status:%s not equal to succ, but code is %d, response:%s", script->fileName, - status->valuestring, retcode, command); - cJSON_Delete(root); - return 0; - } - } - - cJSON *desc = cJSON_GetObjectItem(root, "desc"); - if (desc != NULL) { - if (desc->valuestring == NULL || strlen(desc->valuestring) == 0) { - simError("script:%s, failed to parse json, desc value is null, response:%s", script->fileName, command); - cJSON_Delete(root); - return -1; - } - strcpy(script->auth, desc->valuestring); - cJSON_Delete(root); - return 0; - } - - cJSON *data = cJSON_GetObjectItem(root, "data"); - if (data == NULL) { - simError("script:%s, failed to parse json, data is null, response:%s", script->fileName, command); - cJSON_Delete(root); - return -1; - } - - int32_t rowsize = cJSON_GetArraySize(data); - if (rowsize < 0) { - simError("script:%s, failed to parse json:data, data size %d, response:%s", script->fileName, rowsize, command); - cJSON_Delete(root); - return -1; - } - - int32_t rowIndex = 0; - sprintf(script->rows, "%d", rowsize); - for (int32_t r = 0; r < rowsize; ++r) { - cJSON *row = cJSON_GetArrayItem(data, r); - if (row == NULL) continue; - if (rowIndex++ >= 10) break; - - int32_t colsize = cJSON_GetArraySize(row); - if (colsize < 0) { - break; - } - - colsize = MIN(10, colsize); - for (int32_t c = 0; c < colsize; ++c) { - cJSON *col = cJSON_GetArrayItem(row, c); - if (col->valuestring != NULL) { - strcpy(script->data[r][c], col->valuestring); - } else { - if (col->numberstring[0] == 0) { - strcpy(script->data[r][c], "null"); - } else { - strcpy(script->data[r][c], col->numberstring); - } - } - } - } - - return 0; -} - -int32_t simExecuteRestFulCommand(SScript *script, char *command) { - char buf[5000] = {0}; - sprintf(buf, "%s 2>/dev/null", command); - - FILE *fp = popen(buf, "r"); - if (fp == NULL) { - simError("failed to execute %s", buf); - return -1; - } - - int32_t mallocSize = 2000; - int32_t alreadyReadSize = 0; - char * content = malloc(mallocSize); - - while (!feof(fp)) { - int32_t availSize = mallocSize - alreadyReadSize; - int32_t len = (int32_t)fread(content + alreadyReadSize, 1, availSize, fp); - if (len >= availSize) { - alreadyReadSize += len; - mallocSize *= 2; - content = realloc(content, mallocSize); - } - } - - pclose(fp); - - return simParseHttpCommandResult(script, content); -} - -bool simCreateRestFulConnect(SScript *script, char *user, char *pass) { - char command[4096]; - sprintf(command, "curl 127.0.0.1:6041/rest/login/%s/%s", user, pass); - - bool success = false; - for (int32_t attempt = 0; attempt < 10; ++attempt) { - success = simExecuteRestFulCommand(script, command) == 0; - if (!success) { - simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), - attempt); - taosMsleep(1000); - } else { - simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt); - break; - } - } - - if (!success) { - sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, - taos_errstr(NULL)); - return false; - } - - simDebug("script:%s, connect taosd successed, auth:%p", script->fileName, script->auth); - return true; + simCloseNativeConnect(script); } bool simCreateNativeConnect(SScript *script, char *user, char *pass) { @@ -651,7 +488,7 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) { return false; } - taos = taos_connect(NULL, user, pass, NULL, tsDnodeShellPort); + taos = taos_connect(NULL, user, pass, NULL, 0); if (taos == NULL) { simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), attempt); @@ -684,11 +521,7 @@ bool simCreateTaosdConnect(SScript *script, char *rest) { user = token; } - if (simAsyncQuery) { - return simCreateRestFulConnect(script, user, TSDB_DEFAULT_PASS); - } else { - return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS); - } + return simCreateNativeConnect(script, user, TSDB_DEFAULT_PASS); } bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { @@ -710,7 +543,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { pSql = taos_query(script->taos, rest); ret = taos_errno(pSql); - if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { + if (ret == TSDB_CODE_MND_STB_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret)); ret = 0; @@ -877,41 +710,6 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { return true; } -bool simExecuteRestFulSqlCommand(SScript *script, char *rest) { - SCmdLine *line = &script->lines[script->linePos]; - char command[4096]; - sprintf(command, "curl -H 'Authorization: Taosd %s' -d \"%s\" 127.0.0.1:6041/rest/sql", script->auth, rest); - - int32_t ret = -1; - for (int32_t attempt = 0; attempt < 10; ++attempt) { - ret = simExecuteRestFulCommand(script, command); - if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { - simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, - tstrerror(ret)); - ret = 0; - break; - } else if (ret != 0) { - simDebug("script:%s, taos:%p, %s failed, ret:%d", script->fileName, script->taos, rest, ret); - - if (line->errorJump == SQL_JUMP_TRUE) { - script->linePos = line->jump; - return true; - } - taosMsleep(1000); - } else { - break; - } - } - - if (ret) { - sprintf(script->error, "lineNum:%d. sql:%s failed, ret:%d", line->lineNum, rest, ret); - return false; - } - - script->linePos++; - return true; -} - bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { char buf[3000]; SCmdLine *line = &script->lines[script->linePos]; @@ -935,7 +733,7 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { return true; } - if ((!simAsyncQuery && script->taos == NULL) || (simAsyncQuery && script->auth[0] == 0)) { + if (script->taos == NULL) { if (!simCreateTaosdConnect(script, "connect root")) { if (line->errorJump == SQL_JUMP_TRUE) { script->linePos = line->jump; @@ -951,11 +749,7 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { return true; } - if (simAsyncQuery) { - return simExecuteRestFulSqlCommand(script, rest); - } else { - return simExecuteNativeSqlCommand(script, rest, isSlow); - } + return simExecuteNativeSqlCommand(script, rest, isSlow); } bool simExecuteSqlCmd(SScript *script, char *rest) { @@ -1009,6 +803,7 @@ bool simExecuteRestfulCmd(SScript *script, char *rest) { return simExecuteSystemCmd(script, cmd); } + bool simExecuteSqlErrorCmd(SScript *script, char *rest) { char buf[3000]; SCmdLine *line = &script->lines[script->linePos]; @@ -1032,7 +827,7 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { return true; } - if ((!simAsyncQuery && script->taos == NULL) || (simAsyncQuery && script->auth[0] == 0)) { + if (script->taos == NULL) { if (!simCreateTaosdConnect(script, "connect root")) { if (line->errorJump == SQL_JUMP_TRUE) { script->linePos = line->jump; @@ -1048,17 +843,9 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { return true; } - int32_t ret; - TAOS_RES *pSql = NULL; - if (simAsyncQuery) { - char command[4096]; - sprintf(command, "curl -H 'Authorization: Taosd %s' -d '%s' 127.0.0.1:6041/rest/sql", script->auth, rest); - ret = simExecuteRestFulCommand(script, command); - } else { - pSql = taos_query(script->taos, rest); - ret = taos_errno(pSql); - taos_free_result(pSql); - } + TAOS_RES *pSql = pSql = taos_query(script->taos, rest); + int32_t ret = taos_errno(pSql); + taos_free_result(pSql); if (ret != TSDB_CODE_SUCCESS) { simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", script->fileName, script->taos, @@ -1084,7 +871,11 @@ bool simExecuteLineInsertCmd(SScript *script, char *rest) { simInfo("script:%s, %s", script->fileName, rest); simLogSql(buf, true); char * lines[] = {rest}; +#if 0 int32_t ret = taos_insert_lines(script->taos, lines, 1); +#else + int32_t ret = 0; +#endif if (ret == TSDB_CODE_SUCCESS) { simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest); script->linePos++; @@ -1107,7 +898,11 @@ bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { simInfo("script:%s, %s", script->fileName, rest); simLogSql(buf, true); char * lines[] = {rest}; +#if 0 int32_t ret = taos_insert_lines(script->taos, lines, 1); +#else + int32_t ret = 0; +#endif if (ret == TSDB_CODE_SUCCESS) { sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, script->taos, rest); script->linePos++; diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 7d74c62c7d..438bd41910 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -17,9 +17,7 @@ #include "os.h" #include "tglobal.h" #include "sim.h" -#undef TAOS_MEM_CHECK -bool simAsyncQuery = false; bool simExecSuccess = false; bool abortExecution = false; @@ -38,8 +36,6 @@ int32_t main(int32_t argc, char *argv[]) { tstrncpy(configDir, argv[++i], 128); } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { strcpy(scriptFile, argv[++i]); - } else if (strcmp(argv[i], "-a") == 0) { - simAsyncQuery = true; } else { printf("usage: %s [options] \n", argv[0]); printf(" [-c config]: config directory, default is: %s\n", configDir); diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 1acdcd2ac6..246d0d3829 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -62,7 +62,6 @@ #include "sim.h" #include "simParse.h" #include "tutil.h" -#undef TAOS_MEM_CHECK static SCommand *cmdHashList[MAX_NUM_CMD]; static SCmdLine cmdLine[MAX_CMD_LINES]; diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 65612930ef..283e84c7a8 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -14,15 +14,13 @@ */ #define _DEFAULT_SOURCE -#include "../../../include/client/taos.h" +#include "taos.h" #include "os.h" #include "sim.h" #include "taoserror.h" #include "tglobal.h" -#include "tsocket.h" #include "ttimer.h" #include "tutil.h" -#undef TAOS_MEM_CHECK SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; SCommand simCmdList[SIM_CMD_END]; @@ -81,9 +79,6 @@ char *simParseHostName(char *varName) { } bool simSystemInit() { - if (taos_init()) { - return false; - } taosGetFqdn(simHostName); simInitsimCmdList(); memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); diff --git a/tools/shell/inc/shellCommand.h b/tools/shell/inc/shellCommand.h index 6e4d3e382e..49f7dc0133 100644 --- a/tools/shell/inc/shellCommand.h +++ b/tools/shell/inc/shellCommand.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef __COMMAND_STRUCT__ -#define __COMMAND_STRUCT__ +#ifndef _TD_SHELL_COMMAND_H_ +#define _TD_SHELL_COMMAND_H_ #include "shell.h" From bbeaf1328b1d2f511dfedb5fee2d0424c446d5d6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Dec 2021 17:10:11 +0800 Subject: [PATCH 41/50] [td-11818] fix compiler error. --- source/libs/catalog/test/CMakeLists.txt | 2 +- source/libs/parser/test/CMakeLists.txt | 2 +- source/libs/planner/CMakeLists.txt | 2 +- source/libs/planner/test/CMakeLists.txt | 2 +- source/libs/scheduler/CMakeLists.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/catalog/test/CMakeLists.txt b/source/libs/catalog/test/CMakeLists.txt index 176978cc7f..3c7418bdcc 100644 --- a/source/libs/catalog/test/CMakeLists.txt +++ b/source/libs/catalog/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(catalogTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( catalogTest - PUBLIC os util common catalog transport gtest query taos + PUBLIC os util common catalog transport gtest qcom taos ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index 03b76152da..feae008273 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -15,7 +15,7 @@ TARGET_INCLUDE_DIRECTORIES( TARGET_LINK_LIBRARIES( parserTest - PUBLIC os util common parser catalog transport gtest function planner query + PUBLIC os util common parser catalog transport gtest function planner qcom ) TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc) diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index c42019e23d..7f8c118663 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( planner - PRIVATE os util catalog cjson parser transport function query + PRIVATE os util catalog cjson parser transport function qcom ) ADD_SUBDIRECTORY(test) diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index f00adfaeb2..5874356784 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(plannerTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( plannerTest - PUBLIC os util common planner parser catalog transport gtest function query + PUBLIC os util common planner parser catalog transport gtest function qcom ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 9df723c29e..31f1c25bea 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,5 +9,5 @@ target_include_directories( target_link_libraries( scheduler - PRIVATE os util planner qcom + PRIVATE os util planner qcom common ) \ No newline at end of file From 8940eb4d1af0d7b5d6d58644e2c9a9843ff89bf8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Dec 2021 17:16:39 +0800 Subject: [PATCH 42/50] format sim scripts --- tests/tsim/inc/sim.h | 54 +++++++++++++++++++------------------- tests/tsim/inc/simParse.h | 8 +++--- tests/tsim/src/simExe.c | 54 ++++++++++++++++++-------------------- tests/tsim/src/simMain.c | 6 ++--- tests/tsim/src/simParse.c | 29 ++++++++++---------- tests/tsim/src/simSystem.c | 4 +-- 6 files changed, 77 insertions(+), 78 deletions(-) diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 4cb08d6401..4aa3c62a80 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -102,18 +102,18 @@ typedef struct _cmd_t { int16_t cmdno; int16_t nlen; char name[MAX_SIM_CMD_NAME_LEN]; - bool (*parseCmd)(char *, struct _cmd_t *, int32_t); - bool (*executeCmd)(struct _script_t *script, char *option); + bool (*parseCmd)(char *, struct _cmd_t *, int32_t); + bool (*executeCmd)(struct _script_t *script, char *option); struct _cmd_t *next; } SCommand; typedef struct { int16_t cmdno; - int16_t jump; // jump position - int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag - // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ - int16_t lineNum; // correspodning line number in original file - int32_t optionOffset;// relative option offset + int16_t jump; // jump position + int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag + // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ + int16_t lineNum; // correspodning line number in original file + int32_t optionOffset; // relative option offset } SCmdLine; typedef struct _var_t { @@ -123,24 +123,24 @@ typedef struct _var_t { } SVariable; typedef struct _script_t { - int32_t type; - bool killed; - void * taos; - char rows[12]; // number of rows data retrieved - char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results - char system_exit_code[12]; - char system_ret_content[MAX_SYSTEM_RESULT_LEN]; - int32_t varLen; - int32_t linePos; // current cmd position - int32_t numOfLines; // number of lines in the script - int32_t bgScriptLen; - char fileName[MAX_FILE_NAME_LEN]; // script file name - char error[MAX_ERROR_LEN]; - char * optionBuffer; - SCmdLine *lines; // command list - SVariable variables[MAX_VAR_LEN]; - pthread_t bgPid; - char auth[128]; + int32_t type; + bool killed; + void *taos; + char rows[12]; // number of rows data retrieved + char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results + char system_exit_code[12]; + char system_ret_content[MAX_SYSTEM_RESULT_LEN]; + int32_t varLen; + int32_t linePos; // current cmd position + int32_t numOfLines; // number of lines in the script + int32_t bgScriptLen; + char fileName[MAX_FILE_NAME_LEN]; // script file name + char error[MAX_ERROR_LEN]; + char *optionBuffer; + SCmdLine *lines; // command list + SVariable variables[MAX_VAR_LEN]; + pthread_t bgPid; + char auth[128]; struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM]; } SScript; @@ -154,11 +154,11 @@ extern bool abortExecution; SScript *simParseScript(char *fileName); SScript *simProcessCallOver(SScript *script); -void * simExecuteScript(void *script); +void *simExecuteScript(void *script); void simInitsimCmdList(); bool simSystemInit(); void simSystemCleanUp(); -char * simGetVariable(SScript *script, char *varName, int32_t varLen); +char *simGetVariable(SScript *script, char *varName, int32_t varLen); bool simExecuteExpCmd(SScript *script, char *option); bool simExecuteTestCmd(SScript *script, char *option); bool simExecuteGotoCmd(SScript *script, char *option); diff --git a/tests/tsim/inc/simParse.h b/tests/tsim/inc/simParse.h index addd74bf80..56ee90aadb 100644 --- a/tests/tsim/inc/simParse.h +++ b/tests/tsim/inc/simParse.h @@ -40,10 +40,10 @@ typedef struct { /* block definition */ typedef struct { - char top; /* the number of blocks stacked */ - char type[MAX_NUM_BLOCK]; /* the block type */ - int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ - int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ + char top; /* the number of blocks stacked */ + char type[MAX_NUM_BLOCK]; /* the block type */ + int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ + int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ char numJump[MAX_NUM_BLOCK]; int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 4f64564404..453d653919 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -14,18 +14,18 @@ */ #define _DEFAULT_SOURCE -#include "taos.h" #include "cJSON.h" #include "os.h" #include "sim.h" +#include "taos.h" #include "taoserror.h" #include "tglobal.h" -#include "tutil.h" #include "ttypes.h" +#include "tutil.h" void simLogSql(char *sql, bool useSharp) { static FILE *fp = NULL; - char filename[256]; + char filename[256]; sprintf(filename, "%s/sim.sql", tsScriptDir); if (fp == NULL) { fp = fopen(filename, "w"); @@ -74,7 +74,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { return "null"; } - char * keyName; + char *keyName; int32_t keyLen; paGetToken(varName + 6, &keyName, &keyLen); @@ -91,7 +91,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { return "null"; } - char * keyName; + char *keyName; int32_t keyLen; paGetToken(varName + 7, &keyName, &keyLen); @@ -144,7 +144,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { } int32_t simExecuteExpression(SScript *script, char *exp) { - char * op1, *op2, *var1, *var2, *var3, *rest; + char *op1, *op2, *var1, *var2, *var3, *rest; int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; char t0[1024], t1[1024], t2[1024], t3[2048]; int32_t result; @@ -302,10 +302,10 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { } void simReplaceShToBat(char *dst) { - char* sh = strstr(dst, ".sh"); + char *sh = strstr(dst, ".sh"); if (sh != NULL) { int32_t dstLen = (int32_t)strlen(dst); - char *end = dst + dstLen; + char *end = dst + dstLen; *(end + 1) = 0; for (char *p = end; p >= sh; p--) { @@ -436,7 +436,7 @@ bool simExecuteReturnCmd(SScript *script, char *option) { } void simVisuallizeOption(SScript *script, char *src, char *dst) { - char * var, *token, *value; + char *var, *token, *value; int32_t dstLen, srcLen, tokenLen; dst[0] = 0, dstLen = 0; @@ -475,9 +475,7 @@ void simCloseNativeConnect(SScript *script) { script->taos = NULL; } -void simCloseTaosdConnect(SScript *script) { - simCloseNativeConnect(script); -} +void simCloseTaosdConnect(SScript *script) { simCloseNativeConnect(script); } bool simCreateNativeConnect(SScript *script, char *user, char *pass) { simCloseTaosdConnect(script); @@ -512,8 +510,8 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) { } bool simCreateTaosdConnect(SScript *script, char *rest) { - char * user = TSDB_DEFAULT_USER; - char * token; + char *user = TSDB_DEFAULT_USER; + char *token; int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); rest = paGetToken(rest, &token, &tokenLen); @@ -528,7 +526,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { char timeStr[30] = {0}; time_t tt; struct tm *tp; - SCmdLine * line = &script->lines[script->linePos]; + SCmdLine *line = &script->lines[script->linePos]; int32_t ret = -1; TAOS_RES *pSql = NULL; @@ -589,7 +587,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { while ((row = taos_fetch_row(pSql))) { if (numOfRows < MAX_QUERY_ROW_NUM) { TAOS_FIELD *fields = taos_fetch_fields(pSql); - int32_t * length = taos_fetch_lengths(pSql); + int32_t *length = taos_fetch_lengths(pSql); for (int32_t i = 0; i < num_fields; i++) { char *value = NULL; @@ -613,7 +611,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { sprintf(value, "%d", *((int8_t *)row[i])); break; case TSDB_DATA_TYPE_UTINYINT: - sprintf(value, "%u", *((uint8_t*)row[i])); + sprintf(value, "%u", *((uint8_t *)row[i])); break; case TSDB_DATA_TYPE_SMALLINT: sprintf(value, "%d", *((int16_t *)row[i])); @@ -679,7 +677,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } else if (precision == TSDB_TIME_PRECISION_MICRO) { sprintf(value, "%s.%06d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000)); } else { - sprintf(value, "%s.%09d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000000)); + sprintf(value, "%s.%09d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000000000)); } break; @@ -711,7 +709,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { - char buf[3000]; + char buf[3000]; SCmdLine *line = &script->lines[script->linePos]; simVisuallizeOption(script, rest, buf); @@ -803,9 +801,8 @@ bool simExecuteRestfulCmd(SScript *script, char *rest) { return simExecuteSystemCmd(script, cmd); } - bool simExecuteSqlErrorCmd(SScript *script, char *rest) { - char buf[3000]; + char buf[3000]; SCmdLine *line = &script->lines[script->linePos]; simVisuallizeOption(script, rest, buf); @@ -870,7 +867,7 @@ bool simExecuteLineInsertCmd(SScript *script, char *rest) { simInfo("script:%s, %s", script->fileName, rest); simLogSql(buf, true); - char * lines[] = {rest}; + char *lines[] = {rest}; #if 0 int32_t ret = taos_insert_lines(script->taos, lines, 1); #else @@ -881,8 +878,8 @@ bool simExecuteLineInsertCmd(SScript *script, char *rest) { script->linePos++; return true; } else { - sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest, - ret & 0XFFFF, tstrerror(ret)); + sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); return false; } } @@ -897,19 +894,20 @@ bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { simInfo("script:%s, %s", script->fileName, rest); simLogSql(buf, true); - char * lines[] = {rest}; + char *lines[] = {rest}; #if 0 int32_t ret = taos_insert_lines(script->taos, lines, 1); #else int32_t ret = 0; #endif if (ret == TSDB_CODE_SUCCESS) { - sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, script->taos, rest); + sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, + script->taos, rest); script->linePos++; return false; } else { - simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest, - ret & 0XFFFF, tstrerror(ret)); + simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); return true; } } diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 438bd41910..19d23daace 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -15,8 +15,8 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tglobal.h" #include "sim.h" +#include "tglobal.h" bool simExecSuccess = false; bool abortExecution = false; @@ -24,8 +24,8 @@ bool abortExecution = false; void simHandleSignal(int32_t signo, void *sigInfo, void *context) { simSystemCleanUp(); abortExecution = true; -// runningScript->killed = true; -// exit(1); + // runningScript->killed = true; + // exit(1); } int32_t main(int32_t argc, char *argv[]) { diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 246d0d3829..0dfd5f4f3e 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -60,9 +60,10 @@ #define _DEFAULT_SOURCE #include "os.h" #include "sim.h" -#include "simParse.h" #include "tutil.h" +#include "simParse.h" + static SCommand *cmdHashList[MAX_NUM_CMD]; static SCmdLine cmdLine[MAX_CMD_LINES]; static char parseErr[MAX_ERROR_LEN]; @@ -176,11 +177,11 @@ SScript *simBuildScriptObj(char *fileName) { } SScript *simParseScript(char *fileName) { - FILE * fd; + FILE *fd; int32_t tokenLen, lineNum = 0; char buffer[MAX_LINE_LEN], name[128], *token, *rest; SCommand *pCmd; - SScript * script; + SScript *script; if ((fileName[0] == '.') || (fileName[0] == '/')) { strcpy(name, fileName); @@ -251,7 +252,7 @@ SScript *simParseScript(char *fileName) { } int32_t simCheckExpression(char *exp) { - char * op1, *op2, *op, *rest; + char *op1, *op2, *op, *rest; int32_t op1Len, op2Len, opLen; rest = paGetToken(exp, &op1, &op1Len); @@ -335,7 +336,7 @@ bool simParseExpression(char *token, int32_t lineNum) { } bool simParseIfCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * ret; + char *ret; int32_t expLen; expLen = simCheckExpression(rest); @@ -501,7 +502,7 @@ bool simParseEndwCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -524,7 +525,7 @@ bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseCaseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -665,7 +666,7 @@ bool simParsePrintCmd(char *rest, SCommand *pCmd, int32_t lineNum) { void simCheckSqlOption(char *rest) { int32_t valueLen; - char * value, *xpos; + char *value, *xpos; xpos = strstr(rest, " -x"); // need a blank if (xpos) { @@ -749,7 +750,7 @@ bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; cmdLine[numOfLines].cmdno = SIM_CMD_SLEEP; @@ -768,7 +769,7 @@ bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; cmdLine[numOfLines].cmdno = SIM_CMD_RETURN; @@ -787,7 +788,7 @@ bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -810,7 +811,7 @@ bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) { } bool simParseRunCmd(char *rest, SCommand *pCmd, int32_t lineNum) { - char * token; + char *token; int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -837,7 +838,7 @@ bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) { return true; } -bool simParseLineInsertCmd(char* rest, SCommand* pCmd, int32_t lineNum) { +bool simParseLineInsertCmd(char *rest, SCommand *pCmd, int32_t lineNum) { int32_t expLen; rest++; @@ -853,7 +854,7 @@ bool simParseLineInsertCmd(char* rest, SCommand* pCmd, int32_t lineNum) { return true; } -bool simParseLineInsertErrorCmd(char* rest, SCommand* pCmd, int32_t lineNum) { +bool simParseLineInsertErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { int32_t expLen; rest++; diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 283e84c7a8..db32354616 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -14,9 +14,9 @@ */ #define _DEFAULT_SOURCE -#include "taos.h" #include "os.h" #include "sim.h" +#include "taos.h" #include "taoserror.h" #include "tglobal.h" #include "ttimer.h" @@ -166,7 +166,7 @@ void *simExecuteScript(void *inputScript) { } } else { SCmdLine *line = &script->lines[script->linePos]; - char * option = script->optionBuffer + line->optionOffset; + char *option = script->optionBuffer + line->optionOffset; simDebug("script:%s, line:%d with option \"%s\"", script->fileName, line->lineNum, option); SCommand *cmd = &simCmdList[line->cmdno]; From 1464972bf142a5833d0a810e4031ab6e38c5a707 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Dec 2021 17:26:15 +0800 Subject: [PATCH 43/50] [td-11818] fix compiler error. --- include/libs/qcom/query.h | 2 ++ source/client/src/clientImpl.c | 2 +- source/common/inc/commonInt.h | 31 ------------------------------- 3 files changed, 3 insertions(+), 32 deletions(-) delete mode 100644 source/common/inc/commonInt.h diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index fbfa19e5a5..254d572149 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -81,6 +81,8 @@ typedef struct STableMetaOutput { STableMeta *tbMeta; } STableMetaOutput; +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); + extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d0c6d99c27..b6200de824 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -430,7 +430,7 @@ void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t offset = 0; for (int32_t i = 0; i < numOfCols; ++i) { pResultInfo->length[i] = pResultInfo->fields[i].bytes; - pResultInfo->row[i] = pResultInfo->pData + offset * pResultInfo->numOfRows; + pResultInfo->row[i] = (char*) (pResultInfo->pData + offset * pResultInfo->numOfRows); pResultInfo->pCol[i] = pResultInfo->row[i]; offset += pResultInfo->fields[i].bytes; } diff --git a/source/common/inc/commonInt.h b/source/common/inc/commonInt.h deleted file mode 100644 index 5b71f83faf..0000000000 --- a/source/common/inc/commonInt.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_COMMON_INT_H_ -#define _TD_COMMON_INT_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -extern bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); - - - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_COMMON_INT_H_*/ From ad5c9695346ab45503cec42272ba56c9139a9d56 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 Dec 2021 17:26:45 +0800 Subject: [PATCH 44/50] [td-10564] refactor and add test cases. --- source/libs/qcom/src/queryUtil.c | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 source/libs/qcom/src/queryUtil.c diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c new file mode 100644 index 0000000000..2a13b708ec --- /dev/null +++ b/source/libs/qcom/src/queryUtil.c @@ -0,0 +1,78 @@ +#include "os.h" +#include "taosmsg.h" + +#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) +#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) + +static struct SSchema _s = { + .colId = TSDB_TBNAME_COLUMN_INDEX, + .type = TSDB_DATA_TYPE_BINARY, + .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, + .name = "tbname", +}; + +SSchema* tGetTbnameColumnSchema() { + return &_s; +} + +static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) { + int32_t rowLen = 0; + + for (int32_t i = 0; i < numOfCols; ++i) { + // 1. valid types + if (!isValidDataType(pSchema[i].type)) { + return false; + } + + // 2. valid length for each type + if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) { + if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) { + return false; + } + } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) { + return false; + } + } else { + if (pSchema[i].bytes != tDataTypes[pSchema[i].type].bytes) { + return false; + } + } + + // 3. valid column names + for (int32_t j = i + 1; j < numOfCols; ++j) { + if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) { + return false; + } + } + + rowLen += pSchema[i].bytes; + } + + return rowLen <= maxLen; +} + +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) { + if (!VALIDNUMOFCOLS(numOfCols)) { + return false; + } + + if (!VALIDNUMOFTAGS(numOfTags)) { + return false; + } + + /* first column must be the timestamp, which is a primary key */ + if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) { + return false; + } + + if (!doValidateSchema(pSchema, numOfCols, TSDB_MAX_BYTES_PER_ROW)) { + return false; + } + + if (!doValidateSchema(&pSchema[numOfCols], numOfTags, TSDB_MAX_TAGS_LEN)) { + return false; + } + + return true; +} \ No newline at end of file From 147f5af9b28ed364f9773343a8cc0bfe92a20ccd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 17 Dec 2021 18:23:13 +0800 Subject: [PATCH 45/50] adjust sim script --- CMakeLists.txt | 2 +- tests/script/sh/deploy.sh | 9 ++++++--- tests/script/sh/exec.sh | 11 +++++++---- tests/script/test.sh | 25 +++++-------------------- tests/tsim/src/simSystem.c | 6 ++++++ 5 files changed, 25 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b8a3f9a1c..cda71fb3bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,4 +31,4 @@ add_subdirectory(tests) # docs add_subdirectory(docs) -# tests (TODO) \ No newline at end of file +# tests (TODO) diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index cde27d7dc3..03ce1c2888 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -1,5 +1,8 @@ #!/bin/bash +set +e +#set -x + echo "Executing deploy.sh" if [ $# != 4 ]; then @@ -50,12 +53,12 @@ else fi if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3` else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2` fi -BUILD_DIR=$TAOS_DIR/$BIN_DIR/build +BUILD_DIR=$TAOS_DIR/$BIN_DIR SIM_DIR=$TAOS_DIR/sim diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 80b8cda428..a40912d5f4 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -8,6 +8,9 @@ # exit 1 # fi +set +e +set -x + UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` @@ -62,16 +65,16 @@ else fi if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3` else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2` fi -BUILD_DIR=$TAOS_DIR/$BIN_DIR/build +BUILD_DIR=$TAOS_DIR/$BIN_DIR SIM_DIR=$TAOS_DIR/sim NODE_DIR=$SIM_DIR/$NODE_NAME -EXE_DIR=$BUILD_DIR/bin +EXE_DIR=$BUILD_DIR/source/dnode/mgmt/daemon CFG_DIR=$NODE_DIR/cfg LOG_DIR=$NODE_DIR/log DATA_DIR=$NODE_DIR/data diff --git a/tests/script/test.sh b/tests/script/test.sh index f2dc578987..a744a56aa6 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -22,9 +22,6 @@ do f) FILE_NAME=$OPTARG ;; - a) - ASYNC=1 - ;; v) VALGRIND=1 ;; @@ -60,32 +57,22 @@ else fi if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2,3` else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` + BIN_DIR=`find . -name "taosd"|grep source|head -n1|cut -d '/' ${cut_opt}2` fi -BUILD_DIR=$TOP_DIR/$BIN_DIR/build +BUILD_DIR=$TOP_DIR/$BIN_DIR SIM_DIR=$TOP_DIR/sim -if [ $ASYNC -eq 0 ]; then - PROGRAM=$BUILD_DIR/bin/tsim -else - PROGRAM="$BUILD_DIR/bin/tsim -a" -fi - +PROGRAM=$BUILD_DIR/tests/tsim/tsim PRG_DIR=$SIM_DIR/tsim CFG_DIR=$PRG_DIR/cfg LOG_DIR=$PRG_DIR/log DATA_DIR=$PRG_DIR/data - -ARBITRATOR_PRG_DIR=$SIM_DIR/arbitrator -ARBITRATOR_LOG_DIR=$ARBITRATOR_PRG_DIR/log - - chmod -R 777 $PRG_DIR echo "------------------------------------------------------------------------" echo "Start TDengine Testing Case ..." @@ -96,12 +83,10 @@ echo "CFG_DIR : $CFG_DIR" rm -rf $LOG_DIR rm -rf $CFG_DIR -rm -rf $ARBITRATOR_LOG_DIR mkdir -p $PRG_DIR mkdir -p $LOG_DIR mkdir -p $CFG_DIR -mkdir -p $ARBITRATOR_LOG_DIR TAOS_CFG=$PRG_DIR/cfg/taos.cfg touch -f $TAOS_CFG @@ -115,7 +100,7 @@ echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG echo "serverPort 7100" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG -echo "scriptDir ${CODE_DIR}/../script" >> $TAOS_CFG +echo "scriptDir ${CODE_DIR}" >> $TAOS_CFG echo "numOfLogLines 100000000" >> $TAOS_CFG echo "rpcDebugFlag 143" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index db32354616..cb61e6b814 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -21,6 +21,8 @@ #include "tglobal.h" #include "ttimer.h" #include "tutil.h" +#include "tglobal.h" +#include "tconfig.h" SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; SCommand simCmdList[SIM_CMD_END]; @@ -80,6 +82,10 @@ char *simParseHostName(char *varName) { bool simSystemInit() { taosGetFqdn(simHostName); + + taosInitGlobalCfg(); + taosReadCfgFromFile(); + simInitsimCmdList(); memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); return true; From 48c1390c659d69eb9e1f71a8056c3b956f1816c9 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 17 Dec 2021 06:59:55 -0500 Subject: [PATCH 46/50] TD-12193 physical plan to json --- include/libs/planner/planner.h | 7 +- source/libs/planner/inc/plannerInt.h | 2 + source/libs/planner/src/physicalPlan.c | 11 +- source/libs/planner/src/physicalPlanJson.c | 594 +++++++++++++-------- source/libs/planner/src/planner.c | 4 + 5 files changed, 407 insertions(+), 211 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 28d9a04513..f86ce77803 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -128,6 +128,7 @@ typedef struct SSubplan { } SSubplan; typedef struct SQueryDag { + uint64_t queryId; SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. } SQueryDag; @@ -136,7 +137,7 @@ typedef struct SQueryDag { */ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); -int32_t qSetSuplanExecutionNode(SSubplan* subplan, SArray* nodes); +int32_t qSetSubplanExecutionNode(SSubplan* subplan, SArray* eps); int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); @@ -147,12 +148,14 @@ int32_t qSubPlanToString(const SSubplan* subplan, char** str); int32_t qStringToSubplan(const char* str, SSubplan** subplan); +void qDestroySubplan(SSubplan* pSubplan); + /** * Destroy the physical plan. * @param pQueryPhyNode * @return */ -void qDestroyQueryDag(struct SQueryDag* pDag); +void qDestroyQueryDag(SQueryDag* pDag); #ifdef __cplusplus } diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index c5f948b722..68a9518680 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -116,6 +116,8 @@ void destroyQueryPlan(struct SQueryPlanNode* pQueryNode); */ void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode); +int32_t opNameToOpType(const char* name); + #ifdef __cplusplus } #endif diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 67e5770b75..5a891d4b28 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -179,7 +179,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { assert(false); } if (pPlanNode->pChildren != NULL && taosArrayGetSize(pPlanNode->pChildren) > 0) { - node->pChildren = taosArrayInit(4, POINTER_BYTES); + node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); size_t size = taosArrayGetSize(pPlanNode->pChildren); for(int32_t i = 0; i < size; ++i) { SPhyNode* child = createPhyNode(pCxt, taosArrayGet(pPlanNode->pChildren, i)); @@ -215,3 +215,12 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD *pDag = context.pDag; return TSDB_CODE_SUCCESS; } + +int32_t opNameToOpType(const char* name) { + for (int32_t i = 1; i < sizeof(gOpName) / sizeof(gOpName[0]); ++i) { + if (strcmp(name, gOpName[i])) { + return i; + } + } + return OP_Unknown; +} diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 943c6b8dc0..7c0dc25627 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -17,29 +17,52 @@ #include "parser.h" #include "cJSON.h" -typedef cJSON* (*FToObj)(const void* obj); +typedef bool (*FToJson)(const void* obj, cJSON* json); +typedef bool (*FFromJson)(const cJSON* json, void* obj); -static bool addObject(cJSON* json, const char* name, FToObj func, const void* obj) { +static bool addObject(cJSON* json, const char* name, FToJson func, const void* obj) { if (NULL == obj) { return true; } - cJSON* jObj = func(obj); - if (NULL == jObj) { + cJSON* jObj = cJSON_CreateObject(); + if (NULL == jObj || !func(obj, jObj)) { + cJSON_Delete(jObj); return false; } return cJSON_AddItemToObject(json, name, jObj); } -static bool addItem(cJSON* json, FToObj func, const void* item) { - cJSON* jItem = func(item); - if (NULL == jItem) { +static bool addItem(cJSON* json, FToJson func, const void* obj) { + cJSON* jObj = cJSON_CreateObject(); + if (NULL == jObj || !func(obj, jObj)) { + cJSON_Delete(jObj); return false; } - return cJSON_AddItemToArray(json, jItem); + return cJSON_AddItemToArray(json, jObj); } -static bool addArray(cJSON* json, const char* name, FToObj func, const SArray* array) { +static bool fromObject(const cJSON* json, const char* name, FFromJson func, void* obj, bool required) { + cJSON* jObj = cJSON_GetObjectItem(json, name); + if (NULL == jObj) { + return !required; + } + return func(jObj, obj); +} + +static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson func, void** obj, int32_t size, bool required) { + cJSON* jObj = cJSON_GetObjectItem(json, name); + if (NULL == jObj) { + return !required; + } + *obj = calloc(1, size); + if (NULL == *obj) { + return false; + } + return func(jObj, *obj); +} + +static bool addArray(cJSON* json, const char* name, FToJson func, const SArray* array) { size_t size = (NULL == array) ? 0 : taosArrayGetSize(array); if (size > 0) { cJSON* jArray = cJSON_AddArrayToObject(json, name); @@ -55,7 +78,26 @@ static bool addArray(cJSON* json, const char* name, FToObj func, const SArray* a return true; } -static bool addRawArray(cJSON* json, const char* name, FToObj func, const void* array, int32_t itemSize, int32_t size) { +static bool fromArray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize) { + const cJSON* jArray = cJSON_GetObjectItem(json, name); + int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray)); + if (size > 0) { + *array = taosArrayInit(size, POINTER_BYTES); + if (NULL == *array) { + return false; + } + } + for (int32_t i = 0; i < size; ++i) { + void* item = calloc(1, itemSize); + if (NULL == item || !func(cJSON_GetArrayItem(jArray, i), item)) { + return false; + } + taosArrayPush(*array, &item); + } + return true; +} + +static bool addRawArray(cJSON* json, const char* name, FToJson func, const void* array, int32_t itemSize, int32_t size) { if (size > 0) { cJSON* jArray = cJSON_AddArrayToObject(json, name); if (NULL == jArray) { @@ -70,310 +112,425 @@ static bool addRawArray(cJSON* json, const char* name, FToObj func, const void* return true; } -static cJSON* schemaToJson(const void* obj) { +static const cJSON* getArray(const cJSON* json, const char* name, int32_t* size) { + const cJSON* jArray = cJSON_GetObjectItem(json, name); + *size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray)); + return jArray; +} + +static bool fromItem(const cJSON* jArray, FFromJson func, void* array, int32_t itemSize, int32_t size) { + for (int32_t i = 0; i < size; ++i) { + if (!func(cJSON_GetArrayItem(jArray, i), (char*)array + itemSize)) { + return false; + } + } + return true; +} + +static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) { + const cJSON* jArray = getArray(json, name, size); + if (*size > 0) { + *array = calloc(1, itemSize * (*size)); + if (NULL == *array) { + return false; + } + } + return fromItem(jArray, func, *array, itemSize, *size); +} + +static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) { + const cJSON* jArray = getArray(json, name, size); + return fromItem(jArray, func, array, itemSize, *size); +} + +static char* getString(const cJSON* json, const char* name) { + char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); + char* res = calloc(1, strlen(p) + 1); + strcpy(res, p); + return res; +} + +static void copyString(const cJSON* json, const char* name, char* dst) { + strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name))); +} + +static int64_t getNumber(const cJSON* json, const char* name) { + return cJSON_GetNumberValue(cJSON_GetObjectItem(json, name)); +} + +static const char* jkSchemaType = "Type"; +static const char* jkSchemaColId = "ColId"; +static const char* jkSchemaBytes = "Bytes"; +// The 'name' field do not need to be serialized. +static bool schemaToJson(const void* obj, cJSON* jSchema) { const SSlotSchema* schema = (const SSlotSchema*)obj; - cJSON* jSchema = cJSON_CreateObject(); - if (NULL == jSchema) { - return NULL; - } - - // The 'name' field do not need to be serialized. - - bool res = cJSON_AddNumberToObject(jSchema, "Type", schema->type); + bool res = cJSON_AddNumberToObject(jSchema, jkSchemaType, schema->type); if (res) { - res = cJSON_AddNumberToObject(jSchema, "ColId", schema->colId); + res = cJSON_AddNumberToObject(jSchema, jkSchemaColId, schema->colId); } if (res) { - res = cJSON_AddNumberToObject(jSchema, "Bytes", schema->bytes); + res = cJSON_AddNumberToObject(jSchema, jkSchemaBytes, schema->bytes); } - - if (!res) { - cJSON_Delete(jSchema); - return NULL; - } - return jSchema; + return res; } -static cJSON* columnFilterInfoToJson(const void* obj) { +static bool schemaFromJson(const cJSON* json, void* obj) { + SSlotSchema* schema = (SSlotSchema*)obj; + schema->type = getNumber(json, jkSchemaType); + schema->colId = getNumber(json, jkSchemaColId); + schema->bytes = getNumber(json, jkSchemaBytes); + return true; +} + +static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr"; +static const char* jkColumnFilterInfoUpperRelOptr = "UpperRelOptr"; +static const char* jkColumnFilterInfoFilterstr = "Filterstr"; +static const char* jkColumnFilterInfoLowerBnd = "LowerBnd"; +static const char* jkColumnFilterInfoUpperBnd = "UpperBnd"; + +static bool columnFilterInfoToJson(const void* obj, cJSON* jFilter) { const SColumnFilterInfo* filter = (const SColumnFilterInfo*)obj; - cJSON* jFilter = cJSON_CreateObject(); - if (NULL == jFilter) { - return NULL; - } - - bool res = cJSON_AddNumberToObject(jFilter, "LowerRelOptr", filter->lowerRelOptr); + bool res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerRelOptr, filter->lowerRelOptr); if (res) { - res = cJSON_AddNumberToObject(jFilter, "UpperRelOptr", filter->upperRelOptr); + res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoUpperRelOptr, filter->upperRelOptr); } if (res) { - res = cJSON_AddNumberToObject(jFilter, "Filterstr", filter->filterstr); + res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoFilterstr, filter->filterstr); } if (res) { - res = cJSON_AddNumberToObject(jFilter, "LowerBnd", filter->lowerBndd); + res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerBnd, filter->lowerBndd); } if (res) { - res = cJSON_AddNumberToObject(jFilter, "UpperBnd", filter->upperBndd); + res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoUpperBnd, filter->upperBndd); } - - if (!res) { - cJSON_Delete(jFilter); - return NULL; - } - return jFilter; + return res; } -static cJSON* columnInfoToJson(const void* obj) { +static bool columnFilterInfoFromJson(const cJSON* json, void* obj) { + SColumnFilterInfo* filter = (SColumnFilterInfo*)obj; + filter->lowerRelOptr = getNumber(json, jkColumnFilterInfoLowerRelOptr); + filter->upperRelOptr = getNumber(json, jkColumnFilterInfoUpperRelOptr); + filter->filterstr = getNumber(json, jkColumnFilterInfoFilterstr); + filter->lowerBndd = getNumber(json, jkColumnFilterInfoLowerBnd); + filter->upperBndd = getNumber(json, jkColumnFilterInfoUpperBnd); + return true; +} + +static const char* jkColumnInfoColId = "ColId"; +static const char* jkColumnInfoType = "Type"; +static const char* jkColumnInfoBytes = "Bytes"; +static const char* jkColumnInfoFilterList = "FilterList"; + +static bool columnInfoToJson(const void* obj, cJSON* jCol) { const SColumnInfo* col = (const SColumnInfo*)obj; - cJSON* jCol = cJSON_CreateObject(); - if (NULL == jCol) { - return NULL; - } - - bool res = cJSON_AddNumberToObject(jCol, "ColId", col->colId); + bool res = cJSON_AddNumberToObject(jCol, jkColumnInfoColId, col->colId); if (res) { - res = cJSON_AddNumberToObject(jCol, "Type", col->type); + res = cJSON_AddNumberToObject(jCol, jkColumnInfoType, col->type); } if (res) { - res = cJSON_AddNumberToObject(jCol, "Bytes", col->bytes); + res = cJSON_AddNumberToObject(jCol, jkColumnInfoBytes, col->bytes); } if (res) { - res = addRawArray(jCol, "FilterList", columnFilterInfoToJson, col->flist.filterInfo, sizeof(SColumnFilterInfo), col->flist.numOfFilters); + res = addRawArray(jCol, jkColumnInfoFilterList, columnFilterInfoToJson, col->flist.filterInfo, sizeof(SColumnFilterInfo), col->flist.numOfFilters); } - - if (!res) { - cJSON_Delete(jCol); - return NULL; - } - return jCol; + return res; } -static cJSON* columnToJson(const void* obj) { +static bool columnInfoFromJson(const cJSON* json, void* obj) { + SColumnInfo* col = (SColumnInfo*)obj; + col->colId = getNumber(json, jkColumnInfoColId); + col->type = getNumber(json, jkColumnInfoType); + col->bytes = getNumber(json, jkColumnInfoBytes); + int32_t size = 0; + bool res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson, (void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size); + col->flist.numOfFilters = size; + return res; +} + +static const char* jkColumnTableId = "TableId"; +static const char* jkColumnFlag = "Flag"; +static const char* jkColumnInfo = "Info"; + +static bool columnToJson(const void* obj, cJSON* jCol) { const SColumn* col = (const SColumn*)obj; - cJSON* jCol = cJSON_CreateObject(); - if (NULL == jCol) { - return NULL; - } - - bool res = cJSON_AddNumberToObject(jCol, "TableId", col->uid); + bool res = cJSON_AddNumberToObject(jCol, jkColumnTableId, col->uid); if (res) { - res = cJSON_AddNumberToObject(jCol, "Flag", col->flag); + res = cJSON_AddNumberToObject(jCol, jkColumnFlag, col->flag); } if (res) { - res = addObject(jCol, "Info", columnInfoToJson, &col->info); + res = addObject(jCol, jkColumnInfo, columnInfoToJson, &col->info); } - - if (!res) { - cJSON_Delete(jCol); - return NULL; - } - return jCol; + return res; } -static cJSON* exprNodeToJson(const void* obj); +static bool columnFromJson(const cJSON* json, void* obj) { + SColumn* col = (SColumn*)obj; + col->uid = getNumber(json, jkColumnTableId); + col->flag = getNumber(json, jkColumnFlag); + return fromObject(json, jkColumnInfo, columnInfoFromJson, &col->info, true); +} -static cJSON* operatorToJson(const void* obj) { +static bool exprNodeToJson(const void* obj, cJSON* jExprInfo); +static bool exprNodeFromJson(const cJSON* json, void* obj); + +static const char* jkExprNodeOper = "Oper"; +static const char* jkExprNodeLeft = "Left"; +static const char* jkExprNodeRight = "Right"; + +static bool operatorToJson(const void* obj, cJSON* jOper) { const tExprNode* exprInfo = (const tExprNode*)obj; - cJSON* jOper = cJSON_CreateObject(); - if (NULL == jOper) { - return NULL; - } - - bool res = cJSON_AddNumberToObject(jOper, "Oper", exprInfo->_node.optr); + bool res = cJSON_AddNumberToObject(jOper, jkExprNodeOper, exprInfo->_node.optr); if (res) { - res = addObject(jOper, "Left", exprNodeToJson, exprInfo->_node.pLeft); + res = addObject(jOper, jkExprNodeLeft, exprNodeToJson, exprInfo->_node.pLeft); } if (res) { - res = addObject(jOper, "Right", exprNodeToJson, exprInfo->_node.pRight); + res = addObject(jOper, jkExprNodeRight, exprNodeToJson, exprInfo->_node.pRight); } - - if (!res) { - cJSON_Delete(jOper); - return NULL; - } - return jOper; + return res; } -static cJSON* functionToJson(const void* obj) { +static bool operatorFromJson(const cJSON* json, void* obj) { + tExprNode* exprInfo = (tExprNode*)obj; + exprInfo->_node.optr = getNumber(json, jkExprNodeOper); + bool res = fromObject(json, jkExprNodeLeft, exprNodeFromJson, exprInfo->_node.pLeft, false); + if (res) { + res = fromObject(json, jkExprNodeRight, exprNodeFromJson, exprInfo->_node.pRight, false); + } + return res; +} + +static const char* jkFunctionName = "Name"; +static const char* jkFunctionChild = "Child"; + +static bool functionToJson(const void* obj, cJSON* jFunc) { const tExprNode* exprInfo = (const tExprNode*)obj; - cJSON* jFunc = cJSON_CreateObject(); - if (NULL == jFunc) { - return NULL; - } - - bool res = cJSON_AddStringToObject(jFunc, "Name", exprInfo->_function.functionName); + bool res = cJSON_AddStringToObject(jFunc, jkFunctionName, exprInfo->_function.functionName); if (res) { - res = addRawArray(jFunc, "Child", exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), exprInfo->_function.num); + res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), exprInfo->_function.num); } - - if (!res) { - cJSON_Delete(jFunc); - return NULL; - } - return jFunc; + return res; } -static cJSON* variantToJson(const void* obj) { +static bool functionFromJson(const cJSON* json, void* obj) { + tExprNode* exprInfo = (tExprNode*)obj; + copyString(json, jkFunctionName, exprInfo->_function.functionName); + return fromRawArrayWithAlloc(json, jkFunctionChild, exprNodeFromJson, (void**)exprInfo->_function.pChild, sizeof(tExprNode*), &exprInfo->_function.num); +} + +static const char* jkVariantType = "Type"; +static const char* jkVariantLen = "Len"; +static const char* jkVariantvalues = "values"; +static const char* jkVariantValue = "Value"; + +static bool variantToJson(const void* obj, cJSON* jVar) { const SVariant* var = (const SVariant*)obj; - cJSON* jVar = cJSON_CreateObject(); - if (NULL == jVar) { - return NULL; - } - - bool res = cJSON_AddNumberToObject(jVar, "Type", var->nType); + bool res = cJSON_AddNumberToObject(jVar, jkVariantType, var->nType); if (res) { - res = cJSON_AddNumberToObject(jVar, "Len", var->nLen); + res = cJSON_AddNumberToObject(jVar, jkVariantLen, var->nLen); } if (res) { if (0/* in */) { - res = addArray(jVar, "values", variantToJson, var->arr); + res = addArray(jVar, jkVariantvalues, variantToJson, var->arr); } else if (IS_NUMERIC_TYPE(var->nType)) { - res = cJSON_AddNumberToObject(jVar, "Value", var->d); + res = cJSON_AddNumberToObject(jVar, jkVariantValue, var->d); } else { - res = cJSON_AddStringToObject(jVar, "Value", var->pz); + res = cJSON_AddStringToObject(jVar, jkVariantValue, var->pz); } } - - if (!res) { - cJSON_Delete(jVar); - return NULL; - } - return jVar; + return res; } -static cJSON* exprNodeToJson(const void* obj) { - const tExprNode* exprInfo = (const tExprNode*)obj; - cJSON* jExprInfo = cJSON_CreateObject(); - if (NULL == jExprInfo) { - return NULL; +static bool variantFromJson(const cJSON* json, void* obj) { + SVariant* var = (SVariant*)obj; + var->nType = getNumber(json, jkVariantType); + var->nLen = getNumber(json, jkVariantLen); + if (0/* in */) { + return fromArray(json, jkVariantvalues, variantFromJson, &var->arr, sizeof(SVariant)); + } else if (IS_NUMERIC_TYPE(var->nType)) { + var->d = getNumber(json, jkVariantValue); + } else { + var->pz = getString(json, jkVariantValue); } + return true; +} - bool res = cJSON_AddNumberToObject(jExprInfo, "Type", exprInfo->nodeType); +static const char* jkExprNodeType = "Type"; +static const char* jkExprNodeOperator = "Operator"; +static const char* jkExprNodeFunction = "Function"; +static const char* jkExprNodeColumn = "Column"; +static const char* jkExprNodeValue = "Value"; + +static bool exprNodeToJson(const void* obj, cJSON* jExprInfo) { + const tExprNode* exprInfo = (const tExprNode*)obj; + bool res = cJSON_AddNumberToObject(jExprInfo, jkExprNodeType, exprInfo->nodeType); if (res) { switch (exprInfo->nodeType) { case TEXPR_BINARYEXPR_NODE: case TEXPR_UNARYEXPR_NODE: - res = addObject(jExprInfo, "Operator", operatorToJson, exprInfo); + res = addObject(jExprInfo, jkExprNodeOperator, operatorToJson, exprInfo); break; case TEXPR_FUNCTION_NODE: - res = addObject(jExprInfo, "Function", functionToJson, exprInfo); + res = addObject(jExprInfo, jkExprNodeFunction, functionToJson, exprInfo); break; case TEXPR_COL_NODE: - res = addObject(jExprInfo, "Column", schemaToJson, exprInfo->pSchema); + res = addObject(jExprInfo, jkExprNodeColumn, schemaToJson, exprInfo->pSchema); break; case TEXPR_VALUE_NODE: - res = addObject(jExprInfo, "Value", variantToJson, exprInfo->pVal); + res = addObject(jExprInfo, jkExprNodeValue, variantToJson, exprInfo->pVal); break; default: res = false; break; } } - - if (!res) { - cJSON_Delete(jExprInfo); - return NULL; - } - return jExprInfo; + return res; } -static cJSON* sqlExprToJson(const void* obj) { +static bool exprNodeFromJson(const cJSON* json, void* obj) { + tExprNode* exprInfo = (tExprNode*)obj; + exprInfo->nodeType = getNumber(json, jkExprNodeType); + switch (exprInfo->nodeType) { + case TEXPR_BINARYEXPR_NODE: + case TEXPR_UNARYEXPR_NODE: + return fromObject(json, jkExprNodeOperator, operatorFromJson, exprInfo, false); + case TEXPR_FUNCTION_NODE: + return fromObject(json, jkExprNodeFunction, functionFromJson, exprInfo, false); + case TEXPR_COL_NODE: + return fromObject(json, jkExprNodeColumn, schemaFromJson, exprInfo->pSchema, false); + case TEXPR_VALUE_NODE: + return fromObject(json, jkExprNodeValue, variantFromJson, exprInfo->pVal, false); + default: + break; + } + return false; +} + +static const char* jkSqlExprSchema = "Schema"; +static const char* jkSqlExprColumns = "Columns"; +static const char* jkSqlExprInterBytes = "InterBytes"; +static const char* jkSqlExprParams = "Params"; +// token does not need to be serialized. +static bool sqlExprToJson(const void* obj, cJSON* jExpr) { const SSqlExpr* expr = (const SSqlExpr*)obj; - cJSON* jExpr = cJSON_CreateObject(); - if (NULL == jExpr) { - return NULL; - } - - // token does not need to be serialized. - - bool res = addObject(jExpr, "Schema", schemaToJson, &expr->resSchema); + bool res = addObject(jExpr, jkSqlExprSchema, schemaToJson, &expr->resSchema); if (res) { - res = addRawArray(jExpr, "Columns", columnToJson, expr->pColumns, sizeof(SColumn), expr->numOfCols); + res = addRawArray(jExpr, jkSqlExprColumns, columnToJson, expr->pColumns, sizeof(SColumn), expr->numOfCols); } if (res) { - res = cJSON_AddNumberToObject(jExpr, "InterBytes", expr->interBytes); + res = cJSON_AddNumberToObject(jExpr, jkSqlExprInterBytes, expr->interBytes); } if (res) { - res = addRawArray(jExpr, "Params", variantToJson, expr->param, sizeof(SVariant), expr->numOfParams); + res = addRawArray(jExpr, jkSqlExprParams, variantToJson, expr->param, sizeof(SVariant), expr->numOfParams); } - - if (!res) { - cJSON_Delete(jExpr); - return NULL; - } - return jExpr; + return res; } -static cJSON* exprInfoToJson(const void* obj) { +static bool sqlExprFromJson(const cJSON* json, void* obj) { + SSqlExpr* expr = (SSqlExpr*)obj; + bool res = fromObject(json, jkSqlExprSchema, schemaFromJson, &expr->resSchema, false); + if (res) { + res = fromRawArrayWithAlloc(json, jkSqlExprColumns, columnFromJson, (void**)&expr->pColumns, sizeof(SColumn), &expr->numOfCols); + } + if (res) { + expr->interBytes = getNumber(json, jkSqlExprInterBytes); + } + if (res) { + int32_t size = 0; + res = fromRawArray(json, jkSqlExprParams, variantFromJson, expr->param, sizeof(SVariant), &size); + expr->numOfParams = size; + } + return res; +} + +static const char* jkExprInfoBase = "Base"; +static const char* jkExprInfoExpr = "Expr"; + +static bool exprInfoToJson(const void* obj, cJSON* jExprInfo) { const SExprInfo* exprInfo = (const SExprInfo*)obj; - cJSON* jExprInfo = cJSON_CreateObject(); - if (NULL == jExprInfo) { - return NULL; - } - - bool res = addObject(jExprInfo, "Base", sqlExprToJson, &exprInfo->base); + bool res = addObject(jExprInfo, jkExprInfoBase, sqlExprToJson, &exprInfo->base); if (res) { - res = addObject(jExprInfo, "Expr", exprNodeToJson, exprInfo->pExpr); + res = addObject(jExprInfo, jkExprInfoExpr, exprNodeToJson, exprInfo->pExpr); } - - if (!res) { - cJSON_Delete(jExprInfo); - return NULL; - } - return jExprInfo; + return res; } -static cJSON* phyNodeToJson(const void* obj) { +static bool exprInfoFromJson(const cJSON* json, void* obj) { + SExprInfo* exprInfo = (SExprInfo*)obj; + bool res = fromObject(json, jkExprInfoBase, sqlExprFromJson, &exprInfo->base, true); + if (res) { + res = fromObjectWithAlloc(json, jkExprInfoExpr, exprNodeFromJson, (void**)&exprInfo->pExpr, sizeof(tExprNode), true); + } + return res; +} + +static const char* jkPnodeName = "Name"; +static const char* jkPnodeTargets = "Targets"; +static const char* jkPnodeConditions = "Conditions"; +static const char* jkPnodeSchema = "Schema"; +static const char* jkPnodeChildren = "Children"; +// The 'pParent' field do not need to be serialized. +static bool phyNodeToJson(const void* obj, cJSON* jNode) { const SPhyNode* phyNode = (const SPhyNode*)obj; - cJSON* jNode = cJSON_CreateObject(); - if (NULL == jNode) { - return NULL; - } - - // The 'pParent' field do not need to be serialized. - - bool res = cJSON_AddStringToObject(jNode, "Name", phyNode->info.name); + bool res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name); if (res) { - res = addArray(jNode, "Targets", exprInfoToJson, phyNode->pTargets); + res = addArray(jNode, jkPnodeTargets, exprInfoToJson, phyNode->pTargets); } if (res) { - res = addArray(jNode, "Conditions", exprInfoToJson, phyNode->pConditions); + res = addArray(jNode, jkPnodeConditions, exprInfoToJson, phyNode->pConditions); } if (res) { - res = addRawArray(jNode, "Schema", schemaToJson, phyNode->targetSchema.pSchema, sizeof(SSlotSchema), phyNode->targetSchema.numOfCols); + res = addRawArray(jNode, jkPnodeSchema, schemaToJson, phyNode->targetSchema.pSchema, sizeof(SSlotSchema), phyNode->targetSchema.numOfCols); } if (res) { - res = addArray(jNode, "Children", phyNodeToJson, phyNode->pChildren); + res = addArray(jNode, jkPnodeChildren, phyNodeToJson, phyNode->pChildren); } - - if (!res) { - cJSON_Delete(jNode); - return NULL; - } - return jNode; + return res; } -static cJSON* subplanIdToJson(const void* obj) { +static bool phyNodeFromJson(const cJSON* json, void* obj) { + SPhyNode* node = (SPhyNode*)obj; + node->info.name = getString(json, jkPnodeName); + node->info.type = opNameToOpType(node->info.name); + bool res = fromArray(json, jkPnodeTargets, exprInfoFromJson, &node->pTargets, sizeof(SExprInfo)); + if (res) { + res = fromArray(json, jkPnodeConditions, exprInfoFromJson, &node->pConditions, sizeof(SExprInfo)); + } + if (res) { + res = fromRawArray(json, jkPnodeSchema, schemaFromJson, node->targetSchema.pSchema, sizeof(SSlotSchema), &node->targetSchema.numOfCols); + } + if (res) { + res = fromArray(json, jkPnodeChildren, phyNodeFromJson, &node->pChildren, sizeof(SSlotSchema)); + } + return res; +} + +static const char* jkIdQueryId = "QueryId"; +static const char* jkIdTemplateId = "TemplateId"; +static const char* jkIdSubplanId = "SubplanId"; + +static bool subplanIdToJson(const void* obj, cJSON* jId) { const SSubplanId* id = (const SSubplanId*)obj; - cJSON* jId = cJSON_CreateObject(); - if (NULL == jId) { - return NULL; - } - - bool res = cJSON_AddNumberToObject(jId, "QueryId", id->queryId); + bool res = cJSON_AddNumberToObject(jId, jkIdQueryId, id->queryId); if (res) { - res = cJSON_AddNumberToObject(jId, "TemplateId", id->templateId); + res = cJSON_AddNumberToObject(jId, jkIdTemplateId, id->templateId); } if (res) { - res = cJSON_AddNumberToObject(jId, "SubplanId", id->subplanId); + res = cJSON_AddNumberToObject(jId, jkIdSubplanId, id->subplanId); } - - if (!res) { - cJSON_Delete(jId); - return NULL; - } - return jId; + return res; } +static bool subplanIdFromJson(const cJSON* json, void* obj) { + SSubplanId* id = (SSubplanId*)obj; + id->queryId = getNumber(json, jkIdQueryId); + id->templateId = getNumber(json, jkIdTemplateId); + id->subplanId = getNumber(json, jkIdSubplanId); + return true; +} + +static const char* jkSubplanId = "Id"; +static const char* jkSubplanNode = "Node"; + static cJSON* subplanToJson(const SSubplan* subplan) { cJSON* jSubplan = cJSON_CreateObject(); if (NULL == jSubplan) { @@ -382,9 +539,9 @@ static cJSON* subplanToJson(const SSubplan* subplan) { // The 'type', 'level', 'execEpSet', 'pChildern' and 'pParents' fields do not need to be serialized. - bool res = addObject(jSubplan, "Id", subplanIdToJson, &subplan->id); + bool res = addObject(jSubplan, jkSubplanId, subplanIdToJson, &subplan->id); if (res) { - res = addObject(jSubplan, "Node", phyNodeToJson, subplan->pNode); + res = addObject(jSubplan, jkSubplanNode, phyNodeToJson, subplan->pNode); } if (!res) { @@ -394,6 +551,23 @@ static cJSON* subplanToJson(const SSubplan* subplan) { return jSubplan; } +static SSubplan* subplanFromJson(const cJSON* json) { + SSubplan* subplan = calloc(1, sizeof(SSubplan)); + if (NULL == subplan) { + return NULL; + } + bool res = fromObject(json, jkSubplanId, subplanIdFromJson, &subplan->id, true); + if (res) { + res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, sizeof(SPhyNode), false); + } + + if (!res) { + qDestroySubplan(subplan); + return NULL; + } + return subplan; +} + int32_t subPlanToString(const SSubplan* subplan, char** str) { cJSON* json = subplanToJson(subplan); if (NULL == json) { @@ -405,6 +579,10 @@ int32_t subPlanToString(const SSubplan* subplan, char** str) { } int32_t stringToSubplan(const char* str, SSubplan** subplan) { - // todo - return TSDB_CODE_SUCCESS; + cJSON* json = cJSON_Parse(str); + if (NULL == json) { + return TSDB_CODE_FAILED; + } + *subplan = subplanFromJson(json); + return (NULL == *subplan ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index ee989234d5..3d2bb895d5 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -16,6 +16,10 @@ #include "parser.h" #include "plannerInt.h" +void qDestroySubplan(SSubplan* pSubplan) { + // todo +} + void qDestroyQueryDag(struct SQueryDag* pDag) { // todo } From 653665641d64c52255802acc2f051333434e0529 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Dec 2021 21:35:05 +0800 Subject: [PATCH 47/50] update index cache --- source/libs/index/CMakeLists.txt | 1 + source/libs/index/inc/index_cache.h | 2 ++ source/libs/index/inc/index_fst.h | 1 - source/libs/index/src/index_cache.c | 15 +++++++++++++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 3da2c93b39..f68fc5e61e 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -9,6 +9,7 @@ target_link_libraries( index PUBLIC os PUBLIC util + PUBLIC common ) if (${BUILD_WITH_LUCENE}) diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index ff915a3930..39107a78ac 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -17,6 +17,7 @@ #include "index.h" #include "tlockfree.h" +#include "tskiplist.h" // ----------------- row structure in skiplist --------------------- /* A data row, the format is like below: @@ -30,6 +31,7 @@ extern "C" { typedef struct IndexCache { T_REF_DECLARE() + SSkipList *skiplist; } IndexCache; diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 20037f829a..0dcc25831c 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -315,7 +315,6 @@ typedef struct StreamWithStateResult { FstSlice data; FstOutput out; void *state; - } StreamWithStateResult; StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state); diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 2ecc823ef9..acb8e32157 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -14,7 +14,13 @@ */ #include "index_cache.h" +#include "tcompare.h" +#define MAX_INDEX_KEY_LEN 128 // test only, change later + +static char* getIndexKey(const void *pData) { + return NULL; +} static int32_t compareKey(const void *l, const void *r) { char *lp = (char *)l; char *rp = (char *)r; @@ -85,10 +91,14 @@ static int32_t compareKey(const void *l, const void *r) { } IndexCache *indexCacheCreate() { IndexCache *cache = calloc(1, sizeof(IndexCache)); + cache->skiplist = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); return cache; + } void indexCacheDestroy(IndexCache *cache) { + if (cache == NULL) { return; } + tSkipListDestroy(cache->skiplist); free(cache); } @@ -96,6 +106,7 @@ int indexCachePut(IndexCache *cache, int16_t fieldId, int16_t fieldType, const c uint32_t version, uint64_t uid, int8_t operType) { if (cache == NULL) { return -1;} + // encode data int32_t total = sizeof(int32_t) + sizeof(fieldId) + sizeof(fieldType) + sizeof(fvLen) + fvLen + sizeof(version) + sizeof(uid) + sizeof(operType); char *buf = calloc(1, total); @@ -123,6 +134,10 @@ int indexCachePut(IndexCache *cache, int16_t fieldId, int16_t fieldType, const c memcpy(p, &operType, sizeof(operType)); p += sizeof(operType); + + tSkipListPut(cache->skiplist, (void *)buf); + // encode end + } int indexCacheDel(IndexCache *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { From 8a82da8f6d056ff027edc23a2aaad55ef9ce14a1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 18 Dec 2021 00:26:08 +0800 Subject: [PATCH 48/50] adjust file general/insert/basic1.sim --- source/dnode/mgmt/daemon/src/daemon.c | 1 + tests/script/general/user/basic1.sim | 10 +++++----- tests/script/sh/exec.sh | 2 +- tests/script/test.sh | 1 - 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index 0a725d9975..b9a1e4cecb 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -92,6 +92,7 @@ void dmnPrintVersion() { } int dmnReadConfig(const char *path) { + tstrncpy(configDir, global.configDir, PATH_MAX); taosInitGlobalCfg(); taosReadGlobalLogCfg(); diff --git a/tests/script/general/user/basic1.sim b/tests/script/general/user/basic1.sim index 3670c1ddb0..d4a663c096 100644 --- a/tests/script/general/user/basic1.sim +++ b/tests/script/general/user/basic1.sim @@ -5,7 +5,7 @@ sql connect print =============== show users sql show users -if $rows != 3 then +if $rows != 1 then return -1 endi @@ -21,7 +21,7 @@ sql_error drop account root print =============== create user1 sql create user user1 PASS 'user1' sql show users -if $rows != 4 then +if $rows != 2 then return -1 endi @@ -33,7 +33,7 @@ print $data30 $data31 $data32 print =============== create user2 sql create user user2 PASS 'user2' sql show users -if $rows != 5 then +if $rows != 3 then return -1 endi @@ -46,7 +46,7 @@ print $data40 $data41 $data42 print =============== drop user1 sql drop user user1 sql show users -if $rows != 4 then +if $rows != 2 then return -1 endi @@ -62,7 +62,7 @@ system sh/exec.sh -n dnode1 -s start print =============== show users sql show users -if $rows != 4 then +if $rows != 2 then return -1 endi diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index a40912d5f4..d1572bb513 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -9,7 +9,7 @@ # fi set +e -set -x +#set -x UNAME_BIN=`which uname` OS_TYPE=`$UNAME_BIN` diff --git a/tests/script/test.sh b/tests/script/test.sh index a744a56aa6..88ed759296 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -126,7 +126,6 @@ if [ -n "$FILE_NAME" ]; then else echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME $PROGRAM -c $CFG_DIR -f $FILE_NAME -# valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${CODE_DIR}/../script/valgrind.log $PROGRAM -c $CFG_DIR -f $FILE_NAME fi else echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f basicSuite.sim From 2e507dc8b74bd2ea0c0a09005b54c136c3ab9f95 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 17 Dec 2021 23:01:27 -0500 Subject: [PATCH 49/50] TD-12193 physical plan serialization --- include/libs/planner/planner.h | 4 +- source/libs/planner/src/physicalPlanJson.c | 191 +++++++++++++++++++++ 2 files changed, 194 insertions(+), 1 deletion(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index f86ce77803..16271310e5 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -108,7 +108,7 @@ typedef struct SProjectPhyNode { typedef struct SExchangePhyNode { SPhyNode node; uint64_t srcTemplateId; // template id of datasource suplans - SArray *pSourceEpSet; // SEpSet, scheduler fill by calling qSetSuplanExecutionNode + SArray *pSrcEndPoints; // SEpAddrMsg, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhyNode; typedef struct SSubplanId { @@ -129,6 +129,7 @@ typedef struct SSubplan { typedef struct SQueryDag { uint64_t queryId; + int32_t numOfSubplans; SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. } SQueryDag; @@ -137,6 +138,7 @@ typedef struct SQueryDag { */ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); +// @eps is an array of SEpAddr structures int32_t qSetSubplanExecutionNode(SSubplan* subplan, SArray* eps); int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 7c0dc25627..2510797158 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -463,6 +463,191 @@ static bool exprInfoFromJson(const cJSON* json, void* obj) { return res; } +static const char* jkTimeWindowStartKey = "StartKey"; +static const char* jkTimeWindowEndKey = "EndKey"; + +static bool timeWindowToJson(const void* obj, cJSON* json) { + const STimeWindow* win = (const STimeWindow*)obj; + bool res = cJSON_AddNumberToObject(json, jkTimeWindowStartKey, win->skey); + if (res) { + res = cJSON_AddNumberToObject(json, jkTimeWindowEndKey, win->ekey); + } + return res; +} + +static bool timeWindowFromJson(const cJSON* json, void* obj) { + STimeWindow* win = (STimeWindow*)obj; + win->skey = getNumber(json, jkTimeWindowStartKey); + win->ekey = getNumber(json, jkTimeWindowEndKey); + return true; +} + +static const char* jkScanNodeTableId = "TableId"; +static const char* jkScanNodeTableType = "TableType"; + +static bool scanNodeToJson(const void* obj, cJSON* json) { + const SScanPhyNode* scan = (const SScanPhyNode*)obj; + bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, scan->uid); + if (res) { + res = cJSON_AddNumberToObject(json, jkScanNodeTableType, scan->tableType); + } + return res; +} + +static bool scanNodeFromJson(const cJSON* json, void* obj) { + SScanPhyNode* scan = (SScanPhyNode*)obj; + scan->uid = getNumber(json, jkScanNodeTableId); + scan->tableType = getNumber(json, jkScanNodeTableType); + return true; +} + +static const char* jkTableScanNodeFlag = "Flag"; +static const char* jkTableScanNodeWindow = "Window"; +static const char* jkTableScanNodeTagsConditions = "TagsConditions"; + +static bool tableScanNodeToJson(const void* obj, cJSON* json) { + const STableScanPhyNode* scan = (const STableScanPhyNode*)obj; + bool res = scanNodeToJson(obj, json); + if (res) { + res = cJSON_AddNumberToObject(json, jkTableScanNodeFlag, scan->scanFlag); + } + if (res) { + res = addObject(json, jkTableScanNodeWindow, timeWindowToJson, &scan->window); + } + if (res) { + res = addArray(json, jkTableScanNodeTagsConditions, exprInfoToJson, scan->pTagsConditions); + } + return res; +} + +static bool tableScanNodeFromJson(const cJSON* json, void* obj) { + STableScanPhyNode* scan = (STableScanPhyNode*)obj; + bool res = scanNodeFromJson(json, obj); + if (res) { + scan->scanFlag = getNumber(json, jkTableScanNodeFlag); + } + if (res) { + res = fromObject(json, jkTableScanNodeWindow, timeWindowFromJson, &scan->window, true); + } + if (res) { + res = fromArray(json, jkTableScanNodeTagsConditions, exprInfoFromJson, &scan->pTagsConditions, sizeof(SExprInfo)); + } + return res; +} + +static const char* jkEpAddrFqdn = "Fqdn"; +static const char* jkEpAddrPort = "Port"; + +static bool epAddrToJson(const void* obj, cJSON* json) { + const SEpAddrMsg* ep = (const SEpAddrMsg*)obj; + bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn); + if (res) { + res = cJSON_AddNumberToObject(json, jkEpAddrPort, ep->port); + } + return res; +} + +static bool epAddrFromJson(const cJSON* json, void* obj) { + SEpAddrMsg* ep = (SEpAddrMsg*)obj; + copyString(json, jkEpAddrFqdn, ep->fqdn); + ep->port = getNumber(json, jkEpAddrPort); + return true; +} + +static const char* jkExchangeNodeSrcTemplateId = "SrcTemplateId"; +static const char* jkExchangeNodeSrcEndPoints = "SrcEndPoints"; + +static bool exchangeNodeToJson(const void* obj, cJSON* json) { + const SExchangePhyNode* exchange = (const SExchangePhyNode*)obj; + bool res = cJSON_AddNumberToObject(json, jkExchangeNodeSrcTemplateId, exchange->srcTemplateId); + if (res) { + res = addArray(json, jkExchangeNodeSrcEndPoints, epAddrToJson, exchange->pSrcEndPoints); + } + return res; +} + +static bool exchangeNodeFromJson(const cJSON* json, void* obj) { + SExchangePhyNode* exchange = (SExchangePhyNode*)obj; + exchange->srcTemplateId = getNumber(json, jkExchangeNodeSrcTemplateId); + return fromArray(json, jkExchangeNodeSrcEndPoints, epAddrFromJson, &exchange->pSrcEndPoints, sizeof(SEpAddrMsg)); +} + +static bool specificPhyNodeToJson(const void* obj, cJSON* json) { + const SPhyNode* phyNode = (const SPhyNode*)obj; + switch (phyNode->info.type) { + case OP_TableScan: + case OP_DataBlocksOptScan: + case OP_TableSeqScan: + return tableScanNodeToJson(obj, json); + case OP_TagScan: + case OP_SystemTableScan: + return scanNodeToJson(obj, json); + case OP_Aggregate: + break; // todo + case OP_Project: + return true; + case OP_Groupby: + case OP_Limit: + case OP_SLimit: + case OP_TimeWindow: + case OP_SessionWindow: + case OP_StateWindow: + case OP_Fill: + case OP_MultiTableAggregate: + case OP_MultiTableTimeInterval: + case OP_Filter: + case OP_Distinct: + case OP_Join: + case OP_AllTimeWindow: + case OP_AllMultiTableTimeInterval: + case OP_Order: + break; // todo + case OP_Exchange: + return exchangeNodeToJson(obj, json); + default: + break; + } + return false; +} + +static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { + SPhyNode* phyNode = (SPhyNode*)obj; + switch (phyNode->info.type) { + case OP_TableScan: + case OP_DataBlocksOptScan: + case OP_TableSeqScan: + return tableScanNodeFromJson(json, obj); + case OP_TagScan: + case OP_SystemTableScan: + return scanNodeFromJson(json, obj); + case OP_Aggregate: + break; // todo + case OP_Project: + return true; + case OP_Groupby: + case OP_Limit: + case OP_SLimit: + case OP_TimeWindow: + case OP_SessionWindow: + case OP_StateWindow: + case OP_Fill: + case OP_MultiTableAggregate: + case OP_MultiTableTimeInterval: + case OP_Filter: + case OP_Distinct: + case OP_Join: + case OP_AllTimeWindow: + case OP_AllMultiTableTimeInterval: + case OP_Order: + break; // todo + case OP_Exchange: + return exchangeNodeFromJson(json, obj); + default: + break; + } + return false; +} + static const char* jkPnodeName = "Name"; static const char* jkPnodeTargets = "Targets"; static const char* jkPnodeConditions = "Conditions"; @@ -484,6 +669,9 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) { if (res) { res = addArray(jNode, jkPnodeChildren, phyNodeToJson, phyNode->pChildren); } + if (res) { + res = addObject(jNode, phyNode->info.name, specificPhyNodeToJson, phyNode); + } return res; } @@ -501,6 +689,9 @@ static bool phyNodeFromJson(const cJSON* json, void* obj) { if (res) { res = fromArray(json, jkPnodeChildren, phyNodeFromJson, &node->pChildren, sizeof(SSlotSchema)); } + if (res) { + res = fromObject(json, node->info.name, specificPhyNodeFromJson, node, true); + } return res; } From 1afa3de4c1b7aeb4262aad62a3780c5d356ab652 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 17 Dec 2021 23:34:02 -0500 Subject: [PATCH 50/50] TD-12194 Improve the planner interface. --- include/libs/planner/planner.h | 7 ++++-- source/libs/planner/inc/plannerInt.h | 3 ++- source/libs/planner/src/physicalPlan.c | 30 +++++++++++++++----------- source/libs/planner/src/planner.c | 4 ++++ 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 16271310e5..3f4d21a746 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -138,8 +138,11 @@ typedef struct SQueryDag { */ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); -// @eps is an array of SEpAddr structures -int32_t qSetSubplanExecutionNode(SSubplan* subplan, SArray* eps); +// Set datasource of this subplan, multiple calls may be made to a subplan. +// @subplan subplan to be schedule +// @templateId templateId of a group of datasource subplans of this @subplan +// @eps Execution location of this group of datasource subplans, is an array of SEpAddr structures +int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps); int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 68a9518680..19563a8a0c 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -100,8 +100,9 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag); - +int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps); int32_t subPlanToString(const SSubplan *pPhyNode, char** str); +int32_t stringToSubplan(const char* str, SSubplan** subplan); /** * Destroy the query plan object. diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 5a891d4b28..f187ec0ec9 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -19,6 +19,13 @@ #define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan #define RECOVERY_CURRENT_SUBPLAN(cxt) cxt->pCurrentSubplan = _ +typedef struct SPlanContext { + struct SCatalog* pCatalog; + struct SQueryDag* pDag; + SSubplan* pCurrentSubplan; + SSubplanId nextId; +} SPlanContext; + static const char* gOpName[] = { "Unknown", #define INCLUDE_AS_NAME @@ -26,12 +33,14 @@ static const char* gOpName[] = { #undef INCLUDE_AS_NAME }; -typedef struct SPlanContext { - struct SCatalog* pCatalog; - struct SQueryDag* pDag; - SSubplan* pCurrentSubplan; - SSubplanId nextId; -} SPlanContext; +int32_t opNameToOpType(const char* name) { + for (int32_t i = 1; i < sizeof(gOpName) / sizeof(gOpName[0]); ++i) { + if (strcmp(name, gOpName[i])) { + return i; + } + } + return OP_Unknown; +} static void toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { SWAP(dataBlockSchema->pSchema, pPlanNode->pSchema, SSchema*); @@ -216,11 +225,6 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD return TSDB_CODE_SUCCESS; } -int32_t opNameToOpType(const char* name) { - for (int32_t i = 1; i < sizeof(gOpName) / sizeof(gOpName[0]); ++i) { - if (strcmp(name, gOpName[i])) { - return i; - } - } - return OP_Unknown; +int32_t setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps) { + //todo } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 3d2bb895d5..3a90acb5fd 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -46,6 +46,10 @@ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* return TSDB_CODE_SUCCESS; } +int32_t qSetSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SArray* eps) { + return setSubplanExecutionNode(subplan, templateId, eps); +} + int32_t qSubPlanToString(const SSubplan *subplan, char** str) { return subPlanToString(subplan, str); }