diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5c2d603ccf..ab18554b27 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -114,6 +114,9 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_TABLE, TSDB_MGMT_TABLE_DNODE, TSDB_MGMT_TABLE_MNODE, + TSDB_MGMT_TABLE_QNODE, + TSDB_MGMT_TABLE_SNODE, + TSDB_MGMT_TABLE_BNODE, TSDB_MGMT_TABLE_VGROUP, TSDB_MGMT_TABLE_STB, TSDB_MGMT_TABLE_MODULE, @@ -909,15 +912,15 @@ typedef struct { typedef struct { int32_t dnodeId; -} SCreateQnodeInMsg, SDropQnodeInMsg; +} SMCreateQnodeMsg, SMDropQnodeMsg, SDCreateQnodeMsg, SDDropQnodeMsg; typedef struct { int32_t dnodeId; -} SCreateSnodeInMsg, SDropSnodeInMsg; +} SMCreateSnodeMsg, SMDropSnodeMsg, SDCreateSnodeMsg, SDDropSnodeMsg; typedef struct { int32_t dnodeId; -} SCreateBnodeInMsg, SDropBnodeInMsg; +} SMCreateBnodeMsg, SMDropBnodeMsg, SDCreateBnodeMsg, SDDropBnodeMsg; typedef struct { int32_t dnodeId; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 76ee31d9c9..d6071ebcf3 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -70,6 +70,15 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_CREATE_MNODE, "dnode-create-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE, "dnode-alter-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_MNODE, "dnode-drop-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_QNODE, "dnode-create-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_QNODE, "dnode-alter-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_QNODE, "dnode-drop-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_SNODE, "dnode-create-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_SNODE, "dnode-alter-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_SNODE, "dnode-drop-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_BNODE, "dnode-create-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_BNODE, "dnode-alter-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_DROP_BNODE, "dnode-drop-bnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "dnode-create-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE, "dnode-alter-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL) @@ -90,9 +99,20 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_DROP_USER, "mnode-drop-user", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DNODE, "mnode-create-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_DNODE, "mnode-config-dnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DNODE, "mnode-alter-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_DNODE, "mnode-drop-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_MNODE, "mnode-create-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_MNODE, "mnode-alter-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_MNODE, "mnode-drop-mnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_QNODE, "mnode-create-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_QNODE, "mnode-alter-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_QNODE, "mnode-drop-qnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SNODE, "mnode-create-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_SNODE, "mnode-alter-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_SNODE, "mnode-drop-snode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_BNODE, "mnode-create-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_BNODE, "mnode-alter-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_BNODE, "mnode-drop-bnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DB, "mnode-create-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_DB, "mnode-drop-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_USE_DB, "mnode-use-db", NULL, NULL) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index f94090d7de..33f9dc5a1a 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -157,18 +157,21 @@ typedef enum { SDB_TRANS = 1, SDB_CLUSTER = 2, SDB_MNODE = 3, - SDB_DNODE = 4, - SDB_USER = 5, - SDB_AUTH = 6, - SDB_ACCT = 7, - SDB_CONSUMER = 8, - SDB_CGROUP = 9, - SDB_TOPIC = 10, - SDB_VGROUP = 11, - SDB_STB = 12, - SDB_DB = 13, - SDB_FUNC = 14, - SDB_MAX = 15 + SDB_QNODE = 4, + SDB_SNODE = 5, + SDB_BNODE = 6, + SDB_DNODE = 7, + SDB_USER = 8, + SDB_AUTH = 9, + SDB_ACCT = 10, + SDB_CONSUMER = 11, + SDB_CGROUP = 12, + SDB_TOPIC = 13, + SDB_VGROUP = 14, + SDB_STB = 15, + SDB_DB = 16, + SDB_FUNC = 17, + SDB_MAX = 18 } ESdbType; typedef struct SSdb SSdb; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ba3e122db3..ae36ac7216 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -172,10 +172,15 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_DNODE_EP TAOS_DEF_ERROR_CODE(0, 0x0347) #define TSDB_CODE_MND_INVALID_DNODE_ID TAOS_DEF_ERROR_CODE(0, 0x0348) -// mnode-mnode +// mnode-node #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0350) #define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0351) -#define TSDB_CODE_MND_TOO_MANY_MNODES TAOS_DEF_ERROR_CODE(0, 0x0352) +#define TSDB_CODE_MND_QNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0352) +#define TSDB_CODE_MND_QNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0353) +#define TSDB_CODE_MND_SNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0354) +#define TSDB_CODE_MND_SNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0355) +#define TSDB_CODE_MND_BNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0356) +#define TSDB_CODE_MND_BNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0357) // mnode-acct #define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index 992f6ac0a1..66b619318d 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -256,7 +256,7 @@ static int32_t dndDropBnode(SDnode *pDnode) { } int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateBnodeInMsg *pMsg = pRpcMsg->pCont; + SDCreateBnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { @@ -268,7 +268,7 @@ int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropBnodeInMsg *pMsg = pRpcMsg->pCont; + SDDropBnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { diff --git a/source/dnode/mgmt/impl/src/dndDnode.c b/source/dnode/mgmt/impl/src/dndDnode.c index 98763fce30..8807579564 100644 --- a/source/dnode/mgmt/impl/src/dndDnode.c +++ b/source/dnode/mgmt/impl/src/dndDnode.c @@ -15,9 +15,12 @@ #define _DEFAULT_SOURCE #include "dndDnode.h" +#include "dndBnode.h" +#include "dndMnode.h" +#include "dndQnode.h" +#include "dndSnode.h" #include "dndTransport.h" #include "dndVnodes.h" -#include "dndMnode.h" static int32_t dndInitMgmtWorker(SDnode *pDnode); static void dndCleanupMgmtWorker(SDnode *pDnode); @@ -648,6 +651,24 @@ static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { case TDMT_DND_DROP_MNODE: code = dndProcessDropMnodeReq(pDnode, pMsg); break; + case TDMT_DND_CREATE_QNODE: + code = dndProcessCreateQnodeReq(pDnode, pMsg); + break; + case TDMT_DND_DROP_QNODE: + code = dndProcessDropQnodeReq(pDnode, pMsg); + break; + case TDMT_DND_CREATE_SNODE: + code = dndProcessCreateSnodeReq(pDnode, pMsg); + break; + case TDMT_DND_DROP_SNODE: + code = dndProcessDropSnodeReq(pDnode, pMsg); + break; + case TDMT_DND_CREATE_BNODE: + code = dndProcessCreateBnodeReq(pDnode, pMsg); + break; + case TDMT_DND_DROP_BNODE: + code = dndProcessDropBnodeReq(pDnode, pMsg); + break; case TDMT_DND_CONFIG_DNODE: code = dndProcessConfigDnodeReq(pDnode, pMsg); break; diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 5d04a4f449..1f3d6ee371 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -261,7 +261,7 @@ static int32_t dndDropQnode(SDnode *pDnode) { } int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateQnodeInMsg *pMsg = pRpcMsg->pCont; + SDCreateQnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { @@ -273,7 +273,7 @@ int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropQnodeInMsg *pMsg = pRpcMsg->pCont; + SDDropQnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index 151fc7e6a1..ab4ca191a9 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -256,7 +256,7 @@ static int32_t dndDropSnode(SDnode *pDnode) { } int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateSnodeInMsg *pMsg = pRpcMsg->pCont; + SDCreateSnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { @@ -268,7 +268,7 @@ int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropSnodeInMsg *pMsg = pRpcMsg->pCont; + SDDropSnodeMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 8db09ea4cd..ba93d1913d 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -37,6 +37,18 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_MNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE)] = dndProcessMgmtMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_MNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_QNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_QNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_SNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_SNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_BNODE_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE)] = dndProcessMgmtMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_BNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE)] = dndProcessMgmtMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CREATE_VNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_ALTER_VNODE)] = dndProcessMgmtMsg; @@ -66,6 +78,12 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DNODE)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_MNODE)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_MNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_QNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_QNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_SNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_SNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_BNODE)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_BNODE)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_USE_DB)] = dndProcessMnodeWriteMsg; diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index dcce270d7d..b0596bed08 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -8,6 +8,9 @@ add_subdirectory(db) add_subdirectory(dnode) # add_subdirectory(func) add_subdirectory(mnode) +add_subdirectory(qnode) +add_subdirectory(snode) +add_subdirectory(bnode) add_subdirectory(profile) add_subdirectory(show) add_subdirectory(stb) diff --git a/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt new file mode 100644 index 0000000000..ea769a701c --- /dev/null +++ b/source/dnode/mgmt/impl/test/bnode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. STEST_SRC) +add_executable(dnode_test_bnode ${STEST_SRC}) +target_link_libraries( + dnode_test_bnode + PUBLIC sut +) + +add_test( + NAME dnode_test_bnode + COMMAND dnode_test_bnode +) diff --git a/source/dnode/mgmt/impl/test/bnode/bnode.cpp b/source/dnode/mgmt/impl/test/bnode/bnode.cpp new file mode 100644 index 0000000000..b1a85522c4 --- /dev/null +++ b/source/dnode/mgmt/impl/test/bnode/bnode.cpp @@ -0,0 +1,154 @@ +/** + * @file dnode.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module dnode-msg tests + * @version 0.1 + * @date 2021-12-15 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "base.h" + +class DndTestBnode : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_bnode1", 9068); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9068"; + + server2.Start("/tmp/dnode_test_bnode2", fqdn, 9069, firstEp); + taosMsleep(300); + } + + static void TearDownTestSuite() { + server2.Stop(); + test.Cleanup(); + } + + static Testbase test; + static TestServer server2; +}; + +Testbase DndTestBnode::test; +TestServer DndTestBnode::server2; + +TEST_F(DndTestBnode, 01_ShowBnode) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + CHECK_META("show bnodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); +} + +TEST_F(DndTestBnode, 02_Create_Bnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateBnodeMsg); + + SMCreateBnodeMsg* pReq = (SMCreateBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + CHECK_META("show bnodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9068", TSDB_EP_LEN); + CheckTimestamp(); + } +} + +TEST_F(DndTestBnode, 03_Create_Bnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateBnodeMsg); + + SMCreateBnodeMsg* pReq = (SMCreateBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } +} + +TEST_F(DndTestBnode, 04_Create_Bnode) { + { + // create dnode + int32_t contLen = sizeof(SCreateDnodeMsg); + + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9069); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + taosMsleep(1300); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + // create bnode + int32_t contLen = sizeof(SMCreateBnodeMsg); + + SMCreateBnodeMsg* pReq = (SMCreateBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + + CheckInt16(1); + CheckInt16(2); + CheckBinary("localhost:9068", TSDB_EP_LEN); + CheckBinary("localhost:9069", TSDB_EP_LEN); + CheckTimestamp(); + CheckTimestamp(); + } + + { + // drop bnode + int32_t contLen = sizeof(SMDropBnodeMsg); + + SMDropBnodeMsg* pReq = (SMDropBnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_BNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_BNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9068", TSDB_EP_LEN); + CheckTimestamp(); + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt b/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt new file mode 100644 index 0000000000..2edb6de8b8 --- /dev/null +++ b/source/dnode/mgmt/impl/test/qnode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. QTEST_SRC) +add_executable(dnode_test_qnode ${QTEST_SRC}) +target_link_libraries( + dnode_test_qnode + PUBLIC sut +) + +add_test( + NAME dnode_test_qnode + COMMAND dnode_test_qnode +) diff --git a/source/dnode/mgmt/impl/test/qnode/qnode.cpp b/source/dnode/mgmt/impl/test/qnode/qnode.cpp new file mode 100644 index 0000000000..40464de816 --- /dev/null +++ b/source/dnode/mgmt/impl/test/qnode/qnode.cpp @@ -0,0 +1,154 @@ +/** + * @file dnode.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module dnode-msg tests + * @version 0.1 + * @date 2021-12-15 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "base.h" + +class DndTestQnode : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_qnode1", 9064); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9064"; + + server2.Start("/tmp/dnode_test_qnode2", fqdn, 9065, firstEp); + taosMsleep(300); + } + + static void TearDownTestSuite() { + server2.Stop(); + test.Cleanup(); + } + + static Testbase test; + static TestServer server2; +}; + +Testbase DndTestQnode::test; +TestServer DndTestQnode::server2; + +TEST_F(DndTestQnode, 01_ShowQnode) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); +} + +TEST_F(DndTestQnode, 02_Create_Qnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateQnodeMsg); + + SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9064", TSDB_EP_LEN); + CheckTimestamp(); + } +} + +TEST_F(DndTestQnode, 03_Create_Qnode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateQnodeMsg); + + SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } +} + +TEST_F(DndTestQnode, 04_Create_Qnode) { + { + // create dnode + int32_t contLen = sizeof(SCreateDnodeMsg); + + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9065); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + taosMsleep(1300); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + // create qnode + int32_t contLen = sizeof(SMCreateQnodeMsg); + + SMCreateQnodeMsg* pReq = (SMCreateQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + + CheckInt16(1); + CheckInt16(2); + CheckBinary("localhost:9064", TSDB_EP_LEN); + CheckBinary("localhost:9065", TSDB_EP_LEN); + CheckTimestamp(); + CheckTimestamp(); + } + + { + // drop qnode + int32_t contLen = sizeof(SMDropQnodeMsg); + + SMDropQnodeMsg* pReq = (SMDropQnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_QNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_QNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9064", TSDB_EP_LEN); + CheckTimestamp(); + } +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/snode/CMakeLists.txt b/source/dnode/mgmt/impl/test/snode/CMakeLists.txt new file mode 100644 index 0000000000..180b1ec163 --- /dev/null +++ b/source/dnode/mgmt/impl/test/snode/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. STEST_SRC) +add_executable(dnode_test_snode ${STEST_SRC}) +target_link_libraries( + dnode_test_snode + PUBLIC sut +) + +add_test( + NAME dnode_test_snode + COMMAND dnode_test_snode +) diff --git a/source/dnode/mgmt/impl/test/snode/snode.cpp b/source/dnode/mgmt/impl/test/snode/snode.cpp new file mode 100644 index 0000000000..a14a575beb --- /dev/null +++ b/source/dnode/mgmt/impl/test/snode/snode.cpp @@ -0,0 +1,154 @@ +/** + * @file dnode.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module dnode-msg tests + * @version 0.1 + * @date 2021-12-15 + * + * @copyright Copyright (c) 2021 + * + */ + +#include "base.h" + +class DndTestSnode : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() { + test.Init("/tmp/dnode_test_snode1", 9066); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9066"; + + server2.Start("/tmp/dnode_test_snode2", fqdn, 9067, firstEp); + taosMsleep(300); + } + + static void TearDownTestSuite() { + server2.Stop(); + test.Cleanup(); + } + + static Testbase test; + static TestServer server2; +}; + +Testbase DndTestSnode::test; +TestServer DndTestSnode::server2; + +TEST_F(DndTestSnode, 01_ShowSnode) { + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + CHECK_META("show snodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 0); +} + +TEST_F(DndTestSnode, 02_Create_Snode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateSnodeMsg); + + SMCreateSnodeMsg* pReq = (SMCreateSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + CHECK_META("show snodes", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9066", TSDB_EP_LEN); + CheckTimestamp(); + } +} + +TEST_F(DndTestSnode, 03_Create_Snode_Invalid_Id) { + { + int32_t contLen = sizeof(SMCreateSnodeMsg); + + SMCreateSnodeMsg* pReq = (SMCreateSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST); + } +} + +TEST_F(DndTestSnode, 04_Create_Snode) { + { + // create dnode + int32_t contLen = sizeof(SCreateDnodeMsg); + + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9067); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + taosMsleep(1300); + test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + // create snode + int32_t contLen = sizeof(SMCreateSnodeMsg); + + SMCreateSnodeMsg* pReq = (SMCreateSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 2); + + CheckInt16(1); + CheckInt16(2); + CheckBinary("localhost:9066", TSDB_EP_LEN); + CheckBinary("localhost:9067", TSDB_EP_LEN); + CheckTimestamp(); + CheckTimestamp(); + } + + { + // drop snode + int32_t contLen = sizeof(SMDropSnodeMsg); + + SMDropSnodeMsg* pReq = (SMDropSnodeMsg*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pMsg = test.SendMsg(TDMT_MND_DROP_SNODE, pReq, contLen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + test.SendShowMetaMsg(TSDB_MGMT_TABLE_SNODE, ""); + test.SendShowRetrieveMsg(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckInt16(1); + CheckBinary("localhost:9066", TSDB_EP_LEN); + CheckTimestamp(); + } +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndBalance.h b/source/dnode/mnode/impl/inc/mndBnode.h similarity index 81% rename from source/dnode/mnode/impl/inc/mndBalance.h rename to source/dnode/mnode/impl/inc/mndBnode.h index fa3679d033..cefc0b1859 100644 --- a/source/dnode/mnode/impl/inc/mndBalance.h +++ b/source/dnode/mnode/impl/inc/mndBnode.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_MND_BALANCE_H_ -#define _TD_MND_BALANCE_H_ +#ifndef _TD_MND_BNODE_H_ +#define _TD_MND_BNODE_H_ #include "mndInt.h" @@ -22,11 +22,11 @@ extern "C" { #endif -int32_t mndInitBalance(SMnode *pMnode); -void mndCleanupBalance(SMnode *pMnode); +int32_t mndInitBnode(SMnode *pMnode); +void mndCleanupBnode(SMnode *pMnode); #ifdef __cplusplus } #endif -#endif /*_TD_MND_BALANCE_H_*/ +#endif /*_TD_MND_BNODE_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a874e67210..b93fc0951c 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -144,6 +144,27 @@ typedef struct { SDnodeObj *pDnode; } SMnodeObj; +typedef struct { + int32_t id; + int64_t createdTime; + int64_t updateTime; + SDnodeObj *pDnode; +} SQnodeObj; + +typedef struct { + int32_t id; + int64_t createdTime; + int64_t updateTime; + SDnodeObj *pDnode; +} SSnodeObj; + +typedef struct { + int32_t id; + int64_t createdTime; + int64_t updateTime; + SDnodeObj *pDnode; +} SBnodeObj; + typedef struct { int32_t maxUsers; int32_t maxDbs; diff --git a/source/dnode/mnode/impl/src/mndBalance.c b/source/dnode/mnode/impl/inc/mndQnode.h similarity index 74% rename from source/dnode/mnode/impl/src/mndBalance.c rename to source/dnode/mnode/impl/inc/mndQnode.h index 6ce89b96a3..5d177b3f6d 100644 --- a/source/dnode/mnode/impl/src/mndBalance.c +++ b/source/dnode/mnode/impl/inc/mndQnode.h @@ -13,9 +13,20 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE -#include "os.h" +#ifndef _TD_MND_QNODE_H_ +#define _TD_MND_QNODE_H_ + #include "mndInt.h" -int32_t mndInitBalance(SMnode *pMnode) { return 0; } -void mndCleanupBalance(SMnode *pMnode) {} \ No newline at end of file +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitQnode(SMnode *pMnode); +void mndCleanupQnode(SMnode *pMnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_QNODE_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndSnode.h b/source/dnode/mnode/impl/inc/mndSnode.h new file mode 100644 index 0000000000..8d64879605 --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndSnode.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_SNODE_H_ +#define _TD_MND_SNODE_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitSnode(SMnode *pMnode); +void mndCleanupSnode(SMnode *pMnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_SNODE_H_*/ diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c new file mode 100644 index 0000000000..2d14e0f92e --- /dev/null +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndBnode.h" +#include "mndDnode.h" +#include "mndShow.h" +#include "mndTrans.h" + +#define TSDB_BNODE_VER_NUMBER 1 +#define TSDB_BNODE_RESERVE_SIZE 64 + +static SSdbRaw *mndBnodeActionEncode(SBnodeObj *pObj); +static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); +static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); +static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOldBnode, SBnodeObj *pNewBnode); +static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pMsg); +static int32_t mndGetBnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveBnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); + +int32_t mndInitBnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_BNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndBnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndBnodeActionDecode, + .insertFp = (SdbInsertFp)mndBnodeActionInsert, + .updateFp = (SdbUpdateFp)mndBnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndBnodeActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_BNODE, mndProcessCreateBnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_BNODE, mndProcessDropBnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_BNODE_RSP, mndProcessCreateBnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_BNODE_RSP, mndProcessDropBnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndGetBnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndRetrieveBnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndCancelGetNextBnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupBnode(SMnode *pMnode) {} + +static SBnodeObj *mndAcquireBnode(SMnode *pMnode, int32_t snodeId) { + SSdb *pSdb = pMnode->pSdb; + SBnodeObj *pObj = sdbAcquire(pSdb, SDB_BNODE, &snodeId); + if (pObj == NULL) { + terrno = TSDB_CODE_MND_BNODE_NOT_EXIST; + } + return pObj; +} + +static void mndReleaseBnode(SMnode *pMnode, SBnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pObj); +} + +static SSdbRaw *mndBnodeActionEncode(SBnodeObj *pObj) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_BNODE, TSDB_BNODE_VER_NUMBER, sizeof(SBnodeObj) + TSDB_BNODE_RESERVE_SIZE); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, pObj->id); + SDB_SET_INT64(pRaw, dataPos, pObj->createdTime) + SDB_SET_INT64(pRaw, dataPos, pObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_BNODE_RESERVE_SIZE) + + return pRaw; +} + +static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_BNODE_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode snode since %s", terrstr()); + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SBnodeObj)); + SBnodeObj *pObj = sdbGetRowObj(pRow); + if (pObj == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &pObj->id) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_BNODE_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj) { + mTrace("snode:%d, perform insert action", pObj->id); + pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); + if (pObj->pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("snode:%d, failed to perform insert action since %s", pObj->id, terrstr()); + return -1; + } + + return 0; +} + +static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj) { + mTrace("snode:%d, perform delete action", pObj->id); + if (pObj->pDnode != NULL) { + sdbRelease(pSdb, pObj->pDnode); + pObj->pDnode = NULL; + } + + return 0; +} + +static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOldBnode, SBnodeObj *pNewBnode) { + mTrace("snode:%d, perform update action", pOldBnode->id); + pOldBnode->updateTime = pNewBnode->updateTime; + return 0; +} + +static int32_t mndSetCreateBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndBnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndBnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { + SDCreateBnodeMsg *pMsg = malloc(sizeof(SDCreateBnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDCreateBnodeMsg); + action.msgType = TDMT_DND_CREATE_BNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndCreateBnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateBnodeMsg *pCreate) { + SBnodeObj snodeObj = {0}; + snodeObj.id = pDnode->id; + snodeObj.createdTime = taosGetTimestampMs(); + snodeObj.updateTime = snodeObj.createdTime; + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + goto CREATE_BNODE_OVER; + } + mDebug("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); + + if (mndSetCreateBnodeRedoLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + if (mndSetCreateBnodeCommitLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + if (mndSetCreateBnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto CREATE_BNODE_OVER; + } + + code = 0; + +CREATE_BNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessCreateBnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateBnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->dnodeId = htonl(pCreate->dnodeId); + + mDebug("snode:%d, start to create", pCreate->dnodeId); + + SBnodeObj *pObj = mndAcquireBnode(pMnode, pCreate->dnodeId); + if (pObj != NULL) { + mError("snode:%d, snode already exist", pObj->id); + mndReleaseBnode(pMnode, pObj); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + if (pDnode == NULL) { + mError("snode:%d, dnode not exist", pCreate->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndCreateBnode(pMnode, pMsg, pDnode, pCreate); + mndReleaseDnode(pMnode, pDnode); + + if (code != 0) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndSetDropBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndBnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + return 0; +} + +static int32_t mndSetDropBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndBnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { + SDDropBnodeMsg *pMsg = malloc(sizeof(SDDropBnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropBnodeMsg); + action.msgType = TDMT_DND_DROP_BNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndDropBnode(SMnode *pMnode, SMnodeMsg *pMsg, SBnodeObj *pObj) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to drop since %s", pObj->id, terrstr()); + goto DROP_BNODE_OVER; + } + + mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); + + if (mndSetDropBnodeRedoLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + if (mndSetDropBnodeCommitLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + if (mndSetDropBnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto DROP_BNODE_OVER; + } + + code = 0; + +DROP_BNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessDropBnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMDropBnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("snode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("snode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SBnodeObj *pObj = mndAcquireBnode(pMnode, pDrop->dnodeId); + if (pObj == NULL) { + mError("snode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropBnode(pMnode, pMsg, pObj); + if (code != 0) { + mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pMnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateBnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndProcessDropBnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndGetBnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_BNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveBnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SBnodeObj *pObj = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_BNODE, pShow->pIter, (void **)&pObj); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pObj->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pObj->pDnode->ep, pShow->bytes[cols]); + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pObj->createdTime; + cols++; + + numOfRows++; + sdbRelease(pSdb, pObj); + } + + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 1800fd8e83..04e77c0136 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -329,7 +329,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SCreateMnodeMsg *pCreate) { SMnodeObj mnodeObj = {0}; - mnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_MNODE); + mnodeObj.id = pDnode->id; mnodeObj.createdTime = taosGetTimestampMs(); mnodeObj.updateTime = mnodeObj.createdTime; @@ -562,7 +562,7 @@ static int32_t mndProcessDropMnodeReq(SMnodeMsg *pMsg) { return -1; } - sdbRelease(pMnode->pSdb, pMnode); + sdbRelease(pMnode->pSdb, pObj); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c new file mode 100644 index 0000000000..3b27764c7c --- /dev/null +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndQnode.h" +#include "mndDnode.h" +#include "mndShow.h" +#include "mndTrans.h" + +#define TSDB_QNODE_VER_NUMBER 1 +#define TSDB_QNODE_RESERVE_SIZE 64 + +static SSdbRaw *mndQnodeActionEncode(SQnodeObj *pObj); +static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj); +static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj); +static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOldQnode, SQnodeObj *pNewQnode); +static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pMsg); +static int32_t mndGetQnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveQnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter); + +int32_t mndInitQnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_QNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndQnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndQnodeActionDecode, + .insertFp = (SdbInsertFp)mndQnodeActionInsert, + .updateFp = (SdbUpdateFp)mndQnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndQnodeActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_QNODE, mndProcessCreateQnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_QNODE, mndProcessDropQnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_QNODE_RSP, mndProcessCreateQnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_QNODE_RSP, mndProcessDropQnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_QNODE, mndGetQnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_QNODE, mndRetrieveQnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_QNODE, mndCancelGetNextQnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupQnode(SMnode *pMnode) {} + +static SQnodeObj *mndAcquireQnode(SMnode *pMnode, int32_t qnodeId) { + SSdb *pSdb = pMnode->pSdb; + SQnodeObj *pObj = sdbAcquire(pSdb, SDB_QNODE, &qnodeId); + if (pObj == NULL) { + terrno = TSDB_CODE_MND_QNODE_NOT_EXIST; + } + return pObj; +} + +static void mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pObj); +} + +static SSdbRaw *mndQnodeActionEncode(SQnodeObj *pObj) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_QNODE, TSDB_QNODE_VER_NUMBER, sizeof(SQnodeObj) + TSDB_QNODE_RESERVE_SIZE); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, pObj->id); + SDB_SET_INT64(pRaw, dataPos, pObj->createdTime) + SDB_SET_INT64(pRaw, dataPos, pObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_QNODE_RESERVE_SIZE) + + return pRaw; +} + +static SSdbRow *mndQnodeActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_QNODE_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode qnode since %s", terrstr()); + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SQnodeObj)); + SQnodeObj *pObj = sdbGetRowObj(pRow); + if (pObj == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &pObj->id) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_QNODE_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndQnodeActionInsert(SSdb *pSdb, SQnodeObj *pObj) { + mTrace("qnode:%d, perform insert action", pObj->id); + pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); + if (pObj->pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("qnode:%d, failed to perform insert action since %s", pObj->id, terrstr()); + return -1; + } + + return 0; +} + +static int32_t mndQnodeActionDelete(SSdb *pSdb, SQnodeObj *pObj) { + mTrace("qnode:%d, perform delete action", pObj->id); + if (pObj->pDnode != NULL) { + sdbRelease(pSdb, pObj->pDnode); + pObj->pDnode = NULL; + } + + return 0; +} + +static int32_t mndQnodeActionUpdate(SSdb *pSdb, SQnodeObj *pOldQnode, SQnodeObj *pNewQnode) { + mTrace("qnode:%d, perform update action", pOldQnode->id); + pOldQnode->updateTime = pNewQnode->updateTime; + return 0; +} + +static int32_t mndSetCreateQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { + SDCreateQnodeMsg *pMsg = malloc(sizeof(SDCreateQnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDCreateQnodeMsg); + action.msgType = TDMT_DND_CREATE_QNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndCreateQnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateQnodeMsg *pCreate) { + SQnodeObj qnodeObj = {0}; + qnodeObj.id = pDnode->id; + qnodeObj.createdTime = taosGetTimestampMs(); + qnodeObj.updateTime = qnodeObj.createdTime; + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("qnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + goto CREATE_QNODE_OVER; + } + mDebug("trans:%d, used to create qnode:%d", pTrans->id, pCreate->dnodeId); + + if (mndSetCreateQnodeRedoLogs(pTrans, &qnodeObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + if (mndSetCreateQnodeCommitLogs(pTrans, &qnodeObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + if (mndSetCreateQnodeRedoActions(pTrans, pDnode, &qnodeObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto CREATE_QNODE_OVER; + } + + code = 0; + +CREATE_QNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessCreateQnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateQnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->dnodeId = htonl(pCreate->dnodeId); + + mDebug("qnode:%d, start to create", pCreate->dnodeId); + + SQnodeObj *pObj = mndAcquireQnode(pMnode, pCreate->dnodeId); + if (pObj != NULL) { + mError("qnode:%d, qnode already exist", pObj->id); + mndReleaseQnode(pMnode, pObj); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + if (pDnode == NULL) { + mError("qnode:%d, dnode not exist", pCreate->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndCreateQnode(pMnode, pMsg, pDnode, pCreate); + mndReleaseDnode(pMnode, pDnode); + + if (code != 0) { + mError("qnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndSetDropQnodeRedoLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndQnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + return 0; +} + +static int32_t mndSetDropQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndQnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { + SDDropQnodeMsg *pMsg = malloc(sizeof(SDDropQnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropQnodeMsg); + action.msgType = TDMT_DND_DROP_QNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndDropQnode(SMnode *pMnode, SMnodeMsg *pMsg, SQnodeObj *pObj) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("qnode:%d, failed to drop since %s", pObj->id, terrstr()); + goto DROP_QNODE_OVER; + } + + mDebug("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); + + if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto DROP_QNODE_OVER; + } + + code = 0; + +DROP_QNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessDropQnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMDropQnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("qnode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("qnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SQnodeObj *pObj = mndAcquireQnode(pMnode, pDrop->dnodeId); + if (pObj == NULL) { + mError("qnode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropQnode(pMnode, pMsg, pObj); + if (code != 0) { + mError("qnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pMnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateQnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndProcessDropQnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndGetQnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_QNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveQnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SQnodeObj *pObj = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_QNODE, pShow->pIter, (void **)&pObj); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pObj->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pObj->pDnode->ep, pShow->bytes[cols]); + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pObj->createdTime; + cols++; + + numOfRows++; + sdbRelease(pSdb, pObj); + } + + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextQnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index b89de94452..af633355ef 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -262,6 +262,12 @@ char *mndShowStr(int32_t showType) { return "show dnodes"; case TSDB_MGMT_TABLE_MNODE: return "show mnodes"; + case TSDB_MGMT_TABLE_QNODE: + return "show qnodes"; + case TSDB_MGMT_TABLE_SNODE: + return "show snodes"; + case TSDB_MGMT_TABLE_BNODE: + return "show bnodes"; case TSDB_MGMT_TABLE_VGROUP: return "show vgroups"; case TSDB_MGMT_TABLE_STB: diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c new file mode 100644 index 0000000000..a4825c0e82 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -0,0 +1,446 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndSnode.h" +#include "mndDnode.h" +#include "mndShow.h" +#include "mndTrans.h" + +#define TSDB_SNODE_VER_NUMBER 1 +#define TSDB_SNODE_RESERVE_SIZE 64 + +static SSdbRaw *mndSnodeActionEncode(SSnodeObj *pObj); +static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj); +static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj); +static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOldSnode, SSnodeObj *pNewSnode); +static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg); +static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pMsg); +static int32_t mndGetSnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveSnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter); + +int32_t mndInitSnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_SNODE, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndSnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndSnodeActionDecode, + .insertFp = (SdbInsertFp)mndSnodeActionInsert, + .updateFp = (SdbUpdateFp)mndSnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndSnodeActionDelete}; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SNODE, mndProcessCreateSnodeReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_SNODE, mndProcessDropSnodeReq); + mndSetMsgHandle(pMnode, TDMT_DND_CREATE_SNODE_RSP, mndProcessCreateSnodeRsp); + mndSetMsgHandle(pMnode, TDMT_DND_DROP_SNODE_RSP, mndProcessDropSnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndGetSnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndRetrieveSnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_SNODE, mndCancelGetNextSnode); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupSnode(SMnode *pMnode) {} + +static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) { + SSdb *pSdb = pMnode->pSdb; + SSnodeObj *pObj = sdbAcquire(pSdb, SDB_SNODE, &snodeId); + if (pObj == NULL) { + terrno = TSDB_CODE_MND_SNODE_NOT_EXIST; + } + return pObj; +} + +static void mndReleaseSnode(SMnode *pMnode, SSnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pObj); +} + +static SSdbRaw *mndSnodeActionEncode(SSnodeObj *pObj) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_SNODE, TSDB_SNODE_VER_NUMBER, sizeof(SSnodeObj) + TSDB_SNODE_RESERVE_SIZE); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, pObj->id); + SDB_SET_INT64(pRaw, dataPos, pObj->createdTime) + SDB_SET_INT64(pRaw, dataPos, pObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_SNODE_RESERVE_SIZE) + + return pRaw; +} + +static SSdbRow *mndSnodeActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_SNODE_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + mError("failed to decode snode since %s", terrstr()); + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SSnodeObj)); + SSnodeObj *pObj = sdbGetRowObj(pRow); + if (pObj == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &pObj->id) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_SNODE_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndSnodeActionInsert(SSdb *pSdb, SSnodeObj *pObj) { + mTrace("snode:%d, perform insert action", pObj->id); + pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); + if (pObj->pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("snode:%d, failed to perform insert action since %s", pObj->id, terrstr()); + return -1; + } + + return 0; +} + +static int32_t mndSnodeActionDelete(SSdb *pSdb, SSnodeObj *pObj) { + mTrace("snode:%d, perform delete action", pObj->id); + if (pObj->pDnode != NULL) { + sdbRelease(pSdb, pObj->pDnode); + pObj->pDnode = NULL; + } + + return 0; +} + +static int32_t mndSnodeActionUpdate(SSdb *pSdb, SSnodeObj *pOldSnode, SSnodeObj *pNewSnode) { + mTrace("snode:%d, perform update action", pOldSnode->id); + pOldSnode->updateTime = pNewSnode->updateTime; + return 0; +} + +static int32_t mndSetCreateSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { + SDCreateSnodeMsg *pMsg = malloc(sizeof(SDCreateSnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDCreateSnodeMsg); + action.msgType = TDMT_DND_CREATE_SNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndCreateSnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SMCreateSnodeMsg *pCreate) { + SSnodeObj snodeObj = {0}; + snodeObj.id = pDnode->id; + snodeObj.createdTime = taosGetTimestampMs(); + snodeObj.updateTime = snodeObj.createdTime; + + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + goto CREATE_SNODE_OVER; + } + mDebug("trans:%d, used to create snode:%d", pTrans->id, pCreate->dnodeId); + + if (mndSetCreateSnodeRedoLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + if (mndSetCreateSnodeCommitLogs(pTrans, &snodeObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + if (mndSetCreateSnodeRedoActions(pTrans, pDnode, &snodeObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto CREATE_SNODE_OVER; + } + + code = 0; + +CREATE_SNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMCreateSnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->dnodeId = htonl(pCreate->dnodeId); + + mDebug("snode:%d, start to create", pCreate->dnodeId); + + SSnodeObj *pObj = mndAcquireSnode(pMnode, pCreate->dnodeId); + if (pObj != NULL) { + mError("snode:%d, snode already exist", pObj->id); + mndReleaseSnode(pMnode, pObj); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + if (pDnode == NULL) { + mError("snode:%d, dnode not exist", pCreate->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndCreateSnode(pMnode, pMsg, pDnode, pCreate); + mndReleaseDnode(pMnode, pDnode); + + if (code != 0) { + mError("snode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pRedoRaw = mndSnodeActionEncode(pObj); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + return 0; +} + +static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { + SSdbRaw *pCommitRaw = mndSnodeActionEncode(pObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { + SDDropSnodeMsg *pMsg = malloc(sizeof(SDDropSnodeMsg)); + if (pMsg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pMsg->dnodeId = htonl(pDnode->id); + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + action.pCont = pMsg; + action.contLen = sizeof(SDDropSnodeMsg); + action.msgType = TDMT_DND_DROP_SNODE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pMsg); + return -1; + } + + return 0; +} + +static int32_t mndDropSnode(SMnode *pMnode, SMnodeMsg *pMsg, SSnodeObj *pObj) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); + if (pTrans == NULL) { + mError("snode:%d, failed to drop since %s", pObj->id, terrstr()); + goto DROP_SNODE_OVER; + } + + mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); + + if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + goto DROP_SNODE_OVER; + } + + code = 0; + +DROP_SNODE_OVER: + mndTransDrop(pTrans); + return code; +} + +static int32_t mndProcessDropSnodeReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMDropSnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("snode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("snode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SSnodeObj *pObj = mndAcquireSnode(pMnode, pDrop->dnodeId); + if (pObj == NULL) { + mError("snode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropSnode(pMnode, pMsg, pObj); + if (code != 0) { + mError("snode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pMnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateSnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndProcessDropSnodeRsp(SMnodeMsg *pMsg) { + mndTransProcessRsp(pMsg); + return 0; +} + +static int32_t mndGetSnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->pSchema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htonl(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_SNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveSnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SSnodeObj *pObj = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_SNODE, pShow->pIter, (void **)&pObj); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pObj->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pObj->pDnode->ep, pShow->bytes[cols]); + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pObj->createdTime; + cols++; + + numOfRows++; + sdbRelease(pSdb, pObj); + } + + mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextSnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 06e62d2528..78205d5f6a 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -273,7 +273,7 @@ static SArray *mndBuildDnodesArray(SMnode *pMnode) { static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) { float d1Score = (float)pDnode1->numOfVnodes / pDnode1->numOfSupportVnodes; float d2Score = (float)pDnode2->numOfVnodes / pDnode2->numOfSupportVnodes; - return d1Score > d2Score ? 0 : 1; + return d1Score > d2Score ? 1 : 0; } static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 84076f8c0a..e902e13a36 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -16,14 +16,16 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" #include "mndAuth.h" -#include "mndBalance.h" +#include "mndBnode.h" #include "mndCluster.h" #include "mndDb.h" #include "mndDnode.h" #include "mndFunc.h" #include "mndMnode.h" #include "mndProfile.h" +#include "mndQnode.h" #include "mndShow.h" +#include "mndSnode.h" #include "mndStb.h" #include "mndSync.h" #include "mndTelem.h" @@ -147,6 +149,9 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-trans", mndInitTrans, mndCleanupTrans) != 0) return -1; if (mndAllocStep(pMnode, "mnode-cluster", mndInitCluster, mndCleanupCluster) != 0) return -1; if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-qnode", mndInitSnode, mndCleanupSnode) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-qnode", mndInitBnode, mndCleanupBnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; @@ -162,7 +167,6 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-sdb-read", mndReadSdb, NULL) != 0) return -1; } if (mndAllocStep(pMnode, "mnode-timer", mndInitTimer, NULL) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-balance", mndInitBalance, mndCleanupBalance) != 0) return -1; if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sync", mndInitSync, mndCleanupSync) != 0) return -1; diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 78a90b9a7d..06c2563910 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -24,6 +24,12 @@ static const char *sdbTableName(ESdbType type) { return "cluster"; case SDB_MNODE: return "mnode"; + case SDB_QNODE: + return "qnode"; + case SDB_SNODE: + return "snode"; + case SDB_BNODE: + return "bnode"; case SDB_DNODE: return "dnode"; case SDB_USER: @@ -55,7 +61,8 @@ void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper) { } else if (keyType == SDB_KEY_INT32) { mTrace("%s:%d, refCount:%d oper:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj, pRow->refCount, oper); } else if (keyType == SDB_KEY_INT64) { - mTrace("%s:%" PRId64 ", refCount:%d oper:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, pRow->refCount, oper); + mTrace("%s:%" PRId64 ", refCount:%d oper:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, pRow->refCount, + oper); } else { } } diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index ff1a460228..8d2a72b268 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -24,12 +24,8 @@ void qndClose(SQnode *pQnode) { free(pQnode); } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } -int32_t qndProcessQueryReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int32_t qndProcessMsg(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { *pRsp = NULL; return 0; } -int32_t qndProcessFetchReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - *pRsp = NULL; - return 0; -} diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 7ae4d49059..5d21a037e7 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -24,7 +24,7 @@ void sndClose(SSnode *pSnode) { free(pSnode); } int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } -int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { *pRsp = NULL; return 0; } diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index ba0bd52e10..1504a69a08 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -37,6 +37,7 @@ typedef struct WriterCtx { struct { int fd; bool readOnly; + char buf[256]; } file; struct { int32_t capa; @@ -53,7 +54,7 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off static int writeCtxDoFlush(WriterCtx* ctx); WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity); -void writerCtxDestroy(WriterCtx* w); +void writerCtxDestroy(WriterCtx* w, bool remove); typedef uint32_t CheckSummer; diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index f97a3126c8..d8e69b5fde 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -107,6 +107,7 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName); +TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName); TFileReader* tfileReaderCreate(WriterCtx* ctx); void tfileReaderDestroy(TFileReader* reader); int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* result); @@ -114,10 +115,10 @@ void tfileReaderRef(TFileReader* reader); void tfileReaderUnRef(TFileReader* reader); TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const char* colName, uint8_t type); -void tfileWriteClose(TFileWriter* tw); +void tfileWriterClose(TFileWriter* tw); TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header); void tfileWriterDestroy(TFileWriter* tw); -int tfileWriterPut(TFileWriter* tw, void* data); +int tfileWriterPut(TFileWriter* tw, void* data, bool order); int tfileWriterFinish(TFileWriter* tw); // diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index b78c4ff258..6d64bdbd65 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -25,7 +25,7 @@ #endif #define INDEX_NUM_OF_THREADS 4 -#define INDEX_QUEUE_SIZE 4 +#define INDEX_QUEUE_SIZE 200 void* indexQhandle = NULL; @@ -52,18 +52,17 @@ typedef struct SIdxColInfo { static pthread_once_t isInit = PTHREAD_ONCE_INIT; // static void indexInit(); - static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result); static void indexInterResultsDestroy(SArray* results); static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult); +static int indexGenTFile(SIndex* index, IndexCache* cache, SArray* batch); + int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { // pthread_once(&isInit, indexInit); SIndex* sIdx = calloc(1, sizeof(SIndex)); - if (sIdx == NULL) { - return -1; - } + if (sIdx == NULL) { return -1; } #ifdef USE_LUCENE index_t* index = index_open(path); @@ -99,9 +98,7 @@ void indexClose(SIndex* sIdx) { void* iter = taosHashIterate(sIdx->colObj, NULL); while (iter) { IndexCache** pCache = iter; - if (*pCache) { - indexCacheUnRef(*pCache); - } + if (*pCache) { indexCacheUnRef(*pCache); } iter = taosHashIterate(sIdx->colObj, iter); } taosHashCleanup(sIdx->colObj); @@ -147,9 +144,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) { IndexCache** cache = taosHashGet(index->colObj, p->colName, p->nColName); assert(*cache != NULL); int ret = indexCachePut(*cache, p, uid); - if (ret != 0) { - return ret; - } + if (ret != 0) { return ret; } } #endif @@ -179,9 +174,7 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result int tsz = 0; index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); - for (int i = 0; i < tsz; i++) { - taosArrayPush(result, &tResult[i]); - } + for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); } for (int i = 0; i < nQuery; i++) { free(fields[i]); @@ -238,9 +231,7 @@ void indexOptsDestroy(SIndexOpts* opts) { */ SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); - if (p == NULL) { - return NULL; - } + if (p == NULL) { return NULL; } p->opera = opera; p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); return p; @@ -262,9 +253,7 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, int32_t nColName, const char* colVal, int32_t nColVal) { SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); - if (t == NULL) { - return NULL; - } + if (t == NULL) { return NULL; } t->suid = suid; t->operType = oper; @@ -317,7 +306,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result *result = taosArrayInit(4, sizeof(uint64_t)); // TODO: iterator mem and tidex - STermValueType s; + STermValueType s = kTypeValue; if (0 == indexCacheSearch(cache, query, *result, &s)) { if (s == kTypeDeletion) { indexInfo("col: %s already drop by other opera", term->colName); @@ -336,9 +325,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result return 0; } static void indexInterResultsDestroy(SArray* results) { - if (results == NULL) { - return; - } + if (results == NULL) { return; } size_t sz = taosArrayGetSize(results); for (size_t i = 0; i < sz; i++) { @@ -368,23 +355,43 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType return 0; } -int indexFlushCacheTFile(SIndex* sIdx, void* cache) { - if (sIdx == NULL) { - return -1; +static void indexMergeSameKey(SArray* result, TFileValue* tv) { + int32_t sz = result ? taosArrayGetSize(result) : 0; + if (sz > 0) { + // TODO(yihao): remove duplicate tableid + TFileValue* lv = taosArrayGetP(result, sz - 1); + if (strcmp(lv->colVal, tv->colVal) == 0) { + taosArrayAddAll(lv->tableId, tv->tableId); + tfileValueDestroy(tv); + } else { + taosArrayPush(result, &tv); + } + } else { + taosArrayPush(result, &tv); } +} +static void indexDestroyTempResult(SArray* result) { + int32_t sz = result ? taosArrayGetSize(result) : 0; + for (size_t i = 0; i < sz; i++) { + TFileValue* tv = taosArrayGetP(result, i); + tfileValueDestroy(tv); + } + taosArrayDestroy(result); +} +int indexFlushCacheTFile(SIndex* sIdx, void* cache) { + if (sIdx == NULL) { return -1; } indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); IndexCache* pCache = (IndexCache*)cache; TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->colName); - // handle flush Iterate* cacheIter = indexCacheIteratorCreate(pCache); Iterate* tfileIter = tfileIteratorCreate(pReader); SArray* result = taosArrayInit(1024, sizeof(void*)); - bool cn = cacheIter->next(cacheIter); - bool tn = tfileIter->next(tfileIter); + bool cn = cacheIter ? cacheIter->next(cacheIter) : false; + bool tn = tfileIter ? tfileIter->next(tfileIter) : false; while (cn == true && tn == true) { IterateValue* cv = cacheIter->getValue(cacheIter); IterateValue* tv = tfileIter->getValue(tfileIter); @@ -395,7 +402,7 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { TFileValue* tfv = tfileValueCreate(cv->colVal); taosArrayAddAll(tfv->tableId, cv->val); taosArrayAddAll(tfv->tableId, tv->val); - taosArrayPush(result, &tfv); + indexMergeSameKey(result, tfv); cn = cacheIter->next(cacheIter); tn = tfileIter->next(tfileIter); @@ -403,13 +410,15 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { } else if (comp < 0) { TFileValue* tfv = tfileValueCreate(cv->colVal); taosArrayAddAll(tfv->tableId, cv->val); - taosArrayPush(result, &tfv); + + indexMergeSameKey(result, tfv); // copy to final Result; cn = cacheIter->next(cacheIter); } else { TFileValue* tfv = tfileValueCreate(tv->colVal); - taosArrayPush(result, &tfv); taosArrayAddAll(tfv->tableId, tv->val); + + indexMergeSameKey(result, tfv); // copy to final result tn = tfileIter->next(tfileIter); } @@ -418,33 +427,24 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { IterateValue* cv = cacheIter->getValue(cacheIter); TFileValue* tfv = tfileValueCreate(cv->colVal); taosArrayAddAll(tfv->tableId, cv->val); - taosArrayPush(result, &tfv); + indexMergeSameKey(result, tfv); cn = cacheIter->next(cacheIter); } while (tn == true) { IterateValue* tv = tfileIter->getValue(tfileIter); TFileValue* tfv = tfileValueCreate(tv->colVal); + if (tv->val == NULL) { + // HO + printf("NO...."); + } taosArrayAddAll(tfv->tableId, tv->val); - taosArrayPush(result, &tfv); + indexMergeSameKey(result, tfv); tn = tfileIter->next(tfileIter); } - - int32_t version = CACHE_VERSION(pCache); - uint8_t colType = pCache->type; - - TFileWriter* tw = tfileWriterOpen(sIdx->path, sIdx->suid, version, pCache->colName, colType); - if (tw == NULL) { - indexError("faile to open file to write"); - } else { - int ret = tfileWriterPut(tw, result); - if (ret != 0) { - indexError("faile to write into tindex "); - } - } - // not free later, just put int table cache + int ret = indexGenTFile(sIdx, pCache, result); + indexDestroyTempResult(result); indexCacheDestroyImm(pCache); - tfileWriteClose(tw); indexCacheIteratorDestroy(cacheIter); tfileIteratorDestroy(tfileIter); @@ -455,9 +455,45 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) { void iterateValueDestroy(IterateValue* value, bool destroy) { if (destroy) { taosArrayDestroy(value->val); + value->val = NULL; } else { - taosArrayClear(value->val); + if (value->val != NULL) { taosArrayClear(value->val); } } - free(value->colVal); + // free(value->colVal); value->colVal = NULL; } +static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) { + int32_t version = CACHE_VERSION(cache); + uint8_t colType = cache->type; + + TFileWriter* tw = tfileWriterOpen(sIdx->path, sIdx->suid, version, cache->colName, colType); + if (tw == NULL) { + indexError("failed to open file to write"); + return -1; + } + + int ret = tfileWriterPut(tw, batch, true); + if (ret != 0) { + indexError("failed to write into tindex "); + goto END; + } + tfileWriterClose(tw); + + TFileReader* reader = tfileReaderOpen(sIdx->path, sIdx->suid, version, cache->colName); + + char buf[128] = {0}; + TFileHeader* header = &reader->header; + TFileCacheKey key = {.suid = header->suid, + .colName = header->colName, + .nColName = strlen(header->colName), + .colType = header->colType}; + pthread_mutex_lock(&sIdx->mtx); + + IndexTFile* ifile = (IndexTFile*)sIdx->tindex; + tfileCachePut(ifile->cache, &key, reader); + + pthread_mutex_unlock(&sIdx->mtx); + return ret; +END: + tfileWriterClose(tw); +} diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 217545d23b..882091afb9 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -20,7 +20,7 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later -#define MEM_TERM_LIMIT 1000000 +#define MEM_TERM_LIMIT 200 // ref index_cache.h:22 //#define CACHE_KEY_LEN(p) \ // (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + @@ -66,19 +66,43 @@ void indexCacheDebug(IndexCache* cache) { indexMemRef(tbl); pthread_mutex_unlock(&cache->mtx); - SSkipList* slt = tbl->mem; - SSkipListIterator* iter = tSkipListCreateIter(slt); - while (tSkipListIterNext(iter)) { - SSkipListNode* node = tSkipListIterGet(iter); - CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); - if (ct != NULL) { - // TODO, add more debug info - indexInfo("{colVal: %s, version: %d} \t", ct->colVal, ct->version); + { + SSkipList* slt = tbl->mem; + SSkipListIterator* iter = tSkipListCreateIter(slt); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); + if (ct != NULL) { + // TODO, add more debug info + indexInfo("{colVal: %s, version: %d} \t", ct->colVal, ct->version); + } } - } - tSkipListDestroyIter(iter); + tSkipListDestroyIter(iter); - indexMemUnRef(tbl); + indexMemUnRef(tbl); + } + + { + pthread_mutex_lock(&cache->mtx); + tbl = cache->imm; + indexMemRef(tbl); + pthread_mutex_unlock(&cache->mtx); + if (tbl != NULL) { + SSkipList* slt = tbl->mem; + SSkipListIterator* iter = tSkipListCreateIter(slt); + while (tSkipListIterNext(iter)) { + SSkipListNode* node = tSkipListIterGet(iter); + CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); + if (ct != NULL) { + // TODO, add more debug info + indexInfo("{colVal: %s, version: %d} \t", ct->colVal, ct->version); + } + } + tSkipListDestroyIter(iter); + } + + indexMemUnRef(tbl); + } } void indexCacheDestroySkiplist(SSkipList* slt) { @@ -86,25 +110,26 @@ void indexCacheDestroySkiplist(SSkipList* slt) { while (tSkipListIterNext(iter)) { SSkipListNode* node = tSkipListIterGet(iter); CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); - if (ct != NULL) { - } + if (ct != NULL) {} } tSkipListDestroyIter(iter); tSkipListDestroy(slt); } void indexCacheDestroyImm(IndexCache* cache) { + if (cache == NULL) { return; } + MemTable* tbl = NULL; pthread_mutex_lock(&cache->mtx); tbl = cache->imm; cache->imm = NULL; // or throw int bg thread pthread_mutex_unlock(&cache->mtx); + + indexMemUnRef(tbl); indexMemUnRef(tbl); } void indexCacheDestroy(void* cache) { IndexCache* pCache = cache; - if (pCache == NULL) { - return; - } + if (pCache == NULL) { return; } indexMemUnRef(pCache->mem); indexMemUnRef(pCache->imm); free(pCache->colName); @@ -114,9 +139,11 @@ void indexCacheDestroy(void* cache) { Iterate* indexCacheIteratorCreate(IndexCache* cache) { Iterate* iiter = calloc(1, sizeof(Iterate)); - if (iiter == NULL) { - return NULL; - } + if (iiter == NULL) { return NULL; } + + pthread_mutex_lock(&cache->mtx); + + indexMemRef(cache->imm); MemTable* tbl = cache->imm; iiter->val.val = taosArrayInit(1, sizeof(uint64_t)); @@ -124,12 +151,12 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) { iiter->next = indexCacheIteratorNext; iiter->getValue = indexCacheIteratorGetValue; + pthread_mutex_unlock(&cache->mtx); + return iiter; } void indexCacheIteratorDestroy(Iterate* iter) { - if (iter == NULL) { - return; - } + if (iter == NULL) { return; } tSkipListDestroyIter(iter->iter); iterateValueDestroy(&iter->val, true); free(iter); @@ -155,6 +182,7 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { taosMsleep(50); pthread_mutex_lock(&cache->mtx); } else { + indexCacheRef(cache); cache->imm = cache->mem; cache->mem = indexInternalCacheCreate(cache->type); cache->nTerm = 1; @@ -166,17 +194,13 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) { } int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { - if (cache == NULL) { - return -1; - } + if (cache == NULL) { return -1; } IndexCache* pCache = cache; indexCacheRef(pCache); // encode data CacheTerm* ct = calloc(1, sizeof(CacheTerm)); - if (cache == NULL) { - return -1; - } + if (cache == NULL) { return -1; } // set up key ct->colType = term->colType; ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); @@ -205,32 +229,11 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u IndexCache* pCache = cache; return 0; } -int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { - if (cache == NULL) { - return -1; - } - IndexCache* pCache = cache; - SIndexTerm* term = query->term; - EIndexQueryType qtype = query->qType; - - MemTable *mem = NULL, *imm = NULL; - pthread_mutex_lock(&pCache->mtx); - mem = pCache->mem; - imm = pCache->imm; - indexMemRef(mem); - indexMemRef(imm); - pthread_mutex_unlock(&pCache->mtx); - - CacheTerm* ct = calloc(1, sizeof(CacheTerm)); - if (ct == NULL) { - return -1; - } - ct->colVal = calloc(1, sizeof(char) * (term->nColVal + 1)); - memcpy(ct->colVal, term->colVal, term->nColVal); - ct->version = atomic_load_32(&pCache->version); +static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) { + if (mem == NULL) { return 0; } char* key = getIndexKey(ct); - // TODO handle multi situation later, and refactor + SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); while (tSkipListIterNext(iter)) { SSkipListNode* node = tSkipListIterGet(iter); @@ -251,51 +254,56 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV } } tSkipListDestroyIter(iter); - cacheTermDestroy(ct); - // int32_t keyLen = CACHE_KEY_LEN(term); - // char* buf = calloc(1, keyLen); - if (qtype == QUERY_TERM) { - // - } else if (qtype == QUERY_PREFIX) { - // - } else if (qtype == QUERY_SUFFIX) { - // - } else if (qtype == QUERY_REGEX) { - // + return 0; +} +int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { + if (cache == NULL) { return -1; } + IndexCache* pCache = cache; + + MemTable *mem = NULL, *imm = NULL; + pthread_mutex_lock(&pCache->mtx); + mem = pCache->mem; + imm = pCache->imm; + indexMemRef(mem); + indexMemRef(imm); + pthread_mutex_unlock(&pCache->mtx); + + SIndexTerm* term = query->term; + EIndexQueryType qtype = query->qType; + CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; + indexCacheDebug(pCache); + + int ret = indexQueryMem(mem, &ct, qtype, result, s); + if (ret == 0 && *s != kTypeDeletion) { + // continue search in imm + ret = indexQueryMem(imm, &ct, qtype, result, s); } + // cacheTermDestroy(ct); + indexMemUnRef(mem); indexMemUnRef(imm); - return 0; + + return ret; } void indexCacheRef(IndexCache* cache) { - if (cache == NULL) { - return; - } + if (cache == NULL) { return; } int ref = T_REF_INC(cache); UNUSED(ref); } void indexCacheUnRef(IndexCache* cache) { - if (cache == NULL) { - return; - } + if (cache == NULL) { return; } int ref = T_REF_DEC(cache); - if (ref == 0) { - indexCacheDestroy(cache); - } + if (ref == 0) { indexCacheDestroy(cache); } } void indexMemRef(MemTable* tbl) { - if (tbl == NULL) { - return; - } + if (tbl == NULL) { return; } int ref = T_REF_INC(tbl); UNUSED(ref); } void indexMemUnRef(MemTable* tbl) { - if (tbl == NULL) { - return; - } + if (tbl == NULL) { return; } int ref = T_REF_DEC(tbl); if (ref == 0) { SSkipList* slt = tbl->mem; @@ -305,9 +313,7 @@ void indexMemUnRef(MemTable* tbl) { } static void cacheTermDestroy(CacheTerm* ct) { - if (ct == NULL) { - return; - } + if (ct == NULL) { return; } free(ct->colVal); free(ct); } @@ -322,9 +328,7 @@ static int32_t compareKey(const void* l, const void* r) { // compare colVal int32_t cmp = strcmp(lt->colVal, rt->colVal); - if (cmp == 0) { - return rt->version - lt->version; - } + if (cmp == 0) { return rt->version - lt->version; } return cmp; } @@ -344,9 +348,7 @@ static void doMergeWork(SSchedMsg* msg) { } static bool indexCacheIteratorNext(Iterate* itera) { SSkipListIterator* iter = itera->iter; - if (iter == NULL) { - return false; - } + if (iter == NULL) { return false; } IterateValue* iv = &itera->val; iterateValueDestroy(iv, false); @@ -356,7 +358,8 @@ static bool indexCacheIteratorNext(Iterate* itera) { CacheTerm* ct = (CacheTerm*)SL_GET_NODE_DATA(node); iv->type = ct->operaType; - iv->colVal = ct->colVal; + iv->colVal = calloc(1, strlen(ct->colVal) + 1); + memcpy(iv->colVal, ct->colVal, strlen(ct->colVal)); taosArrayPush(iv->val, &ct->uid); } diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 72983809d1..590ff294bf 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -17,7 +17,9 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { return NULL; } + if (nsv == NULL) { + return NULL; + } nsv->kind = kind; nsv->type = ty; @@ -35,7 +37,9 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueTyp } void startWithStateValueDestroy(void* val) { StartWithStateValue* sv = (StartWithStateValue*)val; - if (sv == NULL) { return; } + if (sv == NULL) { + return; + } if (sv->type == FST_INT) { // @@ -48,7 +52,9 @@ void startWithStateValueDestroy(void* val) { } StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { return NULL; } + if (nsv == NULL) { + return NULL; + } nsv->kind = sv->kind; nsv->type = sv->type; @@ -65,24 +71,12 @@ StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { } // iterate fst -static void* alwaysMatchStart(AutomationCtx* ctx) { - return NULL; -} -static bool alwaysMatchIsMatch(AutomationCtx* ctx, void* state) { - return true; -} -static bool alwaysMatchCanMatch(AutomationCtx* ctx, void* state) { - return true; -} -static bool alwaysMatchWillAlwaysMatch(AutomationCtx* ctx, void* state) { - return true; -} -static void* alwaysMatchAccpet(AutomationCtx* ctx, void* state, uint8_t byte) { - return NULL; -} -static void* alwaysMatchAccpetEof(AutomationCtx* ctx, void* state) { - return NULL; -} +static void* alwaysMatchStart(AutomationCtx* ctx) { return NULL; } +static bool alwaysMatchIsMatch(AutomationCtx* ctx, void* state) { return true; } +static bool alwaysMatchCanMatch(AutomationCtx* ctx, void* state) { return true; } +static bool alwaysMatchWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } +static void* alwaysMatchAccpet(AutomationCtx* ctx, void* state, uint8_t byte) { return NULL; } +static void* alwaysMatchAccpetEof(AutomationCtx* ctx, void* state) { return NULL; } // prefix query, impl later static void* prefixStart(AutomationCtx* ctx) { @@ -97,17 +91,20 @@ static bool prefixCanMatch(AutomationCtx* ctx, void* sv) { StartWithStateValue* ssv = (StartWithStateValue*)sv; return ssv->val >= 0; } -static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { - return true; -} +static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { StartWithStateValue* ssv = (StartWithStateValue*)state; - if (ssv == NULL || ctx == NULL) { return NULL; } + if (ssv == NULL || ctx == NULL) { + return NULL; + } char* data = ctx->data; - if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); } + if (ssv->kind == Done) { + return startWithStateValueCreate(Done, FST_INT, &ssv->val); + } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { - int val = ssv->val + 1; + int val = ssv->val + 1; + StartWithStateValue* nsv = startWithStateValueCreate(Running, FST_INT, &val); if (prefixIsMatch(ctx, nsv)) { nsv->kind = Done; @@ -118,35 +115,22 @@ static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { } return NULL; } -static void* prefixAcceptEof(AutomationCtx* ctx, void* state) { - return NULL; -} +static void* prefixAcceptEof(AutomationCtx* ctx, void* state) { return NULL; } // pattern query, impl later -static void* patternStart(AutomationCtx* ctx) { - return NULL; -} -static bool patternIsMatch(AutomationCtx* ctx, void* data) { - return true; -} -static bool patternCanMatch(AutomationCtx* ctx, void* data) { - return true; -} -static bool patternWillAlwaysMatch(AutomationCtx* ctx, void* state) { - return true; -} +static void* patternStart(AutomationCtx* ctx) { return NULL; } +static bool patternIsMatch(AutomationCtx* ctx, void* data) { return true; } +static bool patternCanMatch(AutomationCtx* ctx, void* data) { return true; } +static bool patternWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } -static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) { - return NULL; -} +static void* patternAccept(AutomationCtx* ctx, void* state, uint8_t byte) { return NULL; } -static void* patternAcceptEof(AutomationCtx* ctx, void* state) { - return NULL; -} +static void* patternAcceptEof(AutomationCtx* ctx, void* state) { return NULL; } AutomationFunc automFuncs[] = { - {alwaysMatchStart, alwaysMatchIsMatch, alwaysMatchCanMatch, alwaysMatchWillAlwaysMatch, alwaysMatchAccpet, alwaysMatchAccpetEof}, + {alwaysMatchStart, alwaysMatchIsMatch, alwaysMatchCanMatch, alwaysMatchWillAlwaysMatch, alwaysMatchAccpet, + alwaysMatchAccpetEof}, {prefixStart, prefixIsMatch, prefixCanMatch, prefixWillAlwaysMatch, prefixAccept, prefixAcceptEof}, {patternStart, patternIsMatch, patternCanMatch, patternWillAlwaysMatch, patternAccept, patternAcceptEof} // add more search type @@ -154,7 +138,9 @@ AutomationFunc automFuncs[] = { AutomationCtx* automCtxCreate(void* data, AutomationType atype) { AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); - if (ctx == NULL) { return NULL; } + if (ctx == NULL) { + return NULL; + } StartWithStateValue* sv = NULL; if (atype == AUTOMATION_ALWAYS) { @@ -170,11 +156,14 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { // add more search type } - char* src = (char*)data; - size_t len = strlen(src); - char* dst = (char*)malloc(len * sizeof(char) + 1); - memcpy(dst, src, len); - dst[len] = 0; + char* dst = NULL; + if (data != NULL) { + char* src = (char*)data; + size_t len = strlen(src); + dst = (char*)malloc(len * sizeof(char) + 1); + memcpy(dst, src, len); + dst[len] = 0; + } ctx->data = dst; ctx->type = atype; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index c8fbdd7690..710db563d9 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -52,7 +52,7 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off } static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { - // tfFsync(ctx->fd); + tfFsync(ctx->file.fd); // tfFlush(ctx->file.fd); } else { // do nothing @@ -73,6 +73,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int } else { ctx->file.fd = tfOpenReadWrite(path); } + memcpy(ctx->file.buf, path, strlen(path)); if (ctx->file.fd < 0) { indexError("open file error %d", errno); goto END; @@ -95,11 +96,15 @@ END: free(ctx); return NULL; } -void writerCtxDestroy(WriterCtx* ctx) { +void writerCtxDestroy(WriterCtx* ctx, bool remove) { if (ctx->type == TMemory) { free(ctx->mem.buf); } else { tfClose(ctx->file.fd); + if (remove) { + indexError("rm file %s", ctx->file.buf); + unlink(ctx->file.buf); + } } free(ctx); } @@ -138,10 +143,8 @@ int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) return nRead; } -uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { - return 0; -} -int fstCountingWriterFlush(FstCountingWriter* write) { +uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) { return 0; } +int fstCountingWriterFlush(FstCountingWriter* write) { WriterCtx* ctx = write->wrt; ctx->flush(ctx); // write->wtr->flush diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index 271528a437..17322e301e 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -32,6 +32,7 @@ typedef struct TFileFstIter { #define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t)) +static int tfileUidCompare(const void* a, const void* b); static int tfileStrCompare(const void* a, const void* b); static int tfileValueCompare(const void* a, const void* b, const void* param); static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds); @@ -52,11 +53,16 @@ static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version); static void tfileSerialCacheKey(TFileCacheKey* key, char* buf); +static TFileReader* tfileReaderCreateImpl(WriterCtx* ctx) { + TFileReader* reader = tfileReaderCreate(ctx); + tfileReaderRef(reader); + // tfileSerialCacheKey(&key, buf); + return reader; +} + TFileCache* tfileCacheCreate(const char* path) { TFileCache* tcache = calloc(1, sizeof(TFileCache)); - if (tcache == NULL) { - return NULL; - } + if (tcache == NULL) { return NULL; } tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->capacity = 64; @@ -75,22 +81,19 @@ TFileCache* tfileCacheCreate(const char* path) { continue; } - WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 64); + WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 1024 * 64); if (wc == NULL) { indexError("failed to open index:%s", file); goto End; } - TFileReader* reader = tfileReaderCreate(wc); - tfileReaderRef(reader); - // loader fst and validate it + char buf[128] = {0}; + TFileReader* reader = tfileReaderCreateImpl(wc); TFileHeader* header = &reader->header; TFileCacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType}; - - char buf[128] = {0}; tfileSerialCacheKey(&key, buf); taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*)); } @@ -102,9 +105,7 @@ End: return NULL; } void tfileCacheDestroy(TFileCache* tcache) { - if (tcache == NULL) { - return; - } + if (tcache == NULL) { return; } // free table cache TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); @@ -125,9 +126,7 @@ TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) { tfileSerialCacheKey(key, buf); TFileReader** reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); - if (reader == NULL) { - return NULL; - } + if (reader == NULL) { return NULL; } tfileReaderRef(*reader); return *reader; @@ -137,7 +136,7 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) tfileSerialCacheKey(key, buf); // remove last version index reader TFileReader** p = taosHashGet(tcache->tableCache, buf, strlen(buf)); - if (*p != NULL) { + if (p != NULL) { TFileReader* oldReader = *p; taosHashRemove(tcache->tableCache, buf, strlen(buf)); tfileReaderUnRef(oldReader); @@ -149,9 +148,7 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader) } TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* reader = calloc(1, sizeof(TFileReader)); - if (reader == NULL) { - return NULL; - } + if (reader == NULL) { return NULL; } // T_REF_INC(reader); reader->ctx = ctx; @@ -172,12 +169,10 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) { return reader; } void tfileReaderDestroy(TFileReader* reader) { - if (reader == NULL) { - return; - } + if (reader == NULL) { return; } // T_REF_INC(reader); fstDestroy(reader->fst); - writerCtxDestroy(reader->ctx); + writerCtxDestroy(reader->ctx, true); free(reader); } @@ -216,7 +211,8 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c char fullname[256] = {0}; snprintf(fullname, sizeof(fullname), "%s/%s", path, filename); - WriterCtx* wcx = writerCtxCreate(TFile, fullname, true, 1024 * 1024); + WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64); + if (wcx == NULL) { return NULL; } TFileHeader tfh = {0}; tfh.suid = suid; @@ -226,6 +222,21 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c return tfileWriterCreate(wcx, &tfh); } +TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const char* colName) { + char filename[128] = {0}; + int32_t coldId = 1; + tfileGenFileName(filename, suid, coldId, version); + + char fullname[256] = {0}; + snprintf(fullname, sizeof(fullname), "%s/%s", path, filename); + WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); + if (wc == NULL) { return NULL; } + + TFileReader* reader = tfileReaderCreateImpl(wc); + return reader; + + // tfileSerialCacheKey(&key, buf); +} TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { // char pathBuf[128] = {0}; // sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version); @@ -249,28 +260,29 @@ TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { return tw; } -int tfileWriterPut(TFileWriter* tw, void* data) { +int tfileWriterPut(TFileWriter* tw, void* data, bool order) { // sort by coltype and write to tindex - __compar_fn_t fn; - - int8_t colType = tw->header.colType; - if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { - fn = tfileStrCompare; - } else { - fn = getComparFunc(colType, 0); + if (order == false) { + __compar_fn_t fn; + int8_t colType = tw->header.colType; + if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) { + fn = tfileStrCompare; + } else { + fn = getComparFunc(colType, 0); + } + taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); } - taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn); - int32_t bufLimit = 4096, offset = 0; - char* buf = calloc(1, sizeof(char) * bufLimit); - char* p = buf; + int32_t bufLimit = 64 * 4096, offset = 0; + // char* buf = calloc(1, sizeof(char) * bufLimit); + // char* p = buf; int32_t sz = taosArrayGetSize((SArray*)data); int32_t fstOffset = tw->offset; // ugly code, refactor later for (size_t i = 0; i < sz; i++) { TFileValue* v = taosArrayGetP((SArray*)data, i); - + // taosArrayRemoveDuplicate(v->tablId, tfileUidCompare, NULL); int32_t tbsz = taosArrayGetSize(v->tableId); fstOffset += TF_TABLE_TATOAL_SIZE(tbsz); } @@ -282,30 +294,39 @@ int tfileWriterPut(TFileWriter* tw, void* data) { int32_t tbsz = taosArrayGetSize(v->tableId); // check buf has enough space or not int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); - if (offset + ttsz > bufLimit) { - // batch write - tw->ctx->write(tw->ctx, buf, offset); - offset = 0; - memset(buf, 0, bufLimit); - p = buf; - } + // if (offset + ttsz >= bufLimit) { + // // batch write + // indexInfo("offset: %d, ttsz: %d", offset, ttsz); + // // std::cout << "offset: " << offset << std::endl; + // // std::cout << "ttsz:" << ttsz < < < std::endl; + // tw->ctx->write(tw->ctx, buf, offset); + // offset = 0; + // memset(buf, 0, bufLimit); + // p = buf; + //} + // if (ttsz >= bufLimit) { + //} + char* buf = calloc(1, ttsz * sizeof(char)); + char* p = buf; tfileSerialTableIdsToBuf(p, v->tableId); - offset += ttsz; - p = buf + offset; + tw->ctx->write(tw->ctx, buf, ttsz); + // offset += ttsz; + // p = buf + offset; // set up value offset v->offset = tw->offset; tw->offset += ttsz; + free(buf); } - if (offset != 0) { - // write reversed data in buf to tindex - tw->ctx->write(tw->ctx, buf, offset); - } - tfree(buf); + // if (offset != 0) { + // write reversed data in buf to tindex + // tw->ctx->write(tw->ctx, buf, offset); + //} + // tfree(buf); tw->fb = fstBuilderCreate(tw->ctx, 0); if (tw->fb == NULL) { - tfileWriterDestroy(tw); + tfileWriterClose(tw); return -1; } // write fst @@ -321,27 +342,20 @@ int tfileWriterPut(TFileWriter* tw, void* data) { tw->fb = NULL; return 0; } -void tfileWriteClose(TFileWriter* tw) { - if (tw == NULL) { - return; - } - writerCtxDestroy(tw->ctx); +void tfileWriterClose(TFileWriter* tw) { + if (tw == NULL) { return; } + writerCtxDestroy(tw->ctx, false); free(tw); } void tfileWriterDestroy(TFileWriter* tw) { - if (tw == NULL) { - return; - } - - writerCtxDestroy(tw->ctx); + if (tw == NULL) { return; } + writerCtxDestroy(tw->ctx, false); free(tw); } IndexTFile* indexTFileCreate(const char* path) { IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); - if (tfile == NULL) { - return NULL; - } + if (tfile == NULL) { return NULL; } tfile->cache = tfileCacheCreate(path); return tfile; @@ -350,18 +364,14 @@ void IndexTFileDestroy(IndexTFile* tfile) { free(tfile); } int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int ret = -1; - if (tfile == NULL) { - return ret; - } + if (tfile == NULL) { return ret; } IndexTFile* pTfile = (IndexTFile*)tfile; SIndexTerm* term = query->term; TFileCacheKey key = { .suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; TFileReader* reader = tfileCacheGet(pTfile->cache, &key); - if (reader == NULL) { - return 0; - } + if (reader == NULL) { return 0; } return tfileReaderSearch(reader, query, result); } @@ -381,9 +391,7 @@ static bool tfileIteratorNext(Iterate* iiter) { TFileFstIter* tIter = iiter->iter; StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL); - if (rt == NULL) { - return false; - } + if (rt == NULL) { return false; } int32_t sz = 0; char* ch = (char*)fstSliceData(&rt->data, &sz); @@ -391,12 +399,9 @@ static bool tfileIteratorNext(Iterate* iiter) { memcpy(colVal, ch, sz); offset = (uint64_t)(rt->out.out); - swsResultDestroy(rt); // set up iterate value - if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { - return false; - } + if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } iv->colVal = colVal; @@ -407,9 +412,8 @@ static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { TFileFstIter* tIter = calloc(1, sizeof(Iterate)); - if (tIter == NULL) { - return NULL; - } + if (tIter == NULL) { return NULL; } + tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); tIter->fb = fstSearch(reader->fst, tIter->ctx); tIter->st = streamBuilderIntoStream(tIter->fb); @@ -418,21 +422,22 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { } Iterate* tfileIteratorCreate(TFileReader* reader) { - Iterate* iter = calloc(1, sizeof(Iterate)); + if (reader == NULL) { return NULL; } + Iterate* iter = calloc(1, sizeof(Iterate)); iter->iter = tfileFstIteratorCreate(reader); if (iter->iter == NULL) { + free(iter); return NULL; } - iter->next = tfileIteratorNext; iter->getValue = tifileIterateGetValue; + iter->val.val = taosArrayInit(1, sizeof(uint64_t)); return iter; } void tfileIteratorDestroy(Iterate* iter) { - if (iter == NULL) { - return; - } + if (iter == NULL) { return; } + IterateValue* iv = &iter->val; iterateValueDestroy(iv, true); @@ -445,18 +450,19 @@ void tfileIteratorDestroy(Iterate* iter) { } TFileReader* tfileGetReaderByCol(IndexTFile* tf, char* colName) { - if (tf == NULL) { - return NULL; - } + if (tf == NULL) { return NULL; } TFileCacheKey key = {.suid = 0, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; return tfileCacheGet(tf->cache, &key); } +static int tfileUidCompare(const void* a, const void* b) { + uint64_t l = *(uint64_t*)a; + uint64_t r = *(uint64_t*)b; + return l - r; +} static int tfileStrCompare(const void* a, const void* b) { int ret = strcmp((char*)a, (char*)b); - if (ret == 0) { - return ret; - } + if (ret == 0) { return ret; } return ret < 0 ? -1 : 1; } @@ -471,17 +477,13 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) { TFileValue* tfileValueCreate(char* val) { TFileValue* tf = calloc(1, sizeof(TFileValue)); - if (tf == NULL) { - return NULL; - } - + if (tf == NULL) { return NULL; } + tf->colVal = val; tf->tableId = taosArrayInit(32, sizeof(uint64_t)); return tf; } int tfileValuePush(TFileValue* tf, uint64_t val) { - if (tf == NULL) { - return -1; - } + if (tf == NULL) { return -1; } taosArrayPush(tf->tableId, &val); return 0; } @@ -501,9 +503,7 @@ static void tfileSerialTableIdsToBuf(char* buf, SArray* ids) { static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) { int32_t fstOffset = offset + sizeof(tw->header.fstOffset); tw->header.fstOffset = fstOffset; - if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { - return -1; - } + if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } tw->offset += sizeof(fstOffset); return 0; } @@ -514,9 +514,7 @@ static int tfileWriteHeader(TFileWriter* writer) { memcpy(buf, (char*)header, sizeof(buf)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); - if (sizeof(buf) != nwrite) { - return -1; - } + if (sizeof(buf) != nwrite) { return -1; } writer->offset = nwrite; return 0; } @@ -547,12 +545,10 @@ static int tfileReaderLoadHeader(TFileReader* reader) { } static int tfileReaderLoadFst(TFileReader* reader) { // current load fst into memory, refactor it later - static int FST_MAX_SIZE = 16 * 1024; + static int FST_MAX_SIZE = 64 * 1024; char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); - if (buf == NULL) { - return -1; - } + if (buf == NULL) { return -1; } WriterCtx* ctx = reader->ctx; int32_t nread = ctx->readFrom(ctx, buf, FST_MAX_SIZE, reader->header.fstOffset); @@ -575,36 +571,33 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* int32_t total = sizeof(uint64_t) * nid; char* buf = calloc(1, total); - if (buf == NULL) { - return -1; - } + if (buf == NULL) { return -1; } nread = ctx->read(ctx, buf, total); assert(total == nread); - for (int32_t i = 0; i < nid; i++) { - taosArrayPush(result, (uint64_t*)buf + i); - } + for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); } free(buf); return 0; } void tfileReaderRef(TFileReader* reader) { + if (reader == NULL) { return; } int ref = T_REF_INC(reader); UNUSED(ref); } void tfileReaderUnRef(TFileReader* reader) { + if (reader == NULL) { return; } int ref = T_REF_DEC(reader); if (ref == 0) { + // do nothing tfileReaderDestroy(reader); } } static int tfileGetFileList(const char* path, SArray* result) { DIR* dir = opendir(path); - if (NULL == dir) { - return -1; - } + if (NULL == dir) { return -1; } struct dirent* entry; while ((entry = readdir(dir)) != NULL) { @@ -632,9 +625,7 @@ static int tfileCompare(const void* a, const void* b) { size_t bLen = strlen(bName); int ret = strncmp(aName, bName, aLen > bLen ? aLen : bLen); - if (ret == 0) { - return ret; - } + if (ret == 0) { return ret; } return ret < 0 ? -1 : 1; } // tfile name suid-colId-version.tindex diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 85bb5e2b15..a4d8bb36f1 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -12,9 +12,7 @@ #include "tskiplist.h" #include "tutil.h" -void* callback(void* s) { - return s; -} +void* callback(void* s) { return s; } static std::string fileName = "/tmp/tindex.tindex"; class FstWriter { @@ -34,7 +32,7 @@ class FstWriter { fstBuilderFinish(_b); fstBuilderDestroy(_b); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, false); } private: @@ -102,7 +100,7 @@ class FstReadMemory { fstCountingWriterDestroy(_w); fstDestroy(_fst); fstSliceDestroy(&_s); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, false); } private: diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index b3e385192f..588205c67f 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -57,7 +57,7 @@ class FstWriter { fstBuilderFinish(_b); fstBuilderDestroy(_b); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, false); } private: @@ -76,9 +76,7 @@ class FstReadMemory { bool init() { char* buf = (char*)calloc(1, sizeof(char) * _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); - if (nRead <= 0) { - return false; - } + if (nRead <= 0) { return false; } _size = nRead; _s = fstSliceCreate((uint8_t*)buf, _size); _fst = fstCreate(&_s); @@ -104,9 +102,7 @@ class FstReadMemory { StreamWithState* st = streamBuilderIntoStream(sb); StreamWithStateResult* rt = NULL; - while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { - result.push_back((uint64_t)(rt->out.out)); - } + while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { result.push_back((uint64_t)(rt->out.out)); } return true; } bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector& result) { @@ -120,7 +116,7 @@ class FstReadMemory { fstCountingWriterDestroy(_w); fstDestroy(_fst); fstSliceDestroy(&_s); - writerCtxDestroy(_wc); + writerCtxDestroy(_wc, true); } private: @@ -182,9 +178,7 @@ void checkFstPerf() { delete fw; FstReadMemory* m = new FstReadMemory(1024 * 64); - if (m->init()) { - printf("success to init fst read"); - } + if (m->init()) { printf("success to init fst read"); } Performance_fstReadRecords(m); delete m; } @@ -346,10 +340,8 @@ class TFileObj { tfileReaderDestroy(reader_); reader_ = NULL; } - if (writer_ == NULL) { - InitWriter(); - } - return tfileWriterPut(writer_, tv); + if (writer_ == NULL) { InitWriter(); } + return tfileWriterPut(writer_, tv, false); } bool InitWriter() { TFileHeader header; @@ -388,12 +380,8 @@ class TFileObj { return tfileReaderSearch(reader_, query, result); } ~TFileObj() { - if (writer_) { - tfileWriterDestroy(writer_); - } - if (reader_) { - tfileReaderDestroy(reader_); - } + if (writer_) { tfileWriterDestroy(writer_); } + if (reader_) { tfileReaderDestroy(reader_); } } private: @@ -439,7 +427,7 @@ static TFileValue* genTFileValue(const char* val) { memcpy(tv->colVal, val, vlen); tv->tableId = (SArray*)taosArrayInit(1, sizeof(uint64_t)); - for (size_t i = 0; i < 10; i++) { + for (size_t i = 0; i < 200; i++) { uint64_t v = i; taosArrayPush(tv->tableId, &v); } @@ -452,22 +440,17 @@ static void destroyTFileValue(void* val) { free(tv); } TEST_F(IndexTFileEnv, test_tfile_write) { - TFileValue* v1 = genTFileValue("c"); - TFileValue* v2 = genTFileValue("ab"); - TFileValue* v3 = genTFileValue("b"); - TFileValue* v4 = genTFileValue("d"); + TFileValue* v1 = genTFileValue("ab"); SArray* data = (SArray*)taosArrayInit(4, sizeof(void*)); taosArrayPush(data, &v1); - taosArrayPush(data, &v2); - taosArrayPush(data, &v3); - taosArrayPush(data, &v4); + // taosArrayPush(data, &v2); + // taosArrayPush(data, &v3); + // taosArrayPush(data, &v4); fObj->Put(data); - for (size_t i = 0; i < taosArrayGetSize(data); i++) { - destroyTFileValue(taosArrayGetP(data, i)); - } + for (size_t i = 0; i < taosArrayGetSize(data); i++) { destroyTFileValue(taosArrayGetP(data, i)); } taosArrayDestroy(data); std::string colName("voltage"); @@ -478,7 +461,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); fObj->Get(&query, result); - assert(taosArrayGetSize(result) == 10); + assert(taosArrayGetSize(result) == 200); indexTermDestroy(term); // tfileWriterDestroy(twrite); @@ -569,7 +552,8 @@ TEST_F(IndexCacheEnv, cache_test) { colVal.c_str(), colVal.size()); coj->Put(term, colId, version++, suid++); } - + coj->Debug(); + std::cout << "--------first----------" << std::endl; { std::string colVal("v3"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), @@ -582,6 +566,8 @@ TEST_F(IndexCacheEnv, cache_test) { colVal.c_str(), colVal.size()); coj->Put(term, othColId, version++, suid++); } + coj->Debug(); + std::cout << "--------second----------" << std::endl; { std::string colVal("v4"); for (size_t i = 0; i < 10; i++) { @@ -602,7 +588,7 @@ TEST_F(IndexCacheEnv, cache_test) { STermValueType valType; coj->Get(&query, colId, 10000, ret, &valType); - // std::cout << "size : " << taosArrayGetSize(ret) << std::endl; + std::cout << "size : " << taosArrayGetSize(ret) << std::endl; assert(taosArrayGetSize(ret) == 4); } { @@ -635,6 +621,20 @@ class IndexObj { } return ret; } + int WriteMillonData(const std::string& colName, const std::string& colVal = "Hello world", + size_t numOfTable = 100 * 10000) { + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + SIndexMultiTerm* terms = indexMultiTermCreate(); + indexMultiTermAdd(terms, term); + for (size_t i = 0; i < numOfTable; i++) { + int ret = Put(terms, i); + assert(ret == 0); + } + indexMultiTermDestroy(terms); + return numOfTable; + } + int Put(SIndexMultiTerm* fvs, uint64_t uid) { numOfWrite += taosArrayGetSize(fvs); return indexPut(idx, fvs, uid); @@ -645,14 +645,25 @@ class IndexObj { return indexSearch(idx, multiQ, result); } + int SearchOne(const std::string& colName, const std::string& colVal) { + SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); + SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + indexMultiTermQueryAdd(mq, term, QUERY_TERM); + + SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); + if (Search(mq, result) == 0) { std::cout << "search one successfully" << std::endl; } + return taosArrayGetSize(result); + // assert(taosArrayGetSize(result) == targetSize); + } void Debug() { std::cout << "numOfWrite:" << numOfWrite << std::endl; std::cout << "numOfRead:" << numOfRead << std::endl; } ~IndexObj() { - indexClose(idx); indexCleanUp(); + indexClose(idx); } private: @@ -665,12 +676,13 @@ class IndexObj { class IndexEnv2 : public ::testing::Test { protected: virtual void SetUp() { + tfInit(); index = new IndexObj(); // } virtual void TearDown() { - // r delete index; + tfCleanup(); } IndexObj* index; }; @@ -681,9 +693,9 @@ TEST_F(IndexEnv2, testIndexOpen) { exit(1); } - int targetSize = 100; + int targetSize = 200; { - std::string colName("tag1"), colVal("Hello world"); + std::string colName("tag1"), colVal("Hello"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); @@ -697,8 +709,8 @@ TEST_F(IndexEnv2, testIndexOpen) { indexMultiTermDestroy(terms); } { - size_t size = 100; - std::string colName("tag1"), colVal("hello world"); + size_t size = 200; + std::string colName("tag1"), colVal("hello"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); @@ -713,7 +725,7 @@ TEST_F(IndexEnv2, testIndexOpen) { } { - std::string colName("tag1"), colVal("Hello world"); + std::string colName("tag1"), colVal("Hello"); SIndexMultiTermQuery* mq = indexMultiTermQueryCreate(MUST); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), @@ -722,27 +734,39 @@ TEST_F(IndexEnv2, testIndexOpen) { SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); index->Search(mq, result); - assert(taosArrayGetSize(result) == targetSize); - } -} -TEST_F(IndexEnv2, testIndex_CachePut) { - std::string path = "/tmp"; - if (index->Init(path) != 0) { + std::cout << "target size: " << taosArrayGetSize(result) << std::endl; + // assert(taosArrayGetSize(result) == targetSize); } } -TEST_F(IndexEnv2, testIndexr_TFilePut) { +TEST_F(IndexEnv2, testIndex_TrigeFlush) { std::string path = "/tmp"; - if (index->Init(path) != 0) { - } + if (index->Init(path) != 0) {} + int numOfTable = 100 * 10000; + index->WriteMillonData("tag1", "Hello world", numOfTable); + int target = index->SearchOne("tag1", "Hello world"); + assert(numOfTable == target); } -TEST_F(IndexEnv2, testIndex_CacheSearch) { +TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { std::string path = "/tmp"; - if (index->Init(path) != 0) { - } + if (index->Init(path) != 0) {} } -TEST_F(IndexEnv2, testIndex_TFileSearch) { +TEST_F(IndexEnv2, testIndex_multi_thread_write) { std::string path = "/tmp"; - if (index->Init(path) != 0) { - } + if (index->Init(path) != 0) {} } +TEST_F(IndexEnv2, testIndex_multi_thread_read) { + std::string path = "/tmp"; + if (index->Init(path) != 0) {} +} + +TEST_F(IndexEnv2, testIndex_restart) { + std::string path = "/tmp"; + if (index->Init(path) != 0) {} +} + +TEST_F(IndexEnv2, testIndex_performance) { + std::string path = "/tmp"; + if (index->Init(path) != 0) {} +} +TEST_F(IndexEnv2, testIndexMultiTag) {} diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3a923c6653..9fa5b3198b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -182,10 +182,15 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG, "Invalid dnode cfg") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_EP, "Invalid dnode end point") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_ID, "Invalid dnode id") -// mnode-mnode +// mnode-node TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_NOT_EXIST, "Mnode not there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "Too many mnodes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_ALREADY_EXIST, "Qnode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_NOT_EXIST, "Qnode not there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_ALREADY_EXIST, "Snode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_NOT_EXIST, "Snode not there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_ALREADY_EXIST, "Bnode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_NOT_EXIST, "Bnode not there") // mnode-acct TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, "Account already exists")