From 5252e1771de533ca87886378ba58276f83218103 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 7 Dec 2021 20:14:22 +0800 Subject: [PATCH 01/18] TD-10431 user manage --- include/dnode/mnode/sdb/sdb.h | 14 +- source/dnode/mgmt/impl/test/sut/deploy.cpp | 1 - .../dnode/mgmt/impl/test/user/CMakeLists.txt | 29 ++ source/dnode/mgmt/impl/test/user/user.cpp | 310 ++++++++++++++++++ source/dnode/mnode/impl/inc/mndDef.h | 5 + source/dnode/mnode/impl/inc/mndSync.h | 2 +- source/dnode/mnode/impl/inc/mndTrans.h | 7 +- source/dnode/mnode/impl/src/mndAcct.c | 16 +- source/dnode/mnode/impl/src/mndCluster.c | 4 +- source/dnode/mnode/impl/src/mndDnode.c | 14 +- source/dnode/mnode/impl/src/mndMnode.c | 10 +- source/dnode/mnode/impl/src/mndSync.c | 12 +- source/dnode/mnode/impl/src/mndTelem.c | 8 +- source/dnode/mnode/impl/src/mndTrans.c | 219 ++++++++----- source/dnode/mnode/impl/src/mndUser.c | 139 +++++++- source/dnode/mnode/impl/src/mnode.c | 15 +- source/dnode/mnode/sdb/inc/sdbInt.h | 1 - source/dnode/mnode/sdb/src/sdb.c | 4 +- source/dnode/mnode/sdb/src/sdbFile.c | 2 +- source/dnode/mnode/sdb/src/sdbHash.c | 9 +- source/dnode/mnode/sdb/src/sdbRaw.c | 7 +- 21 files changed, 676 insertions(+), 152 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/user/CMakeLists.txt create mode 100644 source/dnode/mgmt/impl/test/user/user.cpp diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 8bd3db0217..a42736e4fb 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -125,8 +125,7 @@ typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyTy typedef enum { SDB_STATUS_CREATING = 1, SDB_STATUS_READY = 2, - SDB_STATUS_DROPPING = 3, - SDB_STATUS_DROPPED = 4 + SDB_STATUS_DROPPED = 3 } ESdbStatus; typedef enum { @@ -258,7 +257,7 @@ int32_t sdbDeploy(SSdb *pSdb); int32_t sdbReadFile(SSdb *pSdb); /** - * @brief Parse and write raw data to sdb. + * @brief Parse and write raw data to sdb, then free the pRaw object * * @param pSdb The sdb object. * @param pRaw The raw data. @@ -266,6 +265,15 @@ int32_t sdbReadFile(SSdb *pSdb); */ int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw); +/** + * @brief Parse and write raw data to sdb. + * + * @param pSdb The sdb object. + * @param pRaw The raw data. + * @return int32_t 0 for success, -1 for failure. + */ +int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw); + /** * @brief Acquire a row from sdb * diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index ad1667eac5..92b0967345 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -34,7 +34,6 @@ void initLog(const char* path) { sDebugFlag = 0; tsdbDebugFlag = 0; cqDebugFlag = 0; - debugFlag = 0; char temp[PATH_MAX]; snprintf(temp, PATH_MAX, "%s/taosdlog", path); diff --git a/source/dnode/mgmt/impl/test/user/CMakeLists.txt b/source/dnode/mgmt/impl/test/user/CMakeLists.txt new file mode 100644 index 0000000000..b5f02e41f4 --- /dev/null +++ b/source/dnode/mgmt/impl/test/user/CMakeLists.txt @@ -0,0 +1,29 @@ +add_executable(dndTestUser "") + +target_sources(dndTestUser + PRIVATE + "user.cpp" + "../sut/deploy.cpp" +) + +target_link_libraries( + dndTestUser + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dndTestUser + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../sut" +) + +enable_testing() + +add_test( + NAME dndTestUser + COMMAND dndTestUser +) diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp new file mode 100644 index 0000000000..befe005d55 --- /dev/null +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "deploy.h" + +class DndTestUser : public ::testing::Test { + protected: + void SetUp() override {} + void TearDown() override {} + + static void SetUpTestSuite() { + const char* user = "root"; + const char* pass = "taosdata"; + const char* path = "/tmp/dndTestUser"; + const char* fqdn = "localhost"; + uint16_t port = 9524; + + pServer = createServer(path, fqdn, port); + ASSERT(pServer); + pClient = createClient(user, pass, fqdn, port); + } + + static void TearDownTestSuite() { + dropServer(pServer); + dropClient(pClient); + } + + static SServer* pServer; + static SClient* pClient; + static int32_t connId; +}; + +SServer* DndTestUser::pServer; +SClient* DndTestUser::pClient; +int32_t DndTestUser::connId; + +#if 0 +TEST_F(DndTestUser, ShowUser) { + int32_t showId = 0; + + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_USER; + strcpy(pShow->db, ""); + + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + ASSERT_NE(pShowRsp, nullptr); + pShowRsp->showId = htonl(pShowRsp->showId); + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->contLen = htonl(pMeta->contLen); + pMeta->numOfColumns = htons(pMeta->numOfColumns); + pMeta->sversion = htons(pMeta->sversion); + pMeta->tversion = htons(pMeta->tversion); + pMeta->tid = htonl(pMeta->tid); + pMeta->uid = htobe64(pMeta->uid); + pMeta->suid = htobe64(pMeta->suid); + + showId = pShowRsp->showId; + + EXPECT_NE(pShowRsp->showId, 0); + EXPECT_EQ(pMeta->contLen, 0); + EXPECT_STREQ(pMeta->tableFname, "show users"); + EXPECT_EQ(pMeta->numOfTags, 0); + EXPECT_EQ(pMeta->precision, 0); + EXPECT_EQ(pMeta->tableType, 0); + EXPECT_EQ(pMeta->numOfColumns, 4); + EXPECT_EQ(pMeta->sversion, 0); + EXPECT_EQ(pMeta->tversion, 0); + EXPECT_EQ(pMeta->tid, 0); + EXPECT_EQ(pMeta->uid, 0); + EXPECT_STREQ(pMeta->sTableName, ""); + EXPECT_EQ(pMeta->suid, 0); + + SSchema* pSchema = NULL; + + pSchema = &pMeta->schema[0]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "name"); + + pSchema = &pMeta->schema[1]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, 10 + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "privilege"); + + pSchema = &pMeta->schema[2]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); + EXPECT_EQ(pSchema->bytes, 8); + EXPECT_STREQ(pSchema->name, "create_time"); + + pSchema = &pMeta->schema[3]; + pSchema->bytes = htons(pSchema->bytes); + EXPECT_EQ(pSchema->colId, 0); + EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); + EXPECT_STREQ(pSchema->name, "account"); + + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = htonl(showId); + pRetrieve->free = 0; + + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + ASSERT_NE(pClient->pRsp, nullptr); + ASSERT_EQ(pClient->pRsp->code, 0); + + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + ASSERT_NE(pRetrieveRsp, nullptr); + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + pRetrieveRsp->offset = htobe64(pRetrieveRsp->offset); + pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); + pRetrieveRsp->compLen = htonl(pRetrieveRsp->compLen); + + EXPECT_EQ(pRetrieveRsp->numOfRows, 2); + EXPECT_EQ(pRetrieveRsp->offset, 0); + EXPECT_EQ(pRetrieveRsp->useconds, 0); + EXPECT_EQ(pRetrieveRsp->completed, 1); + EXPECT_EQ(pRetrieveRsp->precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(pRetrieveRsp->compressed, 0); + EXPECT_EQ(pRetrieveRsp->reserved, 0); + EXPECT_EQ(pRetrieveRsp->compLen, 0); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + int64_t int64Val = 0; + + //--- name --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "_root"); + } + + //--- privilege --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += 10; + EXPECT_STREQ(strVal, "super"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += 10; + EXPECT_STREQ(strVal, "writable"); + } + + //--- create_time --- + { + int64Val = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(int64Val, 0); + + int64Val = *((int64_t*)(pData + pos)); + pos += sizeof(int64_t); + EXPECT_GT(int64Val, 0); + } + + //--- account --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + } +} +#endif + +TEST_F(DndTestUser, CreateUser_01) { + ASSERT_NE(pClient, nullptr); + + //--- create user --- + SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; + + sendMsg(pClient, &rpcMsg); + // taosMsleep(10000000); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_USER; + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->numOfColumns = htons(pMeta->numOfColumns); + EXPECT_EQ(pMeta->numOfColumns, 4); + + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = pShowRsp->showId; + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + EXPECT_EQ(pRetrieveRsp->numOfRows, 3); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + + //--- name --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "u1"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "_root"); + } +} + +// TEST_F(DndTestUser, AlterUser) { +// ASSERT_NE(pClient, nullptr); + +// SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); + +// SRpcMsg rpcMsg = {0}; +// rpcMsg.pCont = pReq; +// rpcMsg.contLen = sizeof(SAlterUserMsg); +// rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_ACCT; + +// sendMsg(pClient, &rpcMsg); +// SRpcMsg* pMsg = pClient->pRsp; +// ASSERT_NE(pMsg, nullptr); +// ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); +// } + +// TEST_F(DndTestUser, DropUser) { +// ASSERT_NE(pClient, nullptr); + +// SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); + +// SRpcMsg rpcMsg = {0}; +// rpcMsg.pCont = pReq; +// rpcMsg.contLen = sizeof(SDropUserMsg); +// rpcMsg.msgType = TSDB_MSG_TYPE_DROP_ACCT; + +// sendMsg(pClient, &rpcMsg); +// SRpcMsg* pMsg = pClient->pRsp; +// ASSERT_NE(pMsg, nullptr); +// ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); +// } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9e4f922540..cb5c0c2755 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -318,6 +318,11 @@ typedef struct SMnodeMsg { void *pCont; } SMnodeMsg; +typedef struct { + int32_t id; + void *rpcHandle; +} STransMsg; + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndSync.h b/source/dnode/mnode/impl/inc/mndSync.h index 3e45b0adb1..02ba725be1 100644 --- a/source/dnode/mnode/impl/inc/mndSync.h +++ b/source/dnode/mnode/impl/inc/mndSync.h @@ -25,7 +25,7 @@ extern "C" { int32_t mndInitSync(SMnode *pMnode); void mndCleanupSync(SMnode *pMnode); bool mndIsMaster(SMnode *pMnode); -int32_t mndSyncPropose(SSdbRaw *pRaw, void *pData); +int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 9559a96255..878337e4be 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -33,13 +33,10 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *, void *pMsg); int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *, void *pMsg); -int32_t mndTransPrepare(STrans *pTrans, int32_t (*syncfp)(SSdbRaw *pRaw, void *pData)); -int32_t mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, void *pData, int32_t code); +int32_t mndTransPrepare(STrans *pTrans); +void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code); int32_t mndTransExecute(SSdb *pSdb, int32_t tranId); -SSdbRaw *mndTransActionEncode(STrans *pTrans); -SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index e3d37cd9f9..3d668a76f8 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -65,7 +65,7 @@ static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mTrace("acct:%s, will be created while deploy sdb", acctObj.acct); + mDebug("acct:%s, will be created while deploy sdb", acctObj.acct); return sdbWrite(pMnode->pSdb, pRaw); } @@ -132,14 +132,14 @@ static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { } static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pSrcAcct, SAcctObj *pDstAcct) { - mTrace("acct:%s, perform update action", pSrcAcct->acct); + mTrace("acct:%s, perform update action", pDstAcct->acct); - memcpy(pSrcAcct->acct, pDstAcct->acct, TSDB_USER_LEN); - pSrcAcct->createdTime = pDstAcct->createdTime; - pSrcAcct->updateTime = pDstAcct->updateTime; - pSrcAcct->acctId = pDstAcct->acctId; - pSrcAcct->status = pDstAcct->status; - pSrcAcct->cfg = pDstAcct->cfg; + memcpy(pDstAcct->acct, pSrcAcct->acct, TSDB_USER_LEN); + pDstAcct->createdTime = pSrcAcct->createdTime; + pDstAcct->updateTime = pSrcAcct->updateTime; + pDstAcct->acctId = pSrcAcct->acctId; + pDstAcct->status = pSrcAcct->status; + pDstAcct->cfg = pSrcAcct->cfg; return 0; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 9fa7a06630..f260c83538 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -108,7 +108,7 @@ static int32_t mndClusterActionDelete(SSdb *pSdb, SClusterObj *pCluster) { } static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pSrcCluster, SClusterObj *pDstCluster) { - mTrace("cluster:%d, perform update action", pSrcCluster->id); + mTrace("cluster:%d, perform update action", pDstCluster->id); return 0; } @@ -132,7 +132,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mTrace("cluster:%d, will be created while deploy sdb", clusterObj.id); + mDebug("cluster:%d, will be created while deploy sdb", clusterObj.id); return sdbWrite(pMnode->pSdb, pRaw); } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 241b006293..356b3073bf 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -102,12 +102,12 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) { } static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pSrcDnode, SDnodeObj *pDstDnode) { - mTrace("dnode:%d, perform update action", pSrcDnode->id); - pSrcDnode->id = pDstDnode->id; - pSrcDnode->createdTime = pDstDnode->createdTime; - pSrcDnode->updateTime = pDstDnode->updateTime; - pSrcDnode->port = pDstDnode->port; - memcpy(pSrcDnode->fqdn, pDstDnode->fqdn, TSDB_FQDN_LEN); + mTrace("dnode:%d, perform update action", pDstDnode->id); + pDstDnode->id = pSrcDnode->id; + pDstDnode->createdTime = pSrcDnode->createdTime; + pDstDnode->updateTime = pSrcDnode->updateTime; + pDstDnode->port = pSrcDnode->port; + memcpy(pDstDnode->fqdn, pSrcDnode->fqdn, TSDB_FQDN_LEN); return 0; } @@ -123,7 +123,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mTrace("dnode:%d, will be created while deploy sdb", dnodeObj.id); + mDebug("dnode:%d, will be created while deploy sdb", dnodeObj.id); return sdbWrite(pMnode->pSdb, pRaw); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index eabcce04c5..6fa55bb9f1 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -82,10 +82,10 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pMnodeObj) { } static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pSrcMnode, SMnodeObj *pDstMnode) { - mTrace("mnode:%d, perform update action", pSrcMnode->id); - pSrcMnode->id = pDstMnode->id; - pSrcMnode->createdTime = pDstMnode->createdTime; - pSrcMnode->updateTime = pDstMnode->updateTime; + mTrace("mnode:%d, perform update action", pDstMnode->id); + pDstMnode->id = pSrcMnode->id; + pDstMnode->createdTime = pSrcMnode->createdTime; + pDstMnode->updateTime = pSrcMnode->updateTime; return 0; } @@ -99,7 +99,7 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mTrace("mnode:%d, will be created while deploy sdb", mnodeObj.id); + mDebug("mnode:%d, will be created while deploy sdb", mnodeObj.id); return sdbWrite(pMnode->pSdb, pRaw); } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 7231915311..6e7ee662f8 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -21,9 +21,15 @@ int32_t mndInitSync(SMnode *pMnode) { return 0; } void mndCleanupSync(SMnode *pMnode) {} -int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, void *pData) { - mndTransApply(pMnode, pData, pData, 0); - free(pData); +int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg) { + int32_t code = 0; + + int32_t len = sdbGetRawTotalSize(pRaw); + SSdbRaw *pReceived = calloc(1, len); + memcpy(pReceived, pRaw, len); + mDebug("trans:%d, data:%p recv from sync, code:0x%x pMsg:%p", pMsg->id, pReceived, code & 0xFFFF, pMsg); + + mndTransApply(pMnode, pReceived, pMsg, code); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 663ae76506..5beb1b10e3 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -189,12 +189,12 @@ static void mndSendTelemetryReport(SMnode* pMnode) { char buf[128] = {0}; uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); if (ip == 0xffffffff) { - mTrace("failed to get IP address of " TELEMETRY_SERVER " since :%s", strerror(errno)); + mDebug("failed to get IP address of " TELEMETRY_SERVER " since :%s", strerror(errno)); return; } SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); if (fd < 0) { - mTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); + mDebug("failed to create socket for telemetry, reason:%s", strerror(errno)); return; } @@ -228,7 +228,7 @@ static void mndSendTelemetryReport(SMnode* pMnode) { // read something to avoid nginx error 499 if (taosReadSocket(fd, buf, 10) < 0) { - mTrace("failed to receive response since %s", strerror(errno)); + mDebug("failed to receive response since %s", strerror(errno)); } taosCloseSocket(fd); @@ -297,7 +297,7 @@ int32_t mndInitTelem(SMnode* pMnode) { int32_t code = pthread_create(&pMgmt->thread, &attr, mndTelemThreadFp, pMnode); pthread_attr_destroy(&attr); if (code != 0) { - mTrace("failed to create telemetry thread since :%s", strerror(code)); + mDebug("failed to create telemetry thread since :%s", strerror(code)); } mInfo("mnd telemetry is initialized"); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7eab16895c..8d1521c79a 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,13 +15,33 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "trpc.h" +#include "mndSync.h" #define SDB_TRANS_VER 1 #define TRN_DEFAULT_ARRAY_SIZE 8 -SSdbRaw *mndTransActionEncode(STrans *pTrans) { - int32_t rawDataLen = 10 * sizeof(int32_t); +static SSdbRaw *mndTransActionEncode(STrans *pTrans); +static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); +static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); +static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pTrans, STrans *pDstTrans); +static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); + +int32_t mndInitTrans(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_TRANS, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndTransActionEncode, + .decodeFp = (SdbDecodeFp)mndTransActionDecode, + .insertFp = (SdbInsertFp)mndTransActionInsert, + .updateFp = (SdbUpdateFp)mndTransActionUpdate, + .deleteFp = (SdbDeleteFp)mndTransActionDelete}; + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupTrans(SMnode *pMnode) {} + +static SSdbRaw *mndTransActionEncode(STrans *pTrans) { + int32_t rawDataLen = 16 * sizeof(int32_t); int32_t redoLogNum = taosArrayGetSize(pTrans->redoLogs); int32_t undoLogNum = taosArrayGetSize(pTrans->undoLogs); int32_t commitLogNum = taosArrayGetSize(pTrans->commitLogs); @@ -29,23 +49,23 @@ SSdbRaw *mndTransActionEncode(STrans *pTrans) { int32_t undoActionNum = taosArrayGetSize(pTrans->undoActions); for (int32_t i = 0; i < redoLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->redoLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->redoLogs, i); rawDataLen += sdbGetRawTotalSize(pTmp); } for (int32_t i = 0; i < undoLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->undoLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->undoLogs, i); rawDataLen += sdbGetRawTotalSize(pTmp); } for (int32_t i = 0; i < commitLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->commitLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->commitLogs, i); rawDataLen += sdbGetRawTotalSize(pTmp); } SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, SDB_TRANS_VER, rawDataLen); if (pRaw == NULL) { - mError("trn:%d, failed to alloc raw since %s", pTrans->id, terrstr()); + mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; } @@ -60,31 +80,33 @@ SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_INT32(pRaw, dataPos, undoActionNum) for (int32_t i = 0; i < redoLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->redoLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->redoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); SDB_SET_INT32(pRaw, dataPos, len) SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len) } for (int32_t i = 0; i < undoLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->undoLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->undoLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); SDB_SET_INT32(pRaw, dataPos, len) SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len) } for (int32_t i = 0; i < commitLogNum; ++i) { - SSdbRaw *pTmp = taosArrayGet(pTrans->commitLogs, i); + SSdbRaw *pTmp = taosArrayGetP(pTrans->commitLogs, i); int32_t len = sdbGetRawTotalSize(pTmp); SDB_SET_INT32(pRaw, dataPos, len) SDB_SET_BINARY(pRaw, dataPos, (void *)pTmp, len) } - mDebug("trn:%d, is encoded as raw:%p, len:%d", pTrans->id, pRaw, dataPos); + mTrace("trans:%d, encode to raw:%p, len:%d", pTrans->id, pRaw, dataPos); return pRaw; } -SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { +static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { + int32_t code = 0; + int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) { mError("failed to get soft ver from raw:%p since %s", pRaw, terrstr()); @@ -97,8 +119,8 @@ SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { return NULL; } - SSdbRow *pRow = sdbAllocRow(sizeof(STrans)); - STrans *pTrans = sdbGetRowObj(pRow); + SSdbRow *pRow = sdbAllocRow(sizeof(STrans)); + STrans *pTrans = sdbGetRowObj(pRow); if (pTrans == NULL) { mError("failed to alloc trans from raw:%p since %s", pRaw, terrstr()); return NULL; @@ -112,9 +134,9 @@ SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { if (pTrans->redoLogs == NULL || pTrans->undoLogs == NULL || pTrans->commitLogs == NULL || pTrans->redoActions == NULL || pTrans->undoActions == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mDebug("trn:%d, failed to create array while parsed from raw:%p", pTrans->id, pRaw); - return NULL; + mDebug("trans:%d, failed to create array while parsed from raw:%p", pTrans->id, pRaw); + code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; } int32_t redoLogNum = 0; @@ -133,81 +155,111 @@ SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, pRow, dataPos, &redoActionNum) SDB_GET_INT32(pRaw, pRow, dataPos, &undoActionNum) - int32_t code = 0; for (int32_t i = 0; i < redoLogNum; ++i) { int32_t dataLen = 0; SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen) char *pData = malloc(dataLen); SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen); - void *ret = taosArrayPush(pTrans->redoLogs, pData); + void *ret = taosArrayPush(pTrans->redoLogs, &pData); if (ret == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; break; } } + for (int32_t i = 0; i < undoLogNum; ++i) { + int32_t dataLen = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen) + + char *pData = malloc(dataLen); + SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen); + void *ret = taosArrayPush(pTrans->undoLogs, &pData); + if (ret == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; + break; + } + } + + for (int32_t i = 0; i < commitLogNum; ++i) { + int32_t dataLen = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &dataLen) + + char *pData = malloc(dataLen); + SDB_GET_BINARY(pRaw, pRow, dataPos, pData, dataLen); + void *ret = taosArrayPush(pTrans->commitLogs, &pData); + if (ret == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto TRANS_DECODE_OVER; + break; + } + } + +TRANS_DECODE_OVER: if (code != 0) { - terrno = code; - mError("trn:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, terrstr()); + mError("trans:%d, failed to parse from raw:%p since %s", pTrans->id, pRaw, tstrerror(errno)); mndTransDrop(pTrans); + terrno = code; return NULL; } - mDebug("trn:%d, is parsed from raw:%p", pTrans->id, pRaw); + mTrace("trans:%d, decode from raw:%p", pTrans->id, pRaw); return pRow; } static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { + mTrace("trans:%d, perform insert action, stage:%d", pTrans->id, pTrans->stage); + SArray *pArray = pTrans->redoLogs; int32_t arraySize = taosArrayGetSize(pArray); for (int32_t i = 0; i < arraySize; ++i) { - SSdbRaw *pRaw = taosArrayGet(pArray, i); + SSdbRaw *pRaw = taosArrayGetP(pArray, i); int32_t code = sdbWrite(pSdb, pRaw); if (code != 0) { - mError("trn:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); + mError("trans:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); return code; } } - - mDebug("trn:%d, write to sdb", pTrans->id); return 0; } static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { - SArray *pArray = pTrans->redoLogs; + mTrace("trans:%d, perform delete action, stage:%d", pTrans->id, pTrans->stage); + + SArray *pArray = pTrans->undoLogs; int32_t arraySize = taosArrayGetSize(pArray); for (int32_t i = 0; i < arraySize; ++i) { - SSdbRaw *pRaw = taosArrayGet(pArray, i); + SSdbRaw *pRaw = taosArrayGetP(pArray, i); int32_t code = sdbWrite(pSdb, pRaw); if (code != 0) { - mError("trn:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); + mError("trans:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); return code; } } - mDebug("trn:%d, delete from sdb", pTrans->id); return 0; } static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pTrans, STrans *pDstTrans) { - assert(true); - SArray *pArray = pTrans->redoLogs; + mTrace("trans:%d, perform update action, stage:%d", pTrans->id, pTrans->stage); + + SArray *pArray = pDstTrans->commitLogs; int32_t arraySize = taosArrayGetSize(pArray); for (int32_t i = 0; i < arraySize; ++i) { - SSdbRaw *pRaw = taosArrayGet(pArray, i); + SSdbRaw *pRaw = taosArrayGetP(pArray, i); int32_t code = sdbWrite(pSdb, pRaw); if (code != 0) { - mError("trn:%d, failed to write raw:%p to sdb since %s", pTrans->id, pRaw, terrstr()); + mError("trans:%d, failed to write raw:%p to sdb since %s", pDstTrans->id, pRaw, terrstr()); return code; } } - pTrans->stage = pDstTrans->stage; - mDebug("trn:%d, update in sdb", pTrans->id); + pDstTrans->stage = pTrans->stage; return 0; } @@ -224,6 +276,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { pTrans->id = trnGenerateTransId(); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; + pTrans->pMnode = pMnode; pTrans->rpcHandle = rpcHandle; pTrans->redoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); pTrans->undoLogs = taosArrayInit(TRN_DEFAULT_ARRAY_SIZE, sizeof(void *)); @@ -238,13 +291,13 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { return NULL; } - mDebug("trn:%d, is created, %p", pTrans->id, pTrans); + mDebug("trans:%d, data:%p is created", pTrans->id, pTrans); return pTrans; } static void trnDropArray(SArray *pArray) { for (int32_t i = 0; i < pArray->size; ++i) { - SSdbRaw *pRaw = taosArrayGet(pArray, i); + SSdbRaw *pRaw = taosArrayGetP(pArray, i); tfree(pRaw); } @@ -258,13 +311,13 @@ void mndTransDrop(STrans *pTrans) { trnDropArray(pTrans->redoActions); trnDropArray(pTrans->undoActions); - mDebug("trn:%d, is dropped, %p", pTrans->id, pTrans); + mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans); tfree(pTrans); } void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle) { pTrans->rpcHandle = rpcHandle; - mTrace("trn:%d, set rpc handle:%p", pTrans->id, rpcHandle); + mTrace("trans:%d, set rpc handle:%p", pTrans->id, rpcHandle); } static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) { @@ -273,7 +326,7 @@ static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) { return -1; } - void *ptr = taosArrayPush(pArray, pRaw); + void *ptr = taosArrayPush(pArray, &pRaw); if (ptr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -284,92 +337,92 @@ static int32_t mndTransAppendArray(SArray *pArray, SSdbRaw *pRaw) { int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw) { int32_t code = mndTransAppendArray(pTrans->redoLogs, pRaw); - mTrace("trn:%d, raw:%p append to redo logs, code:%d", pTrans->id, pRaw, code); + mTrace("trans:%d, raw:%p append to redo logs, code:%d", pTrans->id, pRaw, code); return code; } int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw) { int32_t code = mndTransAppendArray(pTrans->undoLogs, pRaw); - mTrace("trn:%d, raw:%p append to undo logs, code:%d", pTrans->id, pRaw, code); + mTrace("trans:%d, raw:%p append to undo logs, code:%d", pTrans->id, pRaw, code); return code; } int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { int32_t code = mndTransAppendArray(pTrans->commitLogs, pRaw); - mTrace("trn:%d, raw:%p append to commit logs, code:%d", pTrans->id, pRaw, code); + mTrace("trans:%d, raw:%p append to commit logs, code:%d", pTrans->id, pRaw, code); return code; } int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) { int32_t code = mndTransAppendArray(pTrans->redoActions, pMsg); - mTrace("trn:%d, msg:%p append to redo actions", pTrans->id, pMsg); + mTrace("trans:%d, msg:%p append to redo actions", pTrans->id, pMsg); return code; } int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, void *pMsg) { int32_t code = mndTransAppendArray(pTrans->undoActions, pMsg); - mTrace("trn:%d, msg:%p append to undo actions", pTrans->id, pMsg); + mTrace("trans:%d, msg:%p append to undo actions", pTrans->id, pMsg); return code; } -int32_t mndInitTrans(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_TRANS, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndTransActionEncode, - .decodeFp = (SdbDecodeFp)mndTransActionDecode, - .insertFp = (SdbInsertFp)mndTransActionInsert, - .updateFp = (SdbUpdateFp)mndTransActionUpdate, - .deleteFp = (SdbDeleteFp)mndTransActionDelete}; - - return sdbSetTable(pMnode->pSdb, table); -} - -void mndCleanupTrans(SMnode *pMnode) {} - -int32_t mndTransPrepare(STrans *pTrans, int32_t (*syncfp)(SSdbRaw *pRaw, void *pData)) { - if (syncfp == NULL) return -1; +int32_t mndTransPrepare(STrans *pTrans) { + mDebug("trans:%d, prepare transaction", pTrans->id); SSdbRaw *pRaw = mndTransActionEncode(pTrans); if (pRaw == NULL) { - mError("trn:%d, failed to decode trans since %s", pTrans->id, terrstr()); + mError("trans:%d, failed to decode trans since %s", pTrans->id, terrstr()); return -1; } sdbSetRawStatus(pRaw, SDB_STATUS_CREATING); - if (sdbWrite(pTrans->pMnode->pSdb, pRaw) != 0) { - mError("trn:%d, failed to write trans since %s", pTrans->id, terrstr()); + if (sdbWriteNotFree(pTrans->pMnode->pSdb, pRaw) != 0) { + mError("trans:%d, failed to write trans since %s", pTrans->id, terrstr()); return -1; } - if ((*syncfp)(pRaw, pTrans->rpcHandle) != 0) { - mError("trn:%d, failed to sync trans since %s", pTrans->id, terrstr()); + STransMsg *pMsg = calloc(1, sizeof(STransMsg)); + pMsg->id = pTrans->id; + pMsg->rpcHandle = pTrans->rpcHandle; + + mDebug("trans:%d, start sync, RPC:%p pMsg:%p", pTrans->id, pTrans->rpcHandle, pMsg); + if (mndSyncPropose(pTrans->pMnode, pRaw, pMsg) != 0) { + mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); + free(pMsg); + sdbFreeRaw(pRaw); return -1; } + sdbFreeRaw(pRaw); return 0; } -static void trnSendRpcRsp(void *rpcHandle, int32_t code) { - if (rpcHandle != NULL) { - SRpcMsg rspMsg = {.handle = rpcHandle, .code = terrno}; +static void trnSendRpcRsp(STransMsg *pMsg, int32_t code) { + mDebug("trans:%d, send rpc rsp, RPC:%p code:0x%x pMsg:%p", pMsg->id, pMsg->rpcHandle, code & 0xFFFF, pMsg); + if (pMsg->rpcHandle != NULL) { + SRpcMsg rspMsg = {.handle = pMsg->rpcHandle, .code = code}; rpcSendResponse(&rspMsg); } + + free(pMsg); } -int32_t mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, void *pData, int32_t code) { - if (code != 0) { - trnSendRpcRsp(pData, terrno); - return 0; +void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code) { + if (code == 0) { + mDebug("trans:%d, commit transaction", pMsg->id); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + if (sdbWrite(pMnode->pSdb, pRaw) != 0) { + code = terrno; + mError("trans:%d, failed to write sdb while commit since %s", pMsg->id, terrstr()); + } + trnSendRpcRsp(pMsg, code); + } else { + mDebug("trans:%d, rollback transaction", pMsg->id); + sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + if (sdbWrite(pMnode->pSdb, pRaw) != 0) { + mError("trans:%d, failed to write sdb while rollback since %s", pMsg->id, terrstr()); + } + trnSendRpcRsp(pMsg, code); } - - if (sdbWrite(pMnode->pSdb, pData) != 0) { - code = terrno; - trnSendRpcRsp(pData, code); - terrno = code; - return -1; - } - - return 0; } static int32_t trnExecuteArray(SMnode *pMnode, SArray *pArray) { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index a3dd255577..13baee4b54 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -32,6 +32,9 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg); static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg); +static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); int32_t mndInitUser(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_USER, @@ -47,6 +50,9 @@ int32_t mndInitUser(SMnode *pMnode) { mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_USER, mndProcessAlterUserMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_USER, mndProcessDropUserMsg); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_USER, mndGetUserMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_USER, mndRetrieveUsers); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_USER, mndCancelGetNextUser); return sdbSetTable(pMnode->pSdb, table); } @@ -70,7 +76,7 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char if (pRaw == NULL) return -1; sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mTrace("user:%s, will be created while deploy sdb", userObj.user); + mDebug("user:%s, will be created while deploy sdb", userObj.user); return sdbWrite(pMnode->pSdb, pRaw); } @@ -164,14 +170,14 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDstUser) { mTrace("user:%s, perform update action", pSrcUser->user); - memcpy(pSrcUser->user, pDstUser->user, TSDB_USER_LEN); - memcpy(pSrcUser->pass, pDstUser->pass, TSDB_KEY_LEN); - memcpy(pSrcUser->acct, pDstUser->acct, TSDB_USER_LEN); - pSrcUser->createdTime = pDstUser->createdTime; - pSrcUser->updateTime = pDstUser->updateTime; - pSrcUser->superAuth = pDstUser->superAuth; - pSrcUser->readAuth = pDstUser->readAuth; - pSrcUser->writeAuth = pDstUser->writeAuth; + memcpy(pDstUser->user, pSrcUser->user, TSDB_USER_LEN); + memcpy(pDstUser->pass, pSrcUser->pass, TSDB_KEY_LEN); + memcpy(pDstUser->acct, pSrcUser->acct, TSDB_USER_LEN); + pDstUser->createdTime = pSrcUser->createdTime; + pDstUser->updateTime = pSrcUser->updateTime; + pDstUser->superAuth = pSrcUser->superAuth; + pDstUser->readAuth = pSrcUser->readAuth; + pDstUser->writeAuth = pSrcUser->writeAuth; return 0; } @@ -197,11 +203,15 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, userObj.writeAuth = 1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); - if (pTrans == NULL) return -1; + if (pTrans == NULL) { + mError("user:%s, failed to create since %s", user, terrstr()); + return -1; + } + mDebug("trans:%d, used to create user:%s", pTrans->id, user); SSdbRaw *pRedoRaw = mndUserActionEncode(&userObj); if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("failed to append redo log since %s", terrstr()); + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } @@ -209,7 +219,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SSdbRaw *pUndoRaw = mndUserActionEncode(&userObj); if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("failed to append undo log since %s", terrstr()); + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } @@ -217,13 +227,14 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SSdbRaw *pCommitRaw = mndUserActionEncode(&userObj); if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("failed to append commit log since %s", terrstr()); + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); - if (mndTransPrepare(pTrans, mndSyncPropose) != 0) { + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; } @@ -236,6 +247,8 @@ static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; + mDebug("user:%s, start to create", pCreate->user); + if (pCreate->user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; mError("user:%s, failed to create since %s", pCreate->user, terrstr()); @@ -284,4 +297,102 @@ static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg) { terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; mError("failed to process drop user msg since %s", terrstr()); return -1; +} + +static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "privilege"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "account"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + strcpy(pMeta->tableFname, "show users"); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_USER); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + return 0; +} + +static int32_t mndRetrieveUsers(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SUserObj *pUser = NULL; + int32_t cols = 0; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_USER, pShow->pIter, (void **)&pUser); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->user, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + if (pUser->superAuth) { + const char *src = "super"; + STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); + } else if (pUser->writeAuth) { + const char *src = "writable"; + STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); + } else { + const char *src = "readable"; + STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); + } + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pUser->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pUser->acct, pShow->bytes[cols]); + cols++; + + numOfRows++; + sdbRelease(pSdb, pUser); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 2fa0838ee8..9ffe8968ae 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -333,7 +333,7 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { SMnodeMsg *pMsg = taosAllocateQitem(sizeof(SMnodeMsg)); if (pMsg == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("failed to create msg since %s", terrstr()); + mError("RPC:%p, app:%p failed to create msg since %s", pRpcMsg->handle, pRpcMsg->ahandle, terrstr()); return NULL; } @@ -341,7 +341,7 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { if (rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { mndCleanupMsg(pMsg); terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - mError("failed to create msg since %s", terrstr()); + mError("RPC:%p, app:%p failed to create msg since %s", pRpcMsg->handle, pRpcMsg->ahandle, terrstr()); return NULL; } memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN); @@ -350,13 +350,13 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { pMsg->rpcMsg = *pRpcMsg; pMsg->createdTime = taosGetTimestampSec(); - mTrace("msg:%p, is created", pMsg); + mTrace("msg:%p, app:%p is created, RPC:%p", pMsg, pRpcMsg->ahandle, pRpcMsg->handle); return pMsg; } void mndCleanupMsg(SMnodeMsg *pMsg) { + mTrace("msg:%p, app:%p is destroyed, RPC:%p", pMsg, pMsg->rpcMsg.ahandle, pMsg->rpcMsg.handle); taosFreeQitem(pMsg); - mTrace("msg:%p, is destroyed", pMsg); } void mndSendRsp(SMnodeMsg *pMsg, int32_t code) { @@ -371,7 +371,7 @@ static void mndProcessRpcMsg(SMnodeMsg *pMsg) { void *ahandle = pMsg->rpcMsg.ahandle; bool isReq = (msgType % 2 == 1); - mTrace("msg:%p, app:%p will be processed", pMsg, ahandle); + mTrace("msg:%p, app:%p type:%s will be processed", pMsg, ahandle, taosMsg[msgType]); if (isReq && !mndIsMaster(pMnode)) { code = TSDB_CODE_APP_NOT_READY; @@ -393,7 +393,10 @@ static void mndProcessRpcMsg(SMnodeMsg *pMsg) { } code = (*fp)(pMsg); - if (code != 0) { + if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mTrace("msg:%p, app:%p in progressing", pMsg, ahandle); + return; + } else if (code != 0) { code = terrno; mError("msg:%p, app:%p failed to process since %s", pMsg, ahandle, terrstr()); goto PROCESS_RPC_END; diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h index e492f28557..a160533bf2 100644 --- a/source/dnode/mnode/sdb/inc/sdbInt.h +++ b/source/dnode/mnode/sdb/inc/sdbInt.h @@ -71,7 +71,6 @@ typedef struct SSdb { } SSdb; int32_t sdbWriteFile(SSdb *pSdb); -int32_t sdbWriteRaw(SSdb *pSdb, SSdbRaw *pRaw); #ifdef __cplusplus } diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 68cb7be68b..1d4888c2eb 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -94,7 +94,7 @@ void sdbCleanup(SSdb *pSdb) { taosHashClear(hash); taosHashCleanup(hash); pSdb->hashObjs[i] = NULL; - mTrace("sdb table:%d is cleaned up", i); + mDebug("sdb table:%d is cleaned up", i); } free(pSdb); @@ -129,7 +129,7 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { pSdb->hashObjs[sdbType] = hash; taosInitRWLatch(&pSdb->locks[sdbType]); - mTrace("sdb table:%d is initialized", sdbType); + mDebug("sdb table:%d is initialized", sdbType); return 0; } \ No newline at end of file diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6f88f08b2c..af37e9e1d5 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -118,7 +118,7 @@ int32_t sdbReadFile(SSdb *pSdb) { break; } - code = sdbWriteRaw(pSdb, pRaw); + code = sdbWriteNotFree(pSdb, pRaw); if (code != 0) { mError("failed to read file:%s since %s", file, terrstr()); goto PARSE_SDB_DATA_ERROR; diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index bdca5eaa98..53fcc3f5b0 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -98,7 +98,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * } SSdbRow *pDstRow = *ppDstRow; - pRow->status = pRaw->status; + pDstRow->status = pRaw->status; taosRUnLockLatch(pLock); SdbUpdateFp updateFp = pSdb->updateFps[pRow->type]; @@ -138,7 +138,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * return code; } -int32_t sdbWriteRaw(SSdb *pSdb, SSdbRaw *pRaw) { +int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw) { SHashObj *hash = sdbGetHash(pSdb, pRaw->type); if (hash == NULL) return terrno; @@ -158,7 +158,6 @@ int32_t sdbWriteRaw(SSdb *pSdb, SSdbRaw *pRaw) { code = sdbInsertRow(pSdb, hash, pRaw, pRow, keySize); break; case SDB_STATUS_READY: - case SDB_STATUS_DROPPING: code = sdbUpdateRow(pSdb, hash, pRaw, pRow, keySize); break; case SDB_STATUS_DROPPED: @@ -170,7 +169,7 @@ int32_t sdbWriteRaw(SSdb *pSdb, SSdbRaw *pRaw) { } int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw) { - int32_t code = sdbWriteRaw(pSdb, pRaw); + int32_t code = sdbWriteNotFree(pSdb, pRaw); sdbFreeRaw(pRaw); return code; } @@ -201,7 +200,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, void *pKey) { case SDB_STATUS_CREATING: terrno = TSDB_CODE_SDB_OBJ_CREATING; break; - case SDB_STATUS_DROPPING: + case SDB_STATUS_DROPPED: terrno = TSDB_CODE_SDB_OBJ_DROPPING; break; default: diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 7ed1a427f5..e37559808e 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -26,10 +26,15 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { pRaw->type = type; pRaw->sver = sver; pRaw->dataLen = dataLen; + + mTrace("raw:%p, is created, len:%d", pRaw, dataLen); return pRaw; } -void sdbFreeRaw(SSdbRaw *pRaw) { free(pRaw); } +void sdbFreeRaw(SSdbRaw *pRaw) { + mTrace("raw:%p, is freed", pRaw); + free(pRaw); +} int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) { if (pRaw == NULL) { From 19d54f6f372a7aa5c1dc0187c025966d9e9e80f6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 7 Dec 2021 21:10:36 +0800 Subject: [PATCH 02/18] TD-10431 user update --- include/dnode/mnode/sdb/sdb.h | 3 +- source/dnode/mgmt/impl/test/CMakeLists.txt | 9 +- source/dnode/mgmt/impl/test/user/user.cpp | 147 ++++++++++++++--- source/dnode/mnode/impl/src/mndAcct.c | 18 +- source/dnode/mnode/impl/src/mndCluster.c | 6 +- source/dnode/mnode/impl/src/mndDnode.c | 14 +- source/dnode/mnode/impl/src/mndMnode.c | 10 +- source/dnode/mnode/impl/src/mndTrans.c | 12 +- source/dnode/mnode/impl/src/mndUser.c | 183 +++++++++++++++++++-- source/dnode/mnode/sdb/src/sdbHash.c | 43 ++--- 10 files changed, 347 insertions(+), 98 deletions(-) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index a42736e4fb..17ce5c952b 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -125,7 +125,8 @@ typedef enum { SDB_KEY_BINARY = 1, SDB_KEY_INT32 = 2, SDB_KEY_INT64 = 3 } EKeyTy typedef enum { SDB_STATUS_CREATING = 1, SDB_STATUS_READY = 2, - SDB_STATUS_DROPPED = 3 + SDB_STATUS_DROPPING = 3, + SDB_STATUS_DROPPED = 4 } ESdbStatus; typedef enum { diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt index e7aa103996..b3b6818a5c 100644 --- a/source/dnode/mgmt/impl/test/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -1,4 +1,5 @@ -add_subdirectory(acct) -add_subdirectory(cluster) -add_subdirectory(profile) -add_subdirectory(show) +# add_subdirectory(acct) +# add_subdirectory(cluster) +# add_subdirectory(profile) +# add_subdirectory(show) +add_subdirectory(user) diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index befe005d55..081229f825 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -222,7 +222,6 @@ TEST_F(DndTestUser, CreateUser_01) { rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_USER; sendMsg(pClient, &rpcMsg); - // taosMsleep(10000000); SRpcMsg* pMsg = pClient->pRsp; ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); @@ -277,34 +276,132 @@ TEST_F(DndTestUser, CreateUser_01) { } } -// TEST_F(DndTestUser, AlterUser) { -// ASSERT_NE(pClient, nullptr); +TEST_F(DndTestUser, AlterUser_01) { + ASSERT_NE(pClient, nullptr); -// SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); + //--- drop user --- + SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p2"); -// SRpcMsg rpcMsg = {0}; -// rpcMsg.pCont = pReq; -// rpcMsg.contLen = sizeof(SAlterUserMsg); -// rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_ACCT; + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; -// sendMsg(pClient, &rpcMsg); -// SRpcMsg* pMsg = pClient->pRsp; -// ASSERT_NE(pMsg, nullptr); -// ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); -// } + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); -// TEST_F(DndTestUser, DropUser) { -// ASSERT_NE(pClient, nullptr); + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_USER; + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; -// SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); + sendMsg(pClient, &showRpcMsg); + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->numOfColumns = htons(pMeta->numOfColumns); + EXPECT_EQ(pMeta->numOfColumns, 4); -// SRpcMsg rpcMsg = {0}; -// rpcMsg.pCont = pReq; -// rpcMsg.contLen = sizeof(SDropUserMsg); -// rpcMsg.msgType = TSDB_MSG_TYPE_DROP_ACCT; + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = pShowRsp->showId; + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; -// sendMsg(pClient, &rpcMsg); -// SRpcMsg* pMsg = pClient->pRsp; -// ASSERT_NE(pMsg, nullptr); -// ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); -// } + sendMsg(pClient, &retrieveRpcMsg); + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + EXPECT_EQ(pRetrieveRsp->numOfRows, 3); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + + //--- name --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "u1"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "_root"); + } +} + +TEST_F(DndTestUser, DropUser_01) { + ASSERT_NE(pClient, nullptr); + + //--- drop user --- + SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); + strcpy(pReq->user, "u1"); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + //--- meta --- + SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); + pShow->type = TSDB_MGMT_TABLE_USER; + SRpcMsg showRpcMsg = {0}; + showRpcMsg.pCont = pShow; + showRpcMsg.contLen = sizeof(SShowMsg); + showRpcMsg.msgType = TSDB_MSG_TYPE_SHOW; + + sendMsg(pClient, &showRpcMsg); + SShowRsp* pShowRsp = (SShowRsp*)pClient->pRsp->pCont; + STableMetaMsg* pMeta = &pShowRsp->tableMeta; + pMeta->numOfColumns = htons(pMeta->numOfColumns); + EXPECT_EQ(pMeta->numOfColumns, 4); + + //--- retrieve --- + SRetrieveTableMsg* pRetrieve = (SRetrieveTableMsg*)rpcMallocCont(sizeof(SRetrieveTableMsg)); + pRetrieve->showId = pShowRsp->showId; + SRpcMsg retrieveRpcMsg = {0}; + retrieveRpcMsg.pCont = pRetrieve; + retrieveRpcMsg.contLen = sizeof(SRetrieveTableMsg); + retrieveRpcMsg.msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + + sendMsg(pClient, &retrieveRpcMsg); + SRetrieveTableRsp* pRetrieveRsp = (SRetrieveTableRsp*)pClient->pRsp->pCont; + pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); + EXPECT_EQ(pRetrieveRsp->numOfRows, 2); + + char* pData = pRetrieveRsp->data; + int32_t pos = 0; + char* strVal = NULL; + + //--- name --- + { + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "root"); + + pos += sizeof(VarDataLenT); + strVal = (char*)(pData + pos); + pos += TSDB_USER_LEN; + EXPECT_STREQ(strVal, "_root"); + } +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 3d668a76f8..a365ad7326 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -24,7 +24,7 @@ static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct); static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw); static int32_t mnodeAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); -static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pSrcAcct, SAcctObj *pDstAcct); +static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct); static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg); @@ -131,15 +131,15 @@ static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { return 0; } -static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pSrcAcct, SAcctObj *pDstAcct) { - mTrace("acct:%s, perform update action", pDstAcct->acct); +static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) { + mTrace("acct:%s, perform update action", pOldAcct->acct); - memcpy(pDstAcct->acct, pSrcAcct->acct, TSDB_USER_LEN); - pDstAcct->createdTime = pSrcAcct->createdTime; - pDstAcct->updateTime = pSrcAcct->updateTime; - pDstAcct->acctId = pSrcAcct->acctId; - pDstAcct->status = pSrcAcct->status; - pDstAcct->cfg = pSrcAcct->cfg; + memcpy(pOldAcct->acct, pNewAcct->acct, TSDB_USER_LEN); + pOldAcct->createdTime = pNewAcct->createdTime; + pOldAcct->updateTime = pNewAcct->updateTime; + pOldAcct->acctId = pNewAcct->acctId; + pOldAcct->status = pNewAcct->status; + pOldAcct->cfg = pNewAcct->cfg; return 0; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index f260c83538..97baeb0a59 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -24,7 +24,7 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw); static int32_t mndClusterActionInsert(SSdb *pSdb, SClusterObj *pCluster); static int32_t mndClusterActionDelete(SSdb *pSdb, SClusterObj *pCluster); -static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pSrcCluster, SClusterObj *pDstCluster); +static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOldCluster, SClusterObj *pNewCluster); static int32_t mndCreateDefaultCluster(SMnode *pMnode); static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveClusters(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); @@ -107,8 +107,8 @@ static int32_t mndClusterActionDelete(SSdb *pSdb, SClusterObj *pCluster) { return 0; } -static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pSrcCluster, SClusterObj *pDstCluster) { - mTrace("cluster:%d, perform update action", pDstCluster->id); +static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOldCluster, SClusterObj *pNewCluster) { + mTrace("cluster:%d, perform update action", pOldCluster->id); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 356b3073bf..d187294b87 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -101,13 +101,13 @@ static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode) { return 0; } -static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pSrcDnode, SDnodeObj *pDstDnode) { - mTrace("dnode:%d, perform update action", pDstDnode->id); - pDstDnode->id = pSrcDnode->id; - pDstDnode->createdTime = pSrcDnode->createdTime; - pDstDnode->updateTime = pSrcDnode->updateTime; - pDstDnode->port = pSrcDnode->port; - memcpy(pDstDnode->fqdn, pSrcDnode->fqdn, TSDB_FQDN_LEN); +static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode) { + mTrace("dnode:%d, perform update action", pOldDnode->id); + pOldDnode->id = pNewDnode->id; + pOldDnode->createdTime = pNewDnode->createdTime; + pOldDnode->updateTime = pNewDnode->updateTime; + pOldDnode->port = pNewDnode->port; + memcpy(pOldDnode->fqdn, pNewDnode->fqdn, TSDB_FQDN_LEN); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 6fa55bb9f1..2490f1fcae 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -81,11 +81,11 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pMnodeObj) { return 0; } -static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pSrcMnode, SMnodeObj *pDstMnode) { - mTrace("mnode:%d, perform update action", pDstMnode->id); - pDstMnode->id = pSrcMnode->id; - pDstMnode->createdTime = pSrcMnode->createdTime; - pDstMnode->updateTime = pSrcMnode->updateTime; +static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOldMnode, SMnodeObj *pNewMnode) { + mTrace("mnode:%d, perform update action", pOldMnode->id); + pOldMnode->id = pNewMnode->id; + pOldMnode->createdTime = pNewMnode->createdTime; + pOldMnode->updateTime = pNewMnode->updateTime; return 0; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 8d1521c79a..0a15f03149 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -23,7 +23,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); -static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pTrans, STrans *pDstTrans); +static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOldTrans); static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); int32_t mndInitTrans(SMnode *pMnode) { @@ -244,22 +244,22 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { return 0; } -static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pTrans, STrans *pDstTrans) { - mTrace("trans:%d, perform update action, stage:%d", pTrans->id, pTrans->stage); +static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewTrans) { + mTrace("trans:%d, perform update action, stage:%d", pOldTrans->id, pNewTrans->stage); - SArray *pArray = pDstTrans->commitLogs; + SArray *pArray = pOldTrans->commitLogs; int32_t arraySize = taosArrayGetSize(pArray); for (int32_t i = 0; i < arraySize; ++i) { SSdbRaw *pRaw = taosArrayGetP(pArray, i); int32_t code = sdbWrite(pSdb, pRaw); if (code != 0) { - mError("trans:%d, failed to write raw:%p to sdb since %s", pDstTrans->id, pRaw, terrstr()); + mError("trans:%d, failed to write raw:%p to sdb since %s", pOldTrans->id, pRaw, terrstr()); return code; } } - pDstTrans->stage = pTrans->stage; + pOldTrans->stage = pNewTrans->stage; return 0; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 13baee4b54..b3c23c0372 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -27,7 +27,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser); static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); -static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDstUser); +static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser); static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pMsg); static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg); static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg); @@ -168,16 +168,16 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { return 0; } -static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDstUser) { - mTrace("user:%s, perform update action", pSrcUser->user); - memcpy(pDstUser->user, pSrcUser->user, TSDB_USER_LEN); - memcpy(pDstUser->pass, pSrcUser->pass, TSDB_KEY_LEN); - memcpy(pDstUser->acct, pSrcUser->acct, TSDB_USER_LEN); - pDstUser->createdTime = pSrcUser->createdTime; - pDstUser->updateTime = pSrcUser->updateTime; - pDstUser->superAuth = pSrcUser->superAuth; - pDstUser->readAuth = pSrcUser->readAuth; - pDstUser->writeAuth = pSrcUser->writeAuth; +static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNewUser) { + mTrace("user:%s, perform update action", pOldUser->user); + memcpy(pOldUser->user, pNewUser->user, TSDB_USER_LEN); + memcpy(pOldUser->pass, pNewUser->pass, TSDB_KEY_LEN); + memcpy(pOldUser->acct, pNewUser->acct, TSDB_USER_LEN); + pOldUser->createdTime = pNewUser->createdTime; + pOldUser->updateTime = pNewUser->updateTime; + pOldUser->superAuth = pNewUser->superAuth; + pOldUser->readAuth = pNewUser->readAuth; + pOldUser->writeAuth = pNewUser->writeAuth; return 0; } @@ -243,6 +243,82 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, return 0; } +static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewUser, SMnodeMsg *pMsg) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("user:%s, failed to update since %s", pOldUser->user, terrstr()); + return -1; + } + mDebug("trans:%d, used to update user:%s", pTrans->id, pOldUser->user); + + SSdbRaw *pRedoRaw = mndUserActionEncode(pNewUser); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndUserActionEncode(pOldUser); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndDropUser(SMnode *pMnode, SUserObj *pUser, SMnodeMsg *pMsg) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("user:%s, failed to drop since %s", pUser->user, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop user:%s", pTrans->id, pUser->user); + + SSdbRaw *pRedoRaw = mndUserActionEncode(pUser); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndUserActionEncode(pUser); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndUserActionEncode(pUser); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; @@ -288,15 +364,88 @@ static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { } static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { - terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; - mError("failed to process alter user msg since %s", terrstr()); - return -1; + SMnode *pMnode = pMsg->pMnode; + SAlterUserMsg *pAlter = pMsg->rpcMsg.pCont; + + mDebug("user:%s, start to alter", pAlter->user); + + if (pAlter->user[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; + mError("user:%s, failed to alter since %s", pAlter->user, terrstr()); + return -1; + } + + if (pAlter->pass[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; + mError("user:%s, failed to alter since %s", pAlter->user, terrstr()); + return -1; + } + + SUserObj *pUser = sdbAcquire(pMnode->pSdb, SDB_USER, pAlter->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_USER_NOT_EXIST; + mError("user:%s, failed to alter since %s", pAlter->user, terrstr()); + return -1; + } + + SUserObj *pOperUser = sdbAcquire(pMnode->pSdb, SDB_USER, pMsg->user); + if (pOperUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + mError("user:%s, failed to alter since %s", pAlter->user, terrstr()); + return -1; + } + + SUserObj newUser = {0}; + memcpy(&newUser, pUser, sizeof(SUserObj)); + memset(pUser->pass, 0, sizeof(pUser->pass)); + taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass); + + int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pMsg); + sdbRelease(pMnode->pSdb, pOperUser); + + if (code != 0) { + mError("user:%s, failed to alter since %s", pAlter->user, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg) { - terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; - mError("failed to process drop user msg since %s", terrstr()); - return -1; + SMnode *pMnode = pMsg->pMnode; + SDropUserMsg *pDrop = pMsg->rpcMsg.pCont; + + mDebug("user:%s, start to drop", pDrop->user); + + if (pDrop->user[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; + mError("user:%s, failed to drop since %s", pDrop->user, terrstr()); + return -1; + } + + SUserObj *pUser = sdbAcquire(pMnode->pSdb, SDB_USER, pDrop->user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_USER_NOT_EXIST; + mError("user:%s, failed to drop since %s", pDrop->user, terrstr()); + return -1; + } + + SUserObj *pOperUser = sdbAcquire(pMnode->pSdb, SDB_USER, pMsg->user); + if (pOperUser == NULL) { + terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; + mError("user:%s, failed to drop since %s", pDrop->user, terrstr()); + return -1; + } + + int32_t code = mndDropUser(pMnode, pUser, pMsg); + sdbRelease(pMnode->pSdb, pOperUser); + + if (code != 0) { + mError("user:%s, failed to drop since %s", pDrop->user, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 53fcc3f5b0..a852389d80 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -52,8 +52,8 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SRWLatch *pLock = &pSdb->locks[pRow->type]; taosWLockLatch(pLock); - SSdbRow *pDstRow = taosHashGet(hash, pRow->pObj, keySize); - if (pDstRow != NULL) { + SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize); + if (pOldRow != NULL) { taosWUnLockLatch(pLock); sdbFreeRow(pRow); return TSDB_CODE_SDB_OBJ_ALREADY_THERE; @@ -85,28 +85,28 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * return 0; } -static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pRow, int32_t keySize) { +static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pNewRow, int32_t keySize) { int32_t code = 0; - SRWLatch *pLock = &pSdb->locks[pRow->type]; + SRWLatch *pLock = &pSdb->locks[pNewRow->type]; taosRLockLatch(pLock); - SSdbRow **ppDstRow = taosHashGet(hash, pRow->pObj, keySize); - if (ppDstRow == NULL || *ppDstRow == NULL) { + SSdbRow **ppOldRow = taosHashGet(hash, pNewRow->pObj, keySize); + if (ppOldRow == NULL || *ppOldRow == NULL) { taosRUnLockLatch(pLock); - return sdbInsertRow(pSdb, hash, pRaw, pRow, keySize); + return sdbInsertRow(pSdb, hash, pRaw, pNewRow, keySize); } - SSdbRow *pDstRow = *ppDstRow; + SSdbRow *pOldRow = *ppOldRow; - pDstRow->status = pRaw->status; + pOldRow->status = pRaw->status; taosRUnLockLatch(pLock); - SdbUpdateFp updateFp = pSdb->updateFps[pRow->type]; + SdbUpdateFp updateFp = pSdb->updateFps[pNewRow->type]; if (updateFp != NULL) { - code = (*updateFp)(pSdb, pRow->pObj, pDstRow->pObj); + code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); } - sdbFreeRow(pRow); + sdbFreeRow(pNewRow); return code; } @@ -116,24 +116,24 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SRWLatch *pLock = &pSdb->locks[pRow->type]; taosWLockLatch(pLock); - SSdbRow **ppDstRow = taosHashGet(hash, pRow->pObj, keySize); - if (ppDstRow == NULL || *ppDstRow == NULL) { + SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize); + if (ppOldRow == NULL || *ppOldRow == NULL) { taosWUnLockLatch(pLock); sdbFreeRow(pRow); return TSDB_CODE_SDB_OBJ_NOT_THERE; } - SSdbRow *pDstRow = *ppDstRow; + SSdbRow *pOldRow = *ppOldRow; - pDstRow->status = pRaw->status; - taosHashRemove(hash, pDstRow->pObj, keySize); + pOldRow->status = pRaw->status; + taosHashRemove(hash, pOldRow->pObj, keySize); taosWUnLockLatch(pLock); - SdbDeleteFp deleteFp = pSdb->deleteFps[pDstRow->type]; + SdbDeleteFp deleteFp = pSdb->deleteFps[pOldRow->type]; if (deleteFp != NULL) { - code = (*deleteFp)(pSdb, pDstRow->pObj); + code = (*deleteFp)(pSdb, pOldRow->pObj); } - sdbRelease(pSdb, pDstRow->pObj); + sdbRelease(pSdb, pOldRow->pObj); sdbFreeRow(pRow); return code; } @@ -158,6 +158,7 @@ int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw) { code = sdbInsertRow(pSdb, hash, pRaw, pRow, keySize); break; case SDB_STATUS_READY: + case SDB_STATUS_DROPPING: code = sdbUpdateRow(pSdb, hash, pRaw, pRow, keySize); break; case SDB_STATUS_DROPPED: @@ -200,7 +201,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, void *pKey) { case SDB_STATUS_CREATING: terrno = TSDB_CODE_SDB_OBJ_CREATING; break; - case SDB_STATUS_DROPPED: + case SDB_STATUS_DROPPING: terrno = TSDB_CODE_SDB_OBJ_DROPPING; break; default: From 552236d4b0fba54e2286ea4ab83d5cf2f65a0237 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 8 Dec 2021 10:12:37 +0800 Subject: [PATCH 03/18] TD-10431 user drop --- source/dnode/mgmt/impl/test/user/user.cpp | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 5 ++++- source/dnode/mnode/sdb/src/sdbHash.c | 12 ++++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 081229f825..9dc8d99596 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -286,7 +286,7 @@ TEST_F(DndTestUser, AlterUser_01) { SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropUserMsg); + rpcMsg.contLen = sizeof(SAlterUserMsg); rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; sendMsg(pClient, &rpcMsg); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 0a15f03149..3a53472d45 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -263,7 +263,10 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewT return 0; } -static int32_t trnGenerateTransId() { return 1; } +static int32_t trnGenerateTransId() { + static int32_t tmp = 0; + return ++tmp; +} STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { STrans *pTrans = calloc(1, sizeof(STrans)); diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index a852389d80..8d8daf5ce5 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -56,7 +56,8 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (pOldRow != NULL) { taosWUnLockLatch(pLock); sdbFreeRow(pRow); - return TSDB_CODE_SDB_OBJ_ALREADY_THERE; + terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; + return terrno; } pRow->refCount = 1; @@ -65,7 +66,8 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { taosWUnLockLatch(pLock); sdbFreeRow(pRow); - return TSDB_CODE_SDB_OBJ_ALREADY_THERE; + terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; + return terrno; } taosWUnLockLatch(pLock); @@ -78,7 +80,8 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosHashRemove(hash, pRow->pObj, keySize); taosWUnLockLatch(pLock); sdbFreeRow(pRow); - return code; + terrno = code; + return terrno; } } @@ -120,7 +123,8 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (ppOldRow == NULL || *ppOldRow == NULL) { taosWUnLockLatch(pLock); sdbFreeRow(pRow); - return TSDB_CODE_SDB_OBJ_NOT_THERE; + terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; + return terrno; } SSdbRow *pOldRow = *ppOldRow; From 6482b5175d766f15bb631a854243835864dd17d7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 8 Dec 2021 13:48:18 +0800 Subject: [PATCH 04/18] TD-10431 func manage --- include/common/taosmsg.h | 50 ++- include/util/taoserror.h | 13 +- include/util/tdef.h | 11 +- source/dnode/mgmt/impl/src/dndTransport.c | 3 +- source/dnode/mnode/impl/inc/mndDef.h | 21 +- source/dnode/mnode/impl/src/mndFunc.c | 497 +++++++++++++++++++++- source/dnode/mnode/impl/src/mndUser.c | 2 +- source/libs/function/src/tfunction.c | 2 +- source/libs/function/src/tudf.c | 2 +- source/util/src/terror.c | 7 +- 10 files changed, 556 insertions(+), 52 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 2769f8bc7a..6b8d55c458 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -74,7 +74,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_TOPIC, "create-topic" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_TOPIC, "drop-topic" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_TOPIC, "alter-topic" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_FUNCTION, "create-function" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_FUNCTION, "alter-function" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RETRIEVE_FUNCTION, "retrieve-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STABLE, "create-stable" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE, "alter-stable" ) @@ -86,7 +86,6 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_HEARTBEAT, "heartbeat" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW, "show" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW_RETRIEVE, "retrieve" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW_RETRIEVE_FUNC, "retrieve-func" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_VNODE, "compact-vnode" ) // message from client to qnode // message from client to dnode @@ -595,38 +594,45 @@ typedef struct { typedef struct { char name[TSDB_FUNC_NAME_LEN]; - char path[PATH_MAX]; - int32_t funcType; + int8_t funcType; + int8_t scriptType; + int8_t align; int8_t outputType; - int16_t outputLen; + int32_t outputLen; int32_t bufSize; - int32_t codeLen; - char code[]; + int64_t sigature; + int32_t commentSize; + int32_t codeSize; + char pCont[]; } SCreateFuncMsg; typedef struct { - int32_t num; - char name[]; + char name[TSDB_FUNC_NAME_LEN]; +} SDropFuncMsg; + +typedef struct { + int32_t numOfFuncs; + char pFuncNames[]; } SRetrieveFuncMsg; typedef struct { char name[TSDB_FUNC_NAME_LEN]; - int32_t funcType; - int8_t resType; - int16_t resBytes; + int8_t funcType; + int8_t scriptType; + int8_t align; + int8_t outputType; + int32_t outputLen; int32_t bufSize; - int32_t len; - char content[]; -} SFunctionInfoMsg; + int64_t sigature; + int32_t commentSize; + int32_t codeSize; + char pCont[]; +} SFuncInfo; typedef struct { - int32_t num; - char content[]; -} SUdfFuncMsg; - -typedef struct { - char name[TSDB_FUNC_NAME_LEN]; -} SDropFuncMsg; + int32_t numOfFuncs; + char pFuncInfos[]; +} SRetrieveFuncRsp; typedef struct { char db[TSDB_TABLE_FNAME_LEN]; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index f6cefa96df..c0f7d8d328 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -200,12 +200,13 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG TAOS_DEF_ERROR_CODE(0, 0x036E) //"Invalid create table message") #define TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) //"Exceed max row bytes") -#define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0370) //"Invalid func name") -#define TSDB_CODE_MND_INVALID_FUNC_LEN TAOS_DEF_ERROR_CODE(0, 0x0371) //"Invalid func length") -#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x0372) //"Invalid func code") -#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0373) //"Func already exists") -#define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0374) //"Invalid func") -#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) //"Invalid func bufSize") +#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0370) +#define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0371) +#define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0372) +#define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0373) +#define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x0374) +#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x0375) +#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0376) #define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") #define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") diff --git a/include/util/tdef.h b/include/util/tdef.h index 897f51f5c1..165c27067c 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -163,9 +163,14 @@ do { \ #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 33 #define TSDB_FULL_DB_NAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN) -#define TSDB_FUNC_NAME_LEN 65 -#define TSDB_FUNC_CODE_LEN (65535 - 512) -#define TSDB_FUNC_BUF_SIZE 512 + +#define TSDB_FUNC_NAME_LEN 65 +#define TSDB_FUNC_COMMENT_LEN 4096 +#define TSDB_FUNC_CODE_LEN (65535 - 512) +#define TSDB_FUNC_BUF_SIZE 512 +#define TSDB_FUNC_TYPE_SCALAR 1 +#define TSDB_FUNC_TYPE_AGGREGATE 2 + #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_FULL_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) #define TSDB_COL_NAME_LEN 65 diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 98a0b8e308..8aadb97837 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -69,7 +69,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TOPIC] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TOPIC] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_FUNCTION] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_RETRIEVE_FUNCTION] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_STABLE] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_STABLE] = dndProcessMnodeWriteMsg; @@ -81,7 +81,6 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE_FUNC] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE] = dndProcessMnodeWriteMsg; // message from client to dnode diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index cb5c0c2755..00d9f31b15 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -271,16 +271,19 @@ typedef struct SSTableObj { typedef struct SFuncObj { char name[TSDB_FUNC_NAME_LEN]; - char path[128]; - int32_t contLen; - char cont[TSDB_FUNC_CODE_LEN]; - int32_t funcType; - int32_t bufSize; int64_t createdTime; - uint8_t resType; - int16_t resBytes; - int64_t sig; - int16_t type; + int8_t funcType; + int8_t scriptType; + int8_t align; + int8_t outputType; + int32_t outputLen; + int32_t bufSize; + int64_t sigature; + int32_t commentSize; + int32_t codeSize; + char *pComment; + char *pCode; + char pData[]; } SFuncObj; typedef struct SShowObj SShowObj; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index d859da029f..ab2da8e0d5 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -14,8 +14,497 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndFunc.h" +#include "mndShow.h" +#include "mndSync.h" +#include "mndTrans.h" -int32_t mndInitFunc(SMnode *pMnode) { return 0; } -void mndCleanupFunc(SMnode *pMnode) {} \ No newline at end of file +#define SDB_FUNC_VER 1 + +static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc); +static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); +static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc); +static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc); +static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc); +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncMsg *pCreate); +static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc); +static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg); +static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg); +static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter); + +int32_t mndInitFunc(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_FUNC, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndFuncActionEncode, + .decodeFp = (SdbDecodeFp)mndFuncActionDecode, + .insertFp = (SdbInsertFp)mndFuncActionInsert, + .updateFp = (SdbUpdateFp)mndFuncActionUpdate, + .deleteFp = (SdbDeleteFp)mndFuncActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_FUNCTION, mndProcessCreateFuncMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_FUNCTION, mndProcessDropFuncMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_RETRIEVE_FUNCTION, mndProcessRetrieveFuncMsg); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndGetFuncMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndRetrieveFuncs); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndCancelGetNextFunc); + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupFunc(SMnode *pMnode) {} + +static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { + int32_t size = pFunc->commentSize + pFunc->codeSize + sizeof(SFuncObj); + SSdbRaw *pRaw = sdbAllocRaw(SDB_FUNC, SDB_FUNC_VER, size); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN) + SDB_SET_INT64(pRaw, dataPos, pFunc->createdTime) + SDB_SET_INT8(pRaw, dataPos, pFunc->funcType) + SDB_SET_INT8(pRaw, dataPos, pFunc->scriptType) + SDB_SET_INT8(pRaw, dataPos, pFunc->align) + SDB_SET_INT8(pRaw, dataPos, pFunc->outputType) + SDB_SET_INT32(pRaw, dataPos, pFunc->outputLen) + SDB_SET_INT32(pRaw, dataPos, pFunc->bufSize) + SDB_SET_INT64(pRaw, dataPos, pFunc->sigature) + SDB_SET_INT32(pRaw, dataPos, pFunc->commentSize) + SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize) + SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize) + SDB_SET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize) + SDB_SET_DATALEN(pRaw, dataPos); + + return pRaw; +} + +static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != SDB_FUNC_VER) { + mError("failed to decode func since %s", terrstr()); + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + return NULL; + } + + int32_t size = sizeof(SFuncObj) + TSDB_FUNC_COMMENT_LEN + TSDB_FUNC_CODE_LEN; + SSdbRow *pRow = sdbAllocRow(size); + SFuncObj *pFunc = sdbGetRowObj(pRow); + if (pFunc == NULL) return NULL; + char *tmp = (char *)pFunc + sizeof(SFuncObj); + + int32_t dataPos = 0; + SDB_GET_BINARY(pRaw, pRow, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN) + SDB_GET_INT64(pRaw, pRow, dataPos, &pFunc->createdTime) + SDB_GET_INT8(pRaw, pRow, dataPos, &pFunc->funcType) + SDB_GET_INT8(pRaw, pRow, dataPos, &pFunc->scriptType) + SDB_GET_INT8(pRaw, pRow, dataPos, &pFunc->align) + SDB_GET_INT8(pRaw, pRow, dataPos, &pFunc->outputType) + SDB_GET_INT32(pRaw, pRow, dataPos, &pFunc->outputLen) + SDB_GET_INT32(pRaw, pRow, dataPos, &pFunc->bufSize) + SDB_GET_INT64(pRaw, pRow, dataPos, &pFunc->sigature) + SDB_GET_INT32(pRaw, pRow, dataPos, &pFunc->commentSize) + SDB_GET_INT32(pRaw, pRow, dataPos, &pFunc->codeSize) + SDB_GET_BINARY(pRaw, pRow, dataPos, pFunc->pData, pFunc->commentSize + pFunc->codeSize) + pFunc->pComment = pFunc->pData; + pFunc->pCode = (pFunc->pData + pFunc->commentSize); + + return pRow; +} + +static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc) { + mTrace("func:%s, perform insert action", pFunc->name); + return 0; +} + +static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { + mTrace("func:%s, perform delete action", pFunc->name); + return 0; +} + +static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc) { + mTrace("func:%s, perform update action", pOldFunc->name); + return 0; +} + +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncMsg *pCreate) { + SFuncObj *pFunc = calloc(1, sizeof(SFuncObj) + pCreate->commentSize + pCreate->codeSize); + pFunc->createdTime = taosGetTimestampMs(); + pFunc->funcType = pCreate->funcType; + pFunc->scriptType = pCreate->scriptType; + pFunc->outputType = pCreate->outputType; + pFunc->outputLen = pCreate->outputLen; + pFunc->bufSize = pCreate->bufSize; + pFunc->sigature = pCreate->sigature; + pFunc->commentSize = pCreate->commentSize; + pFunc->codeSize = pCreate->codeSize; + pFunc->pComment = pFunc->pData; + memcpy(pFunc->pComment, pCreate->pCont, pCreate->commentSize); + pFunc->pCode = pFunc->pData + pCreate->commentSize; + memcpy(pFunc->pCode, pCreate->pCont + pCreate->commentSize, pFunc->codeSize); + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + free(pFunc); + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + mDebug("trans:%d, used to create func:%s", pTrans->id, pCreate->name); + + SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + free(pFunc); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); + + SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + free(pFunc); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + free(pFunc); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + free(pFunc); + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("func:%s, failed to drop since %s", pFunc->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); + + SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + + SCreateFuncMsg *pCreate = pMsg->rpcMsg.pCont; + pCreate->outputLen = htonl(pCreate->outputLen); + pCreate->bufSize = htonl(pCreate->bufSize); + pCreate->sigature = htobe64(pCreate->sigature); + pCreate->commentSize = htonl(pCreate->commentSize); + pCreate->codeSize = htonl(pCreate->codeSize); + + mDebug("func:%s, start to create", pCreate->name); + + SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pCreate->name); + if (pFunc != NULL) { + sdbRelease(pMnode->pSdb, pFunc); + terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (pCreate->name[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (pCreate->commentSize <= 0 || pCreate->commentSize > TSDB_FUNC_COMMENT_LEN) { + terrno = TSDB_CODE_MND_INVALID_FUNC_COMMENT; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (pCreate->codeSize <= 0 || pCreate->codeSize > TSDB_FUNC_CODE_LEN) { + terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (pCreate->pCont[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (pCreate->bufSize < 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) { + terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + int32_t code = mndCreateFunc(pMnode, pMsg, pCreate); + + if (code != 0) { + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SDropFuncMsg *pDrop = pMsg->rpcMsg.pCont; + + mDebug("func:%s, start to drop", pDrop->name); + + if (pDrop->name[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_FUNC_NAME; + mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pDrop->name); + if (pFunc == NULL) { + terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; + mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + int32_t code = mndDropFunc(pMnode, pMsg, pFunc); + + if (code != 0) { + mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + + SRetrieveFuncMsg *pRetrieve = pMsg->rpcMsg.pCont; + pRetrieve->numOfFuncs = htonl(pRetrieve->numOfFuncs); + + int32_t size = sizeof(SRetrieveFuncRsp) + (sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN) * pRetrieve->numOfFuncs + 16384; + + SRetrieveFuncRsp *pRetrieveRsp = rpcMallocCont(size); + pRetrieveRsp->numOfFuncs = htonl(pRetrieve->numOfFuncs); + char *pOutput = pRetrieveRsp->pFuncInfos; + + for (int32_t i = 0; i < pRetrieve->numOfFuncs; ++i) { + char funcName[TSDB_FUNC_NAME_LEN] = {0}; + memcpy(funcName, pRetrieve->pFuncNames + i * TSDB_FUNC_NAME_LEN, TSDB_FUNC_NAME_LEN); + + SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, funcName); + if (pFunc == NULL) { + terrno = TSDB_CODE_MND_INVALID_FUNC; + mError("func:%s, failed to retrieve since %s", funcName, terrstr()); + return -1; + } + + SFuncInfo *pFuncInfo = (SFuncInfo *)pOutput; + + strncpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN); + pFuncInfo->funcType = pFunc->funcType; + pFuncInfo->scriptType = pFunc->scriptType; + pFuncInfo->outputType = pFunc->outputType; + pFuncInfo->outputLen = htonl(pFunc->outputLen); + pFuncInfo->bufSize = htonl(pFunc->bufSize); + pFuncInfo->sigature = htobe64(pFunc->sigature); + pFuncInfo->commentSize = htonl(pFunc->commentSize); + pFuncInfo->codeSize = htonl(pFunc->codeSize); + memcpy(pFuncInfo->pCont, pFunc->pCode, pFunc->commentSize + pFunc->codeSize); + + pOutput += sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize; + } + + pMsg->pCont = pRetrieveRsp; + pMsg->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp); + + return 0; +} + +static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = PATH_MAX + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "comment"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "aggregate"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "outputtype"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "code_len"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "bufsize"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, "show funcs"); + + return 0; +} + +static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t len) { + char *msg = "unknown"; + if (type >= sizeof(tDataTypes) / sizeof(tDataTypes[0])) { + return msg; + } + + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + int32_t bytes = len > 0 ? (int)(len - VARSTR_HEADER_SIZE) : len; + + snprintf(buf, buflen - 1, "%s(%d)", tDataTypes[type].name, type == TSDB_DATA_TYPE_NCHAR ? bytes / 4 : bytes); + buf[buflen - 1] = 0; + + return buf; + } + + return tDataTypes[type].name; +} + +static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SFuncObj *pFunc = NULL; + int32_t cols = 0; + char *pWrite; + char buf[TSDB_TYPE_STR_MAX_LEN]; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_FUNC, pShow->pIter, (void **)&pFunc); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pFunc->name, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pFunc->pComment, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pFunc->funcType == TSDB_FUNC_TYPE_AGGREGATE ? 1 : 0; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, mnodeGenTypeStr(buf, TSDB_TYPE_STR_MAX_LEN, pFunc->outputType, pFunc->outputLen), + pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pFunc->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pFunc->codeSize; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pFunc->bufSize; + cols++; + + numOfRows++; + sdbRelease(pSdb, pFunc); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index b3c23c0372..4dc114e17a 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -480,7 +480,6 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p cols++; pMeta->numOfColumns = htons(cols); - strcpy(pMeta->tableFname, "show users"); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -490,6 +489,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->numOfRows = sdbGetSize(pSdb, SDB_USER); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, "show users"); return 0; } diff --git a/source/libs/function/src/tfunction.c b/source/libs/function/src/tfunction.c index 9e70b9a68d..36c9e2513f 100644 --- a/source/libs/function/src/tfunction.c +++ b/source/libs/function/src/tfunction.c @@ -343,7 +343,7 @@ bool isProjectionQueryOnSTable(SArray* pFunctionIdList, int32_t tableIndex) { // // if (functionId < 0) { // SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1); -// if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { +// if (pUdfInfo->funcType == TSDB_FUNC_TYPE_AGGREGATE) { // return false; // } // diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 095c5a6bb0..2771a9ecf8 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -146,7 +146,7 @@ void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, i pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes, &pUdfInfo->init); } - if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) { + if (pUdfInfo->funcType == TSDB_FUNC_TYPE_AGGREGATE) { pCtx->resultInfo->numOfRes = output; } else { pCtx->resultInfo->numOfRes += output; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3a36ad9b42..c8904d6248 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -211,11 +211,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message") TAOS_DEFINE_ERROR(TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES, "Exceed max row bytes") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_LEN, "Invalid func length") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_NOT_EXIST, "Func not exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") From 47e1d6954d4f192193a7bfc1e73207a131f84f76 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 8 Dec 2021 14:30:28 +0800 Subject: [PATCH 05/18] TD-10431 dnode manage --- source/dnode/mnode/impl/src/mndDnode.c | 99 ++++++++++++++------------ source/dnode/mnode/impl/src/mndMnode.c | 83 +++++++++++---------- 2 files changed, 101 insertions(+), 81 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d187294b87..d62e1464a6 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -36,6 +36,53 @@ static char *offlineReason[] = { "unknown", }; +static int32_t mndCreateDefaultDnode(SMnode *pMnode); +static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); +static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); +static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); +static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode); +static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg); + +int32_t mndInitDnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_DNODE, + .keyType = SDB_KEY_INT32, + .deployFp = (SdbDeployFp)mndCreateDefaultDnode, + .encodeFp = (SdbEncodeFp)mndDnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndDnodeActionDecode, + .insertFp = (SdbInsertFp)mndDnodeActionInsert, + .updateFp = (SdbUpdateFp)mndDnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndDnodeActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_DNODE, mndProcessCreateDnodeMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_DNODE, mndProcessDropDnodeMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONFIG_DNODE, mndProcessConfigDnodeMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_STATUS, mndProcessStatusMsg); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupDnode(SMnode *pMnode) {} + +static int32_t mndCreateDefaultDnode(SMnode *pMnode) { + SDnodeObj dnodeObj = {0}; + dnodeObj.id = 1; + dnodeObj.createdTime = taosGetTimestampMs(); + dnodeObj.updateTime = dnodeObj.createdTime; + dnodeObj.port = pMnode->replicas[0].port; + memcpy(&dnodeObj.fqdn, pMnode->replicas[0].fqdn, TSDB_FQDN_LEN); + + SSdbRaw *pRaw = mndDnodeActionEncode(&dnodeObj); + if (pRaw == NULL) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + + mDebug("dnode:%d, will be created while deploy sdb", dnodeObj.id); + return sdbWrite(pMnode->pSdb, pRaw); +} + static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, SDB_DNODE_VER, sizeof(SDnodeObj)); if (pRaw == NULL) return NULL; @@ -107,24 +154,18 @@ static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj pOldDnode->createdTime = pNewDnode->createdTime; pOldDnode->updateTime = pNewDnode->updateTime; pOldDnode->port = pNewDnode->port; - memcpy(pOldDnode->fqdn, pNewDnode->fqdn, TSDB_FQDN_LEN); + memcpy(pOldDnode->fqdn, pNewDnode->fqdn, TSDB_FQDN_LEN); return 0; } -static int32_t mndCreateDefaultDnode(SMnode *pMnode) { - SDnodeObj dnodeObj = {0}; - dnodeObj.id = 1; - dnodeObj.createdTime = taosGetTimestampMs(); - dnodeObj.updateTime = dnodeObj.createdTime; - dnodeObj.port = pMnode->replicas[0].port; - memcpy(&dnodeObj.fqdn, pMnode->replicas[0].fqdn, TSDB_FQDN_LEN); +SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_DNODE, &dnodeId); +} - SSdbRaw *pRaw = mndDnodeActionEncode(&dnodeObj); - if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - - mDebug("dnode:%d, will be created while deploy sdb", dnodeObj.id); - return sdbWrite(pMnode->pSdb, pRaw); +void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pDnode); } static SDnodeObj *mndAcquireDnodeByEp(SMnode *pMnode, char *pEpStr) { @@ -322,33 +363,3 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { return 0; } - -int32_t mndInitDnode(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_DNODE, - .keyType = SDB_KEY_INT32, - .deployFp = (SdbDeployFp)mndCreateDefaultDnode, - .encodeFp = (SdbEncodeFp)mndDnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndDnodeActionDecode, - .insertFp = (SdbInsertFp)mndDnodeActionInsert, - .updateFp = (SdbUpdateFp)mndDnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndDnodeActionDelete}; - - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_DNODE, mndProcessCreateDnodeMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_DNODE, mndProcessDropDnodeMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONFIG_DNODE, mndProcessConfigDnodeMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_STATUS, mndProcessStatusMsg); - - return sdbSetTable(pMnode->pSdb, table); -} - -void mndCleanupDnode(SMnode *pMnode) {} - -SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_DNODE, &dnodeId); -} - -void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode) { - SSdb *pSdb = pMnode->pSdb; - sdbRelease(pSdb, pDnode); -} diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 2490f1fcae..be66db3b01 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -18,6 +18,47 @@ #define SDB_MNODE_VER 1 +static int32_t mndCreateDefaultMnode(SMnode *pMnode); +static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj); +static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw); +static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pMnodeObj); +static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pMnodeObj); +static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOldMnode, SMnodeObj *pNewMnode); +static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg); + +int32_t mndInitMnode(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_MNODE, + .keyType = SDB_KEY_INT32, + .deployFp = (SdbDeployFp)mndCreateDefaultMnode, + .encodeFp = (SdbEncodeFp)mndMnodeActionEncode, + .decodeFp = (SdbDecodeFp)mndMnodeActionDecode, + .insertFp = (SdbInsertFp)mndMnodeActionInsert, + .updateFp = (SdbUpdateFp)mndMnodeActionUpdate, + .deleteFp = (SdbDeleteFp)mndMnodeActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE, mndProcessCreateMnodeMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE, mndProcessDropMnodeMsg); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupMnode(SMnode *pMnode) {} + +static int32_t mndCreateDefaultMnode(SMnode *pMnode) { + SMnodeObj mnodeObj = {0}; + mnodeObj.id = 1; + mnodeObj.createdTime = taosGetTimestampMs(); + mnodeObj.updateTime = mnodeObj.createdTime; + + SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj); + if (pRaw == NULL) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + + mDebug("mnode:%d, will be created while deploy sdb", mnodeObj.id); + return sdbWrite(pMnode->pSdb, pRaw); +} + static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj) { SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, SDB_MNODE_VER, sizeof(SMnodeObj)); if (pRaw == NULL) return NULL; @@ -89,42 +130,6 @@ static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOldMnode, SMnodeObj return 0; } -static int32_t mndCreateDefaultMnode(SMnode *pMnode) { - SMnodeObj mnodeObj = {0}; - mnodeObj.id = 1; - mnodeObj.createdTime = taosGetTimestampMs(); - mnodeObj.updateTime = mnodeObj.createdTime; - - SSdbRaw *pRaw = mndMnodeActionEncode(&mnodeObj); - if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - - mDebug("mnode:%d, will be created while deploy sdb", mnodeObj.id); - return sdbWrite(pMnode->pSdb, pRaw); -} - -static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) { return 0; } - -static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) { return 0; } - -int32_t mndInitMnode(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_MNODE, - .keyType = SDB_KEY_INT32, - .deployFp = (SdbDeployFp)mndCreateDefaultMnode, - .encodeFp = (SdbEncodeFp)mndMnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndMnodeActionDecode, - .insertFp = (SdbInsertFp)mndMnodeActionInsert, - .updateFp = (SdbUpdateFp)mndMnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndMnodeActionDelete}; - - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE, mndProcessCreateMnodeMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE, mndProcessDropMnodeMsg); - - return sdbSetTable(pMnode->pSdb, table); -} - -void mndCleanupMnode(SMnode *pMnode) {} - bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { SSdb *pSdb = pMnode->pSdb; @@ -157,4 +162,8 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { pEpSet->numOfEps++; } -} \ No newline at end of file +} + +static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) { return 0; } From 23da7d4230211e26e052a755fb23becab097e3e8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 8 Dec 2021 18:50:52 +0800 Subject: [PATCH 06/18] TD-10431 create dnode --- include/common/taosmsg.h | 16 +- include/common/tglobal.h | 1 - include/util/tutil.h | 23 +- source/common/src/tglobal.c | 18 - source/dnode/mnode/impl/inc/mndDef.h | 32 +- source/dnode/mnode/impl/inc/mndDnode.h | 1 + source/dnode/mnode/impl/inc/mndShow.h | 1 + source/dnode/mnode/impl/inc/mndVgroup.h | 1 + source/dnode/mnode/impl/src/mndDnode.c | 428 +++++++++++++++++++++++- source/dnode/mnode/impl/src/mndShow.c | 7 +- source/dnode/mnode/impl/src/mndUser.c | 7 +- source/dnode/mnode/impl/src/mndVgroup.c | 114 ++++++- source/util/src/tutil.c | 21 +- 13 files changed, 589 insertions(+), 81 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 6b8d55c458..08d8eacce9 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -829,7 +829,16 @@ typedef struct SShowRsp { typedef struct { char ep[TSDB_EP_LEN]; // end point, hostname:port -} SCreateDnodeMsg, SDropDnodeMsg; +} SCreateDnodeMsg; + +typedef struct { + int32_t dnodeId; +} SDropDnodeMsg; + +typedef struct { + int32_t dnodeId; + char config[128]; +} SCfgDnodeMsg; typedef struct { int32_t dnodeId; @@ -849,11 +858,6 @@ typedef struct { int32_t vgId; } SConfigVnodeMsg; -typedef struct { - char ep[TSDB_EP_LEN]; // end point, hostname:port - char config[64]; -} SCfgDnodeMsg; - typedef struct { char sql[TSDB_SHOW_SQL_LEN]; int32_t queryId; diff --git a/include/common/tglobal.h b/include/common/tglobal.h index d0f95b786a..e62083e999 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -196,7 +196,6 @@ extern SDiskCfg tsDiskCfg[]; void taosInitGlobalCfg(); int32_t taosCheckGlobalCfg(); int32_t taosCfgDynamicOptions(char *msg); -int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); void taosAddDataDir(int index, char *v1, int level, int primary); void taosReadDataDirCfg(char *v1, char *v2, char *v3); diff --git a/include/util/tutil.h b/include/util/tutil.h index 573dee9339..41eb2ca26b 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -21,22 +21,22 @@ extern "C" { #endif #include "os.h" -#include "tmd5.h" #include "tcrc32c.h" #include "tdef.h" +#include "tmd5.h" int32_t strdequote(char *src); -int32_t strndequote(char *dst, const char* z, int32_t len); +int32_t strndequote(char *dst, const char *z, int32_t len); int32_t strRmquote(char *z, int32_t len); size_t strtrim(char *src); -char * strnchr(char *haystack, char needle, int32_t len, bool skipquote); -char ** strsplit(char *src, const char *delim, int32_t *num); -char * strtolower(char *dst, const char *src); -char * strntolower(char *dst, const char *src, int32_t n); -char * strntolower_s(char *dst, const char *src, int32_t n); +char *strnchr(char *haystack, char needle, int32_t len, bool skipquote); +char **strsplit(char *src, const char *delim, int32_t *num); +char *strtolower(char *dst, const char *src); +char *strntolower(char *dst, const char *src, int32_t n); +char *strntolower_s(char *dst, const char *src, int32_t n); int64_t strnatoi(char *num, int32_t len); -char * strbetween(char *string, char *begin, char *end); -char * paGetToken(char *src, char **token, int32_t *tokenLen); +char *strbetween(char *string, char *begin, char *end); +char *paGetToken(char *src, char **token, int32_t *tokenLen); int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); @@ -45,11 +45,12 @@ char *taosIpStr(uint32_t ipInt); uint32_t ip2uint(const char *const ip_addr); void taosIp2String(uint32_t ip, char *str); void taosIpPort2String(uint32_t ip, uint16_t port, char *str); +int32_t taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { MD5_CTX context; MD5Init(&context); - MD5Update(&context, inBuf, (unsigned int)inLen); + MD5Update(&context, inBuf, (uint32_t)inLen); MD5Final(&context); memcpy(target, context.digest, TSDB_KEY_LEN); } @@ -58,4 +59,4 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *tar } #endif -#endif /*_TD_UTIL_UTIL_H*/ +#endif /*_TD_UTIL_UTIL_H*/ diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 1737bd9def..117f908faf 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1738,24 +1738,6 @@ int32_t taosCheckGlobalCfg() { return 0; } -int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { - *port = 0; - strcpy(fqdn, ep); - - char *temp = strchr(fqdn, ':'); - if (temp) { - *temp = 0; - *port = atoi(temp+1); - } - - if (*port == 0) { - *port = tsServerPort; - return -1; - } - - return 0; -} - /* * alter dnode 1 balance "vnode:1-dnode:2" */ diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 00d9f31b15..8840df848f 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -286,23 +286,21 @@ typedef struct SFuncObj { char pData[]; } SFuncObj; -typedef struct SShowObj SShowObj; -typedef struct SShowObj { - int8_t type; - int8_t maxReplica; - int16_t numOfColumns; - int32_t id; - int32_t rowSize; - int32_t numOfRows; - int32_t numOfReads; - uint16_t payloadLen; - void *pIter; - void *pVgIter; - SMnode *pMnode; - char db[TSDB_FULL_DB_NAME_LEN]; - int16_t offset[TSDB_MAX_COLUMNS]; - int32_t bytes[TSDB_MAX_COLUMNS]; - char payload[]; +typedef struct { + int32_t id; + int8_t type; + int8_t replica; + int16_t numOfColumns; + int32_t rowSize; + int32_t numOfRows; + int32_t numOfReads; + int32_t payloadLen; + void *pIter; + SMnode *pMnode; + char db[TSDB_FULL_DB_NAME_LEN]; + int16_t offset[TSDB_MAX_COLUMNS]; + int32_t bytes[TSDB_MAX_COLUMNS]; + char payload[]; } SShowObj; typedef struct SMnodeMsg { diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index 9bb1ab7acd..a25784e5bb 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -26,6 +26,7 @@ int32_t mndInitDnode(SMnode *pMnode); void mndCleanupDnode(SMnode *pMnode); SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId); void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode); +SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index e7a3fcd45f..37fb9159dc 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -28,6 +28,7 @@ void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp); void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp); void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType msgType, ShowFreeIterFp fp); void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); +char *mndShowStr(int32_t showType); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index c75bdb5949..5509c81a2e 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -24,6 +24,7 @@ extern "C" { int32_t mndInitVgroup(SMnode *pMnode); void mndCleanupVgroup(SMnode *pMnode); +int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d62e1464a6..358b07c967 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -16,12 +16,17 @@ #define _DEFAULT_SOURCE #include "mndDnode.h" #include "mndMnode.h" +#include "mndShow.h" #include "mndTrans.h" #include "ttime.h" +#include "tutil.h" -#define SDB_DNODE_VER 1 +#define TSDB_DNODE_VER 1 +#define TSDB_CONFIG_OPTION_LEN 16 +#define TSDB_CONIIG_VALUE_LEN 48 +#define TSDB_CONFIG_NUMBER 8 -static char *offlineReason[] = { +static const char *offlineReason[] = { "", "status msg timeout", "status not received", @@ -36,16 +41,27 @@ static char *offlineReason[] = { "unknown", }; +static const char *dnodeStatus[] = {"offline", "ready", "creating", "dropping"}; + static int32_t mndCreateDefaultDnode(SMnode *pMnode); static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOldDnode, SDnodeObj *pNewDnode); -static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg); -static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg); -static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg); + +static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg); + +static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveConfigs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); +static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); int32_t mndInitDnode(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_DNODE, @@ -60,8 +76,16 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_DNODE, mndProcessCreateDnodeMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_DNODE, mndProcessDropDnodeMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONFIG_DNODE, mndProcessConfigDnodeMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONFIG_DNODE_IN_RSP, mndProcessConfigDnodeRsp); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_STATUS, mndProcessStatusMsg); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_VARIABLES, mndGetConfigMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VARIABLES, mndRetrieveConfigs); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VARIABLES, mndCancelGetNextConfig); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DNODE, mndGetDnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DNODE, mndRetrieveDnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DNODE, mndCancelGetNextDnode); + return sdbSetTable(pMnode->pSdb, table); } @@ -84,7 +108,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { } static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, SDB_DNODE_VER, sizeof(SDnodeObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj)); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -102,7 +126,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_DNODE_VER) { + if (sver != TSDB_DNODE_VER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to decode dnode since %s", terrstr()); return NULL; @@ -168,6 +192,12 @@ void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode) { sdbRelease(pSdb, pDnode); } +SEpSet mndGetDnodeEpset(SDnodeObj *pDnode) { + SEpSet epSet = {.inUse = 0, .numOfEps = 1, .port[0] = pDnode->port}; + memcpy(epSet.fqdn[0], pDnode->fqdn, TSDB_FQDN_LEN); + return epSet; +} + static SDnodeObj *mndAcquireDnodeByEp(SMnode *pMnode, char *pEpStr) { SSdb *pSdb = pMnode->pSdb; @@ -358,8 +388,384 @@ static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { return 0; } +static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *pCreate) { + SDnodeObj dnodeObj = {0}; + dnodeObj.id = 1; // todo + dnodeObj.createdTime = taosGetTimestampMs(); + dnodeObj.updateTime = dnodeObj.createdTime; + taosGetFqdnPortFromEp(pCreate->ep, dnodeObj.fqdn, &dnodeObj.port); -static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { return 0; } + if (dnodeObj.fqdn[0] == 0 || dnodeObj.port <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); + return terrno; + } -static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { return 0; } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); + return -1; + } + mDebug("trans:%d, used to create dnode:%s", pTrans->id, pCreate->ep); + + SSdbRaw *pRedoRaw = mndDnodeActionEncode(&dnodeObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); + + SSdbRaw *pUndoRaw = mndDnodeActionEncode(&dnodeObj); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + SSdbRaw *pCommitRaw = mndDnodeActionEncode(&dnodeObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + mDebug("dnode:%s, start to create", pCreate->ep); + + if (pCreate->ep[0] == 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnodeByEp(pMnode, pCreate->ep); + if (pDnode != NULL) { + mError("dnode:%d, already exist", pDnode->id); + sdbRelease(pMnode->pSdb, pDnode); + terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST; + return -1; + } + + int32_t code = mndCreateDnode(pMnode, pMsg, pCreate); + + if (code != 0) { + mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndDropDnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("dnode:%d, failed to drop since %s", pDnode->id, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop user:%d", pTrans->id, pDnode->id); + + SSdbRaw *pRedoRaw = mndDnodeActionEncode(pDnode); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndDnodeActionEncode(pDnode); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndDnodeActionEncode(pDnode); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SDropDnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("dnode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("dnode:%d, failed to create since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDrop->dnodeId); + if (pDnode == NULL) { + mError("dnode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropDnode(pMnode, pMsg, pDnode); + + if (code != 0) { + mError("dnode:%d, failed to create since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pDnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCfgDnodeMsg *pCfg = pMsg->rpcMsg.pCont; + pCfg->dnodeId = htonl(pCfg->dnodeId); + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCfg->dnodeId); + if (pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + mError("dnode:%d, failed to cfg since %s ", pCfg->dnodeId, terrstr()); + return -1; + } + + SEpSet epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + SCfgDnodeMsg *pCfgDnode = rpcMallocCont(sizeof(SCfgDnodeMsg)); + pCfgDnode->dnodeId = htonl(pCfg->dnodeId); + memcpy(pCfgDnode->config, pCfg->config, 128); + + SRpcMsg rpcMsg = {.msgType = TSDB_MSG_TYPE_CONFIG_DNODE_IN, .pCont = pCfgDnode, .contLen = sizeof(SCfgDnodeMsg)}; + + mInfo("dnode:%d, is configured", pCfg->dnodeId); + mndSendMsgToDnode(pMnode, &epSet, &rpcMsg); + + return 0; +} + +static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg) { mInfo("cfg dnode rsp is received"); } + +static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = TSDB_CONFIG_NUMBER; + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + pShow->pIter = NULL; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveConfigs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + int32_t numOfRows = 0; + char *cfgOpts[TSDB_CONFIG_NUMBER] = {0}; + char cfgVals[TSDB_CONFIG_NUMBER][TSDB_CONIIG_VALUE_LEN + 1] = {0}; + char *pWrite; + int32_t cols = 0; + + cfgOpts[numOfRows] = "statusInterval"; + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%d", pMnode->cfg.statusInterval); + numOfRows++; + + cfgOpts[numOfRows] = "timezone"; + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.timezone); + numOfRows++; + + cfgOpts[numOfRows] = "locale"; + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.locale); + numOfRows++; + + cfgOpts[numOfRows] = "charset"; + snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.charset); + numOfRows++; + + for (int32_t i = 0; i < numOfRows; i++) { + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfgOpts[i], TSDB_CONFIG_OPTION_LEN); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfgVals[i], TSDB_CONIIG_VALUE_LEN); + cols++; + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {} + +static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "end point"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "vnodes"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "cores"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "status"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "offline reason"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_DNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SDnodeObj *pDnode = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_DNODE, pShow->pIter, (void **)&pDnode); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->ep, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->numOfVnodes; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->numOfCores; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + const char *status = dnodeStatus[pDnode->status]; + STR_TO_VARSTR(pWrite, status); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pDnode->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, offlineReason[pDnode->offlineReason]); + cols++; + + numOfRows++; + sdbRelease(pSdb, pDnode); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 747ea39237..ac33d89517 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -20,7 +20,6 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); -static char *mndShowStr(int32_t showType); static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg); static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMsg); static bool mndCheckRetrieveFinished(SShowObj *pShow); @@ -88,10 +87,6 @@ static void mndFreeShowObj(SShowObj *pShow) { ShowFreeIterFp freeFp = pMgmt->freeIterFps[pShow->type]; if (freeFp != NULL) { - if (pShow->pVgIter != NULL) { - // only used in 'show vnodes "ep"' - (*freeFp)(pMnode, pShow->pVgIter); - } if (pShow->pIter != NULL) { (*freeFp)(pMnode, pShow->pIter); } @@ -259,7 +254,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { return TSDB_CODE_SUCCESS; } -static char *mndShowStr(int32_t showType) { +char *mndShowStr(int32_t showType) { switch (showType) { case TSDB_MGMT_TABLE_ACCT: return "show accounts"; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 4dc114e17a..0be3ffc80b 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -277,7 +277,7 @@ static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOldUser, SUserObj *pNewU return 0; } -static int32_t mndDropUser(SMnode *pMnode, SUserObj *pUser, SMnodeMsg *pMsg) { +static int32_t mndDropUser(SMnode *pMnode, SMnodeMsg *pMsg, SUserObj *pUser) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("user:%s, failed to drop since %s", pUser->user, terrstr()); @@ -437,7 +437,7 @@ static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg) { return -1; } - int32_t code = mndDropUser(pMnode, pUser, pMsg); + int32_t code = mndDropUser(pMnode, pMsg, pUser); sdbRelease(pMnode->pSdb, pOperUser); if (code != 0) { @@ -489,7 +489,8 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->numOfRows = sdbGetSize(pSdb, SDB_USER); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tableFname, "show users"); + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index b98468b63e..eb820fa3c2 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -14,8 +14,114 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndVgroup.h" +#include "mndDnode.h" +#include "mndShow.h" +#include "mndTrans.h" +#include "ttime.h" -int32_t mndInitVgroup(SMnode *pMnode) { return 0; } -void mndCleanupVgroup(SMnode *pMnode) {} \ No newline at end of file +static char *syncRole[] = {"unsynced", "slave", "master"}; + +static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); + +int32_t mndInitVgroup(SMnode *pMnode) { + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndGetVnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndRetrieveVnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndCancelGetNextVnode); + + return 0; +} + +void mndCleanupVgroup(SMnode *pMnode) {} + +int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { + if (dnodeId == 0) { + return 0; + } + + return 0; +} + +static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "vgId"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "status"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + int32_t dnodeId = 0; + if (pShow->payloadLen > 0) { + dnodeId = atoi(pShow->payload); + } + + pShow->replica = dnodeId; + pShow->numOfRows = mndGetVnodesNum(pMnode, dnodeId); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SVgObj *pVgroup = NULL; + char *pWrite; + int32_t cols = 0; + int32_t dnodeId = pShow->replica; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); + if (pShow->pIter == NULL) break; + + for (int32_t i = 0; i < pVgroup->numOfVnodes && numOfRows < rows; ++i) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + if (pVgid->dnodeId != dnodeId) continue; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(uint32_t *)pWrite = pVgroup->vgId; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, syncRole[pVgid->role]); + cols++; + numOfRows++; + } + + sdbRelease(pSdb, pVgroup); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index b9d2da6939..3c3c8a1dfc 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -187,7 +187,7 @@ char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) { } char* strtolower(char *dst, const char *src) { - int esc = 0; + int32_t esc = 0; char quote = 0, *p = dst, c; assert(dst != NULL); @@ -214,7 +214,7 @@ char* strtolower(char *dst, const char *src) { } char* strntolower(char *dst, const char *src, int32_t n) { - int esc = 0; + int32_t esc = 0; char quote = 0, *p = dst, c; assert(dst != NULL); @@ -347,7 +347,7 @@ char *strbetween(char *string, char *begin, char *end) { char *_begin = strstr(string, begin); if (_begin != NULL) { char *_end = strstr(_begin + strlen(begin), end); - int size = (int)(_end - _begin); + int32_t size = (int32_t)(_end - _begin); if (_end != NULL && size > 0) { result = (char *)calloc(1, size); memcpy(result, _begin + strlen(begin), size - +strlen(begin)); @@ -402,7 +402,7 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) { char *taosIpStr(uint32_t ipInt) { static char ipStrArray[3][30]; - static int ipStrIndex = 0; + static int32_t ipStrIndex = 0; char *ipStr = ipStrArray[(ipStrIndex++) % 3]; //sprintf(ipStr, "0x%x:%u.%u.%u.%u", ipInt, ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); @@ -416,4 +416,17 @@ void taosIp2String(uint32_t ip, char *str) { void taosIpPort2String(uint32_t ip, uint16_t port, char *str) { sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port); +} + +int32_t taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { + *port = 0; + strcpy(fqdn, ep); + + char *temp = strchr(fqdn, ':'); + if (temp) { + *temp = 0; + *port = atoi(temp + 1); + } + + return 0; } \ No newline at end of file From 722ffd6814c6fd2f1cc5b6a1b0ac7406cb88cada Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 8 Dec 2021 19:34:54 +0800 Subject: [PATCH 07/18] TD-10431 mnode manage --- include/common/taosmsg.h | 12 +- source/dnode/mgmt/impl/src/dndMnode.c | 13 +- source/dnode/mnode/impl/inc/mndMnode.h | 1 + source/dnode/mnode/impl/src/mndDnode.c | 5 +- source/dnode/mnode/impl/src/mndMnode.c | 304 +++++++++++++++++++++++- source/dnode/mnode/impl/src/mndVgroup.c | 5 +- 6 files changed, 325 insertions(+), 15 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 08d8eacce9..5c1787472a 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -840,12 +840,20 @@ typedef struct { char config[128]; } SCfgDnodeMsg; +typedef struct { + int32_t dnodeId; +} SCreateMnodeMsg, SDropMnodeMsg; + typedef struct { int32_t dnodeId; + int8_t align[3]; int8_t replica; - int8_t reserved[3]; SReplica replicas[TSDB_MAX_REPLICA]; -} SCreateMnodeMsg, SAlterMnodeMsg, SDropMnodeMsg; +} SCreateMnodeInMsg, SAlterMnodeInMsg; + +typedef struct { + int32_t dnodeId; +} SDropMnodeInMsg; typedef struct { int32_t dnodeId; diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 374a2f2b2c..d3336fb079 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -366,7 +366,7 @@ static void dndBuildMnodeOpenOption(SDnode *pDnode, SMnodeOpt *pOption) { memcpy(&pOption->replicas, pMgmt->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); } -static int32_t dndBuildMnodeOptionFromMsg(SDnode *pDnode, SMnodeOpt *pOption, SCreateMnodeMsg *pMsg) { +static int32_t dndBuildMnodeOptionFromMsg(SDnode *pDnode, SMnodeOpt *pOption, SCreateMnodeInMsg *pMsg) { dndInitMnodeOption(pDnode, pOption); pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); @@ -488,8 +488,8 @@ static int32_t dndDropMnode(SDnode *pDnode) { return 0; } -static SCreateMnodeMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { - SCreateMnodeMsg *pMsg = pRpcMsg->pCont; +static SCreateMnodeInMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { + SCreateMnodeInMsg *pMsg = pRpcMsg->pCont; pMsg->dnodeId = htonl(pMsg->dnodeId); for (int32_t i = 0; i < pMsg->replica; ++i) { pMsg->replicas[i].id = htonl(pMsg->replicas[i].id); @@ -500,7 +500,7 @@ static SCreateMnodeMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) { } static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SCreateMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); + SCreateMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_ID_INVALID; @@ -516,7 +516,7 @@ static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } static int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SAlterMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); + SAlterMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_ID_INVALID; @@ -531,7 +531,8 @@ static int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } static int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDropMnodeMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont); + SDropMnodeInMsg *pMsg = pRpcMsg->pCont; + pMsg->dnodeId = htonl(pMsg->dnodeId); if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_ID_INVALID; diff --git a/source/dnode/mnode/impl/inc/mndMnode.h b/source/dnode/mnode/impl/inc/mndMnode.h index 64bbb13b60..906d11aec2 100644 --- a/source/dnode/mnode/impl/inc/mndMnode.h +++ b/source/dnode/mnode/impl/inc/mndMnode.h @@ -26,6 +26,7 @@ int32_t mndInitMnode(SMnode *pMnode); void mndCleanupMnode(SMnode *pMnode); bool mndIsMnode(SMnode *pMnode, int32_t dnodeId); void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet); +char *mndGetRoleStr(int32_t role); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 358b07c967..c34d12c6e2 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -523,7 +523,7 @@ static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { if (pDrop->dnodeId <= 0) { terrno = TSDB_CODE_SDB_APP_ERROR; - mError("dnode:%d, failed to create since %s", pDrop->dnodeId, terrstr()); + mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); return -1; } @@ -537,7 +537,7 @@ static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { int32_t code = mndDropDnode(pMnode, pMsg, pDnode); if (code != 0) { - mError("dnode:%d, failed to create since %s", pDrop->dnodeId, terrstr()); + mError("dnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); return -1; } @@ -762,6 +762,7 @@ static int32_t mndRetrieveDnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); pShow->numOfReads += numOfRows; + return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index be66db3b01..d295e232a3 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -14,6 +14,9 @@ */ #define _DEFAULT_SOURCE +#include "mndMnode.h" +#include "mndDnode.h" +#include "mndShow.h" #include "mndTrans.h" #define SDB_MNODE_VER 1 @@ -26,6 +29,11 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pMnodeObj); static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOldMnode, SMnodeObj *pNewMnode); static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg); +static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pMsg); +static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveMnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter); int32_t mndInitMnode(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_MNODE, @@ -39,12 +47,41 @@ int32_t mndInitMnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE, mndProcessCreateMnodeMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE, mndProcessDropMnodeMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE_IN_RSP, mndProcessCreateMnodeRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE_IN_RSP, mndProcessDropMnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndGetMnodeMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndRetrieveMnodes); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndCancelGetNextMnode); return sdbSetTable(pMnode->pSdb, table); } void mndCleanupMnode(SMnode *pMnode) {} +static SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_MNODE, &mnodeId); +} + +static void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pMnodeObj) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pMnodeObj); +} + +char *mndGetRoleStr(int32_t showType) { + switch (showType) { + case TAOS_SYNC_STATE_FOLLOWER: + return "unsynced"; + case TAOS_SYNC_STATE_CANDIDATE: + return "slave"; + case TAOS_SYNC_STATE_LEADER: + return "master"; + default: + return "undefined"; + } +} + static int32_t mndCreateDefaultMnode(SMnode *pMnode) { SMnodeObj mnodeObj = {0}; mnodeObj.id = 1; @@ -164,6 +201,269 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { } } -static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) { return 0; } +static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateMnodeMsg *pCreate) { + SMnodeObj mnodeObj = {0}; + mnodeObj.id = 1; // todo + mnodeObj.createdTime = taosGetTimestampMs(); + mnodeObj.updateTime = mnodeObj.createdTime; -static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) { return 0; } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("dnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + mDebug("trans:%d, used to create dnode:%d", pTrans->id, pCreate->dnodeId); + + SSdbRaw *pRedoRaw = mndMnodeActionEncode(&mnodeObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); + + SSdbRaw *pUndoRaw = mndMnodeActionEncode(&mnodeObj); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + SSdbRaw *pCommitRaw = mndMnodeActionEncode(&mnodeObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCreateMnodeMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->dnodeId = htonl(pCreate->dnodeId); + + mDebug("mnode:%d, start to create", pCreate->dnodeId); + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId); + if (pDnode == NULL) { + mError("mnode:%d, dnode not exist", pDnode->id); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + mndReleaseDnode(pMnode, pDnode); + + SMnodeObj *pMnodeObj = mndAcquireMnode(pMnode, pCreate->dnodeId); + if (pMnodeObj != NULL) { + mError("mnode:%d, mnode already exist", pMnodeObj->id); + terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST; + return -1; + } + + int32_t code = mndCreateMnode(pMnode, pMsg, pCreate); + + if (code != 0) { + mError("mnode:%d, failed to create since %s", pCreate->dnodeId, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndDropMnode(SMnode *pMnode, SMnodeMsg *pMsg, SMnodeObj *pMnodeObj) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("mnode:%d, failed to drop since %s", pMnodeObj->id, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop user:%d", pTrans->id, pMnodeObj->id); + + SSdbRaw *pRedoRaw = mndMnodeActionEncode(pMnodeObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndMnodeActionEncode(pMnodeObj); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndMnodeActionEncode(pMnodeObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SDropMnodeMsg *pDrop = pMsg->rpcMsg.pCont; + pDrop->dnodeId = htonl(pDrop->dnodeId); + + mDebug("mnode:%d, start to drop", pDrop->dnodeId); + + if (pDrop->dnodeId <= 0) { + terrno = TSDB_CODE_SDB_APP_ERROR; + mError("mnode:%d, failed to drop since %s", pDrop->dnodeId, terrstr()); + return -1; + } + + SMnodeObj *pMnodeObj = mndAcquireMnode(pMnode, pDrop->dnodeId); + if (pMnodeObj == NULL) { + mError("mnode:%d, not exist", pDrop->dnodeId); + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + return -1; + } + + int32_t code = mndDropMnode(pMnode, pMsg, pMnodeObj); + + if (code != 0) { + mError("mnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr()); + return -1; + } + + sdbRelease(pMnode->pSdb, pMnode); + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "end point"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "role"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "role time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_MNODE); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveMnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SMnodeObj *pMnodeObj = NULL; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_MNODE, pShow->pIter, (void **)&pMnodeObj); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pMnodeObj->id; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pMnodeObj->id); + if (pDnode != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->ep, pShow->bytes[cols]); + } else { + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, "invalid ep", pShow->bytes[cols]); + } + mndReleaseDnode(pMnode, pDnode); + + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + char *roles = mndGetRoleStr(pMnodeObj->role); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, roles, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pMnodeObj->roleTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pMnodeObj->createdTime; + cols++; + + numOfRows++; + sdbRelease(pSdb, pMnodeObj); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextMnode(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index eb820fa3c2..68ee373d8c 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -16,12 +16,11 @@ #define _DEFAULT_SOURCE #include "mndVgroup.h" #include "mndDnode.h" +#include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" #include "ttime.h" -static char *syncRole[] = {"unsynced", "slave", "master"}; - static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); @@ -108,7 +107,7 @@ static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, syncRole[pVgid->role]); + STR_TO_VARSTR(pWrite, mndGetRoleStr(pVgid->role)); cols++; numOfRows++; } From 34b10f850626ee1fc29bd64c58518b2637cd4ba0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 8 Dec 2021 19:38:44 +0800 Subject: [PATCH 08/18] minor changes --- source/dnode/mnode/impl/inc/mndOper.h | 30 --------------------------- source/dnode/mnode/impl/src/mndOper.c | 21 ------------------- source/dnode/mnode/impl/src/mnode.c | 1 - 3 files changed, 52 deletions(-) delete mode 100644 source/dnode/mnode/impl/inc/mndOper.h delete mode 100644 source/dnode/mnode/impl/src/mndOper.c diff --git a/source/dnode/mnode/impl/inc/mndOper.h b/source/dnode/mnode/impl/inc/mndOper.h deleted file mode 100644 index 5ad5059a0f..0000000000 --- a/source/dnode/mnode/impl/inc/mndOper.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_MND_OPER_H_ -#define _TD_MND_OPER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -int32_t mndInitOper(SMnode *pMnode); -void mndCleanupOper(SMnode *pMnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_MND_OPER_H_*/ diff --git a/source/dnode/mnode/impl/src/mndOper.c b/source/dnode/mnode/impl/src/mndOper.c deleted file mode 100644 index 4416a1fc2d..0000000000 --- a/source/dnode/mnode/impl/src/mndOper.c +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" - -int32_t mndInitOper(SMnode *pMnode) { return 0; } -void mndCleanupOper(SMnode *pMnode) {} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 9ffe8968ae..3e0ac746fc 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -22,7 +22,6 @@ #include "mndDnode.h" #include "mndFunc.h" #include "mndMnode.h" -#include "mndOper.h" #include "mndProfile.h" #include "mndShow.h" #include "mndStable.h" From e1baf72a3ee9b4dfa9d949e22a41ed951d24ea52 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 8 Dec 2021 20:23:16 +0800 Subject: [PATCH 09/18] TD-10431 db manage --- include/common/taosmsg.h | 5 +- source/dnode/mgmt/impl/src/dndTransport.c | 5 +- source/dnode/mnode/impl/inc/mndDef.h | 21 +- source/dnode/mnode/impl/src/mndDb.c | 389 +++++++++++++++++++++- 4 files changed, 394 insertions(+), 26 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 5c1787472a..22b8929c0f 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -70,9 +70,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_DB, "drop-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_USE_DB, "use-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_DB, "alter-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SYNC_DB, "sync-db" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_TOPIC, "create-topic" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_TOPIC, "drop-topic" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_TOPIC, "alter-topic" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_DB, "compact-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_FUNCTION, "create-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RETRIEVE_FUNCTION, "retrieve-function" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_FUNCTION, "drop-function" ) @@ -86,7 +84,6 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_HEARTBEAT, "heartbeat" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW, "show" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW_RETRIEVE, "retrieve" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_VNODE, "compact-vnode" ) // message from client to qnode // message from client to dnode TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" ) diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 8aadb97837..a36c56ea3d 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -65,9 +65,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_USE_DB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_DB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SYNC_DB] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_TOPIC] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_DROP_TOPIC] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_ALTER_TOPIC] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_DB] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_CREATE_FUNCTION] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_RETRIEVE_FUNCTION] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_DROP_FUNCTION] = dndProcessMnodeWriteMsg; @@ -81,7 +79,6 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SHOW_RETRIEVE] = dndProcessMnodeReadMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_COMPACT_VNODE] = dndProcessMnodeWriteMsg; // message from client to dnode pMgmt->msgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dndProcessDnodeReq; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 8840df848f..24ba2b254f 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -212,24 +212,15 @@ typedef struct { int8_t update; int8_t cacheLastRow; int8_t dbType; - int16_t partitions; } SDbCfg; typedef struct SDbObj { - char name[TSDB_FULL_DB_NAME_LEN]; - char acct[TSDB_USER_LEN]; - int64_t createdTime; - int64_t updateTime; - SDbCfg cfg; - int64_t uid; - int8_t status; - int32_t numOfVgroups; - int32_t numOfTables; - int32_t numOfSuperTables; - int32_t vgListSize; - int32_t vgListIndex; - SVgObj **vgList; - SAcctObj *pAcct; + char name[TSDB_FULL_DB_NAME_LEN]; + char acct[TSDB_USER_LEN]; + int64_t createdTime; + int64_t updateTime; + int64_t uid; + SDbCfg cfg; } SDbObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f01a43dfef..9b48d121e1 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,16 +15,145 @@ #define _DEFAULT_SOURCE #include "mndDb.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndTrans.h" -static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg); +#define TSDB_DB_VER 1 + +static SSdbRaw *mndDbActionEncode(SDbObj *pDb); +static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); +static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); +static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb); +static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb); +static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg); +static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg); +static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); int32_t mndInitDb(SMnode *pMnode) { - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_USE_DB, mnodeProcessUseMsg); + SSdbTable table = {.sdbType = SDB_USER, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndDbActionEncode, + .decodeFp = (SdbDecodeFp)mndDbActionDecode, + .insertFp = (SdbInsertFp)mndDbActionInsert, + .updateFp = (SdbUpdateFp)mndDbActionUpdate, + .deleteFp = (SdbDeleteFp)mndDbActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_DB, mndProcessCreateDbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_DB, mndProcessAlterDbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_DB, mndProcessDropDbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_USE_DB, mndProcessUseDbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SYNC_DB, mndProcessSyncDbMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_COMPACT_DB, mndProcessCompactDbMsg); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DB, mndGetDbMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb); + return 0; } void mndCleanupDb(SMnode *pMnode) {} +static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER, sizeof(SDbObj)); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pDb->name, TSDB_FULL_DB_NAME_LEN) + SDB_SET_BINARY(pRaw, dataPos, pDb->acct, TSDB_USER_LEN) + SDB_SET_INT64(pRaw, dataPos, pDb->createdTime) + SDB_SET_INT64(pRaw, dataPos, pDb->updateTime) + SDB_SET_INT64(pRaw, dataPos, pDb->uid) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxTables) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep2) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRowsPerFileBlock) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRowsPerFileBlock) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.commitTime) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.precision) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.replications) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.quorum) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.update) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.dbType) + SDB_SET_DATALEN(pRaw, dataPos); + + return pRaw; +} + +static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_DB_VER) { + mError("failed to decode db since %s", terrstr()); + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SDbObj)); + SDbObj *pDb = sdbGetRowObj(pRow); + if (pDb == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_BINARY(pRaw, pRow, dataPos, pDb->name, TSDB_FULL_DB_NAME_LEN) + SDB_GET_BINARY(pRaw, pRow, dataPos, pDb->acct, TSDB_USER_LEN) + SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->updateTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.maxTables) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep0) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep1) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep2) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.minRowsPerFileBlock) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.maxRowsPerFileBlock) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.commitTime) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.fsyncPeriod) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.precision) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.compression) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.walLevel) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.replications) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.quorum) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.update) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.cacheLastRow) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.dbType) + + return pRow; +} + +static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb) { + mTrace("db:%s, perform insert action", pDb->name); + return 0; +} + +static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { + mTrace("db:%s, perform delete action", pDb->name); + return 0; +} + +static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { + mTrace("db:%s, perform update action", pOldDb->name); + memcpy(pOldDb, pNewDb, sizeof(SDbObj)); + return 0; +} + SDbObj *mndAcquireDb(SMnode *pMnode, char *db) { SSdb *pSdb = pMnode->pSdb; return sdbAcquire(pSdb, SDB_DB, db); @@ -35,7 +164,13 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) { +static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SUseDbMsg *pUse = pMsg->rpcMsg.pCont; @@ -50,3 +185,251 @@ static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) { return -1; } } + +static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "replica"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "quorum"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "days"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "keep0,keep1,keep2"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "cache(MB)"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "blocks"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "minrows"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "maxrows"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "wallevel"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "fsync"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "comp"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "cachelast"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 3 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "precision"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "update"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = sdbGetSize(pSdb, SDB_DB); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +char *mnodeGetDbStr(char *src) { + char *pos = strstr(src, TS_PATH_DELIMITER); + if (pos != NULL) ++pos; + + return pos; +} + +static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SDbObj *pDb = NULL; + char *pWrite; + int32_t cols = 0; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_DB, pShow->pIter, (void **)&pDb); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + char *name = mnodeGetDbStr(pDb->name); + if (name != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]); + } else { + STR_TO_VARSTR(pWrite, "NULL"); + } + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pDb->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDb->cfg.replications; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDb->cfg.quorum; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDb->cfg.daysPerFile; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + char tmp[128] = {0}; + if (pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep1 || pDb->cfg.daysToKeep0 > pDb->cfg.daysToKeep2) { + sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2, pDb->cfg.daysToKeep0); + } else { + sprintf(tmp, "%d,%d,%d", pDb->cfg.daysToKeep0, pDb->cfg.daysToKeep1, pDb->cfg.daysToKeep2); + } + STR_WITH_SIZE_TO_VARSTR(pWrite, tmp, strlen(tmp)); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.cacheBlockSize; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.totalBlocks; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.maxRowsPerFileBlock; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.walLevel; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDb->cfg.fsyncPeriod; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.compression; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.cacheLastRow; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + char *prec = NULL; + switch (pDb->cfg.precision) { + case TSDB_TIME_PRECISION_MILLI: + prec = TSDB_TIME_PRECISION_MILLI_STR; + break; + case TSDB_TIME_PRECISION_MICRO: + prec = TSDB_TIME_PRECISION_MICRO_STR; + break; + case TSDB_TIME_PRECISION_NANO: + prec = TSDB_TIME_PRECISION_NANO_STR; + break; + default: + assert(false); + break; + } + STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.update; + cols++; + + numOfRows++; + sdbRelease(pSdb, pDb); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file From ec389d9bea801b886b25db7073dba73d365a124e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 9 Dec 2021 13:07:00 +0800 Subject: [PATCH 10/18] TD-10431 db manage --- include/common/taosmsg.h | 51 ++- include/dnode/mnode/sdb/sdb.h | 20 + include/util/taoserror.h | 38 +- include/util/tdef.h | 49 ++- source/common/src/tglobal.c | 51 --- source/dnode/mnode/impl/inc/mndDef.h | 4 +- source/dnode/mnode/impl/inc/mndDnode.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 496 ++++++++++++++++++++++++- source/dnode/mnode/impl/src/mndDnode.c | 2 +- source/dnode/vnode/impl/CMakeLists.txt | 6 +- source/libs/parser/src/astGenerator.c | 7 - source/util/src/terror.c | 32 +- 12 files changed, 610 insertions(+), 147 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 22b8929c0f..2b2383d0cd 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -567,7 +567,6 @@ typedef struct { char db[TSDB_FULL_DB_NAME_LEN]; int32_t cacheBlockSize; // MB int32_t totalBlocks; - int32_t maxTables; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; @@ -576,18 +575,51 @@ typedef struct { int32_t maxRowsPerFileBlock; int32_t commitTime; int32_t fsyncPeriod; + int8_t walLevel; int8_t precision; // time resolution int8_t compression; - int8_t walLevel; int8_t replications; int8_t quorum; - int8_t ignoreExist; int8_t update; int8_t cacheLastRow; - int8_t dbType; - int16_t partitions; - int8_t reserve[5]; -} SCreateDbMsg, SAlterDbMsg; + int8_t ignoreExist; + int32_t reserve[8]; +} SCreateDbMsg; + +typedef struct { + char db[TSDB_FULL_DB_NAME_LEN]; + int32_t totalBlocks; + int32_t daysToKeep0; + int32_t daysToKeep1; + int32_t daysToKeep2; + int32_t fsyncPeriod; + int8_t walLevel; + int8_t quorum; + int8_t cacheLastRow; + int32_t reserve[8]; +} SAlterDbMsg; + +typedef struct { + char db[TSDB_TABLE_FNAME_LEN]; + int8_t ignoreNotExists; + int32_t reserve[8]; +} SDropDbMsg; + +typedef struct { + char db[TSDB_TABLE_FNAME_LEN]; + int8_t ignoreNotExists; + int32_t reserve[8]; +} SUseDbMsg; + +typedef struct { + char db[TSDB_TABLE_FNAME_LEN]; + int32_t reserve[8]; +} SSyncDbMsg; + +typedef struct { + char db[TSDB_TABLE_FNAME_LEN]; + int32_t reserve[8]; +} SCompactDbMsg; typedef struct { char name[TSDB_FUNC_NAME_LEN]; @@ -631,11 +663,6 @@ typedef struct { char pFuncInfos[]; } SRetrieveFuncRsp; -typedef struct { - char db[TSDB_TABLE_FNAME_LEN]; - int8_t ignoreNotExists; -} SDropDbMsg, SUseDbMsg, SSyncDbMsg; - typedef struct { int32_t statusInterval; int32_t mnodeEqualVnodeNum; diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index 17ce5c952b..d1f7d9a958 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -65,6 +65,16 @@ extern "C" { dataPos += valLen; \ } +#define SDB_GET_RESERVE(pRaw, pRow, dataPos, valLen) \ + { \ + char val[valLen] = {0}; \ + if (sdbGetRawBinary(pRaw, dataPos, val, valLen) != 0) { \ + sdbFreeRow(pRow); \ + return NULL; \ + } \ + dataPos += valLen; \ + } + #define SDB_SET_INT64(pRaw, dataPos, val) \ { \ if (sdbSetRawInt64(pRaw, dataPos, val) != 0) { \ @@ -110,6 +120,16 @@ extern "C" { dataPos += valLen; \ } +#define SDB_SET_RESERVE(pRaw, dataPos, valLen) \ + { \ + char val[valLen] = {0}; \ + if (sdbSetRawBinary(pRaw, dataPos, val, valLen) != 0) { \ + sdbFreeRaw(pRaw); \ + return NULL; \ + } \ + dataPos += valLen; \ + } + #define SDB_SET_DATALEN(pRaw, dataLen) \ { \ if (sdbSetRawDataLen(pRaw, dataLen) != 0) { \ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c0f7d8d328..39efdb3d04 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -211,22 +211,30 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length") #define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length") -#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) //"Database not specified or available") -#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) //"Database already exists") -#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) //"Invalid database options") -#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0383) //"Invalid database name") -#define TSDB_CODE_MND_MONITOR_DB_FORBIDDEN TAOS_DEF_ERROR_CODE(0, 0x0384) //"Cannot delete monitor database") +#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) +#define TSDB_CODE_MND_DB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) +#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) +#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) +#define TSDB_CODE_MND_INVALID_DB_CACHE_SIZE TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_DAYS TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_KEEP0 TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_KEEP1 TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_KEEP2 TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_MIN_ROWS TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_MAX_ROWS TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_COMMIT_TIME TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_WAL_LEVEL TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_PRECISION TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_COMP TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_REPLICA TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_QUORUM TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_UPDATE TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_INVALID_DB_CACHE_LAST TAOS_DEF_ERROR_CODE(0, 0x0383) +#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x0383) #define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0385) //"Too many databases for account") -#define TSDB_CODE_MND_DB_IN_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0386) //"Database not available") -#define TSDB_CODE_MND_VGROUP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0387) //"Database unsynced") - -#define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) //"Invalid database option: days out of range") -#define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) //"Invalid database option: keep >= keep1 >= keep0 >= days") - -#define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x0392) //"Invalid topic name) -#define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x0393) //"Invalid topic option) -#define TSDB_CODE_MND_INVALID_TOPIC_PARTITONS TAOS_DEF_ERROR_CODE(0, 0x0394) //"Invalid topic partitons num, valid range: [1, 1000]) -#define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) //"Topic already exists) // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) diff --git a/include/util/tdef.h b/include/util/tdef.h index 165c27067c..3ab515671a 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -250,12 +250,6 @@ do { \ #define TSDB_MAX_TOTAL_BLOCKS 10000 #define TSDB_DEFAULT_TOTAL_BLOCKS 6 -#define TSDB_MIN_TABLES 4 -#define TSDB_MAX_TABLES 10000000 -#define TSDB_DEFAULT_TABLES 1000000 -#define TSDB_TABLES_STEP 1000 -#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default - #define TSDB_MIN_DAYS_PER_FILE 1 #define TSDB_MAX_DAYS_PER_FILE 3650 #define TSDB_DEFAULT_DAYS_PER_FILE 10 @@ -264,18 +258,26 @@ do { \ #define TSDB_MAX_KEEP 365000 // data in db to be reserved. #define TSDB_DEFAULT_KEEP 3650 // ten years -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 #define TSDB_MIN_MIN_ROW_FBLOCK 10 #define TSDB_MAX_MIN_ROW_FBLOCK 1000 +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 #define TSDB_MIN_MAX_ROW_FBLOCK 200 #define TSDB_MAX_MAX_ROW_FBLOCK 10000 +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 #define TSDB_MIN_COMMIT_TIME 30 #define TSDB_MAX_COMMIT_TIME 40960 #define TSDB_DEFAULT_COMMIT_TIME 3600 +#define TSDB_MIN_FSYNC_PERIOD 0 +#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond +#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second + +#define TSDB_MIN_WAL_LEVEL 0 +#define TSDB_MAX_WAL_LEVEL 2 +#define TSDB_DEFAULT_WAL_LEVEL 1 + #define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI #define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO #define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI @@ -284,9 +286,13 @@ do { \ #define TSDB_MAX_COMP_LEVEL 2 #define TSDB_DEFAULT_COMP_LEVEL 2 -#define TSDB_MIN_WAL_LEVEL 0 -#define TSDB_MAX_WAL_LEVEL 2 -#define TSDB_DEFAULT_WAL_LEVEL 1 +#define TSDB_MIN_DB_REPLICA_OPTION 1 +#define TSDB_MAX_DB_REPLICA_OPTION 3 +#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 + +#define TSDB_MIN_DB_QUORUM_OPTION 1 +#define TSDB_MAX_DB_QUORUM_OPTION 2 +#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 #define TSDB_MIN_DB_UPDATE 0 #define TSDB_MAX_DB_UPDATE 2 @@ -296,22 +302,6 @@ do { \ #define TSDB_MAX_DB_CACHE_LAST_ROW 3 #define TSDB_DEFAULT_CACHE_LAST_ROW 0 -#define TSDB_MIN_FSYNC_PERIOD 0 -#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond -#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second - -#define TSDB_MIN_DB_REPLICA_OPTION 1 -#define TSDB_MAX_DB_REPLICA_OPTION 3 -#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 - -#define TSDB_MIN_DB_PARTITON_OPTION 0 -#define TSDB_MAX_DB_PARTITON_OPTION 1000 -#define TSDB_DEFAULT_DB_PARTITON_OPTION 4 - -#define TSDB_MIN_DB_QUORUM_OPTION 1 -#define TSDB_MAX_DB_QUORUM_OPTION 2 -#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 - #define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_UNION_CLAUSE 5 @@ -325,6 +315,11 @@ do { \ #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type #define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode + +#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default + + + /* * 1. ordinary sub query for select * from super_table * 2. all sqlobj generated by createSubqueryObj with this flag diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 117f908faf..3cf5e52c44 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -142,13 +142,9 @@ int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; -int16_t tsPartitons = TSDB_DEFAULT_DB_PARTITON_OPTION; int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; int32_t tsMaxVgroupsPerDb = 0; -int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; -int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; -int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO; // tsdb config @@ -736,37 +732,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - // database configs - cfg.option = "maxTablesPerVnode"; - cfg.ptr = &tsMaxTablePerVnode; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TABLES; - cfg.maxValue = TSDB_MAX_TABLES; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "minTablesPerVnode"; - cfg.ptr = &tsMinTablePerVnode; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TABLES; - cfg.maxValue = TSDB_MAX_TABLES; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - - cfg.option = "tableIncStepPerVnode"; - cfg.ptr = &tsTableIncStepPerVnode; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_TABLES; - cfg.maxValue = TSDB_MAX_TABLES; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - cfg.option = "cache"; cfg.ptr = &tsCacheBlockSize; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -877,16 +842,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "partitions"; - cfg.ptr = &tsPartitons; - cfg.valType = TAOS_CFG_VTYPE_INT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MIN_DB_PARTITON_OPTION; - cfg.maxValue = TSDB_MAX_DB_PARTITON_OPTION; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - cfg.option = "quorum"; cfg.ptr = &tsQuorum; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1704,12 +1659,6 @@ int32_t taosCheckGlobalCfg() { } } - if (tsMaxTablePerVnode < tsMinTablePerVnode) { - uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)", - tsMaxTablePerVnode, tsMinTablePerVnode, tsMinTablePerVnode); - tsMaxTablePerVnode = tsMinTablePerVnode; - } - // todo refactor tsVersion = 0; for (int ver = 0, i = 0; i < TSDB_VERSION_LEN; ++i) { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 24ba2b254f..1003645143 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -195,7 +195,6 @@ typedef struct SUserObj { typedef struct { int32_t cacheBlockSize; int32_t totalBlocks; - int32_t maxTables; int32_t daysPerFile; int32_t daysToKeep0; int32_t daysToKeep1; @@ -204,14 +203,13 @@ typedef struct { int32_t maxRowsPerFileBlock; int32_t commitTime; int32_t fsyncPeriod; + int8_t walLevel; int8_t precision; int8_t compression; - int8_t walLevel; int8_t replications; int8_t quorum; int8_t update; int8_t cacheLastRow; - int8_t dbType; } SDbCfg; typedef struct SDbObj { diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index a25784e5bb..04ceb2820a 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -27,6 +27,7 @@ void mndCleanupDnode(SMnode *pMnode); SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId); void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode); SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); +int32_t mndGetDnodeSize(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9b48d121e1..3e6caba732 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,11 +15,14 @@ #define _DEFAULT_SOURCE #include "mndDb.h" +#include "mndDnode.h" #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" +#include "mndUser.h" -#define TSDB_DB_VER 1 +#define TSDB_DB_VER_NUM 1 +#define TSDB_DB_RESERVE_SIZE 64 static SSdbRaw *mndDbActionEncode(SDbObj *pDb); static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); @@ -62,7 +65,7 @@ int32_t mndInitDb(SMnode *pMnode) { void mndCleanupDb(SMnode *pMnode) {} static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER, sizeof(SDbObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj)); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -73,7 +76,6 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT64(pRaw, dataPos, pDb->uid) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxTables) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1) @@ -82,14 +84,14 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRowsPerFileBlock) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.commitTime) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod) + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.precision) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.compression) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.replications) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.quorum) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.update) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow) - SDB_SET_INT8(pRaw, dataPos, pDb->cfg.dbType) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_DB_RESERVE_SIZE) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -99,7 +101,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_DB_VER) { + if (sver != TSDB_DB_VER_NUM) { mError("failed to decode db since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -117,7 +119,6 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks) - SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.maxTables) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep0) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep1) @@ -126,14 +127,14 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.maxRowsPerFileBlock) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.commitTime) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.fsyncPeriod) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.walLevel) SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.precision) SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.compression) - SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.walLevel) SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.replications) SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.quorum) SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.update) SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.cacheLastRow) - SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.dbType) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_DB_RESERVE_SIZE) return pRow; } @@ -164,20 +165,459 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { return 0; } +static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { + if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { + terrno = TSDB_CODE_MND_INVALID_DB_CACHE_SIZE; + return -1; + } -static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { return 0; } + if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) { + terrno = TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS; + return -1; + } -static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { return 0; } + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { + terrno = TSDB_CODE_MND_INVALID_DB_DAYS; + return -1; + } + + if (pCfg->daysToKeep0 < pCfg->daysPerFile) { + terrno = TSDB_CODE_MND_INVALID_DB_KEEP0; + return -1; + } + + if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP || pCfg->daysToKeep0 > pCfg->daysToKeep1) { + terrno = TSDB_CODE_MND_INVALID_DB_KEEP0; + return -1; + } + + if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) { + terrno = TSDB_CODE_MND_INVALID_DB_KEEP1; + return -1; + } + + if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) { + terrno = TSDB_CODE_MND_INVALID_DB_KEEP1; + return -1; + } + + if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { + terrno = TSDB_CODE_MND_INVALID_DB_MIN_ROWS; + return -1; + } + + if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { + terrno = TSDB_CODE_MND_INVALID_DB_MAX_ROWS; + return -1; + } + + if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { + terrno = TSDB_CODE_MND_INVALID_DB_MIN_ROWS; + return -1; + } + + if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) { + terrno = TSDB_CODE_MND_INVALID_DB_COMMIT_TIME; + return -1; + } + + if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) { + terrno = TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD; + return -1; + } + + if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) { + terrno = TSDB_CODE_MND_INVALID_DB_WAL_LEVEL; + return -1; + } + + if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) { + terrno = TSDB_CODE_MND_INVALID_DB_PRECISION; + return -1; + } + + if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) { + terrno = TSDB_CODE_MND_INVALID_DB_COMP; + return -1; + } + + if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) { + terrno = TSDB_CODE_MND_INVALID_DB_REPLICA; + return -1; + } + + if (pCfg->replications > mndGetDnodeSize(pMnode)) { + terrno = TSDB_CODE_MND_INVALID_DB_REPLICA; + return -1; + } + + if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) { + terrno = TSDB_CODE_MND_INVALID_DB_QUORUM; + return -1; + } + + if (pCfg->quorum > pCfg->replications) { + terrno = TSDB_CODE_MND_INVALID_DB_QUORUM; + return -1; + } + + if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { + terrno = TSDB_CODE_MND_INVALID_DB_UPDATE; + return -1; + } + + if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) { + terrno = TSDB_CODE_MND_INVALID_DB_CACHE_LAST; + return -1; + } + + return TSDB_CODE_SUCCESS; +} + +static void mndSetDefaultDbCfg(SDbCfg *pCfg) { + if (pCfg->cacheBlockSize < 0) pCfg->cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; + if (pCfg->totalBlocks < 0) pCfg->totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; + if (pCfg->daysPerFile < 0) pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; + if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP; + if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = TSDB_DEFAULT_KEEP; + if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = TSDB_DEFAULT_KEEP; + if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; + if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; + if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME; + if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; + if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL; + if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION; + if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL; + if (pCfg->replications < 0) pCfg->replications = TSDB_DEFAULT_DB_REPLICA_OPTION; + if (pCfg->quorum < 0) pCfg->quorum = TSDB_DEFAULT_DB_QUORUM_OPTION; + if (pCfg->update < 0) pCfg->update = TSDB_DEFAULT_DB_UPDATE_OPTION; + if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; +} + +static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, char *acct) { + SDbObj dbObj = {0}; + tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); + tstrncpy(dbObj.acct, acct, TSDB_USER_LEN); + dbObj.createdTime = taosGetTimestampMs(); + dbObj.updateTime = dbObj.createdTime; + dbObj.uid = 1234; + dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize, + .totalBlocks = pCreate->totalBlocks, + .daysPerFile = pCreate->daysPerFile, + .daysToKeep0 = pCreate->daysToKeep0, + .daysToKeep1 = pCreate->daysToKeep1, + .daysToKeep2 = pCreate->daysToKeep2, + .minRowsPerFileBlock = pCreate->minRowsPerFileBlock, + .maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock, + .fsyncPeriod = pCreate->fsyncPeriod, + .commitTime = pCreate->commitTime, + .precision = pCreate->precision, + .compression = pCreate->compression, + .walLevel = pCreate->walLevel, + .replications = pCreate->replications, + .quorum = pCreate->quorum, + .update = pCreate->update, + .cacheLastRow = pCreate->cacheLastRow}; + + mndSetDefaultDbCfg(&dbObj.cfg); + + if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) { + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); + + SSdbRaw *pRedoRaw = mndDbActionEncode(&dbObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); + + SSdbRaw *pUndoRaw = mndDbActionEncode(&dbObj); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + + SSdbRaw *pCommitRaw = mndDbActionEncode(&dbObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCreateDbMsg *pCreate = pMsg->rpcMsg.pCont; + + pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); + pCreate->totalBlocks = htonl(pCreate->totalBlocks); + pCreate->daysPerFile = htonl(pCreate->daysPerFile); + pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0); + pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); + pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); + pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); + pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); + pCreate->commitTime = htonl(pCreate->commitTime); + pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); + + mDebug("db:%s, start to create", pCreate->db); + + SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db); + if (pDb != NULL) { + sdbRelease(pMnode->pSdb, pDb); + if (pCreate->ignoreExist) { + mDebug("db:%s, already exist, ignore exist is set", pCreate->db); + return 0; + } else { + terrno = TSDB_CODE_MND_DB_ALREADY_EXIST; + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + } + + SUserObj *pOperUser = mndAcquireUser(pMnode, pMsg->user); + if (pOperUser == NULL) { + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + + int32_t code = mndCreateDb(pMnode, pMsg, pCreate, pOperUser->acct); + mndReleaseUser(pMnode, pOperUser); + + if (code != 0) { + terrno = code; + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mnodeSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { + bool changed = false; + + if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { + pDb->cfg.totalBlocks = pAlter->totalBlocks; + changed = true; + } + + if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { + pDb->cfg.daysToKeep0 = pAlter->daysToKeep0; + changed = true; + } + + if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { + pDb->cfg.daysToKeep1 = pAlter->daysToKeep1; + changed = true; + } + + if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { + pDb->cfg.daysToKeep2 = pAlter->daysToKeep2; + changed = true; + } + + if (pAlter->fsyncPeriod >= 0 && pAlter->fsyncPeriod != pDb->cfg.fsyncPeriod) { + pDb->cfg.fsyncPeriod = pAlter->fsyncPeriod; + changed = true; + } + + if (pAlter->walLevel >= 0 && pAlter->walLevel != pDb->cfg.walLevel) { + pDb->cfg.walLevel = pAlter->walLevel; + changed = true; + } + + if (pAlter->quorum >= 0 && pAlter->quorum != pDb->cfg.quorum) { + pDb->cfg.quorum = pAlter->quorum; + changed = true; + } + + if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) { + pDb->cfg.cacheLastRow = pAlter->cacheLastRow; + changed = true; + } + + if (!changed) { + terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; + return -1; + } + + return 0; +} + +static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbObj *pNewDb) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("db:%s, failed to update since %s", pOldDb->name, terrstr()); + return terrno; + } + + mDebug("trans:%d, used to update db:%s", pTrans->id, pOldDb->name); + + SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont; + pAlter->totalBlocks = htonl(pAlter->totalBlocks); + pAlter->daysToKeep0 = htonl(pAlter->daysToKeep0); + pAlter->daysToKeep1 = htonl(pAlter->daysToKeep1); + pAlter->daysToKeep2 = htonl(pAlter->daysToKeep2); + pAlter->fsyncPeriod = htonl(pAlter->fsyncPeriod); + + mDebug("db:%s, start to alter", pAlter->db); + + SDbObj *pDb = mndAcquireDb(pMnode, pAlter->db); + if (pDb == NULL) { + mError("db:%s, failed to alter since %s", pAlter->db, terrstr()); + return TSDB_CODE_MND_DB_NOT_EXIST; + } + + SDbObj dbObj = {0}; + memcpy(&dbObj, pDb, sizeof(SDbObj)); + + int32_t code = mnodeSetDbCfgFromAlterDbMsg(&dbObj, pAlter); + if (code != 0) { + mndReleaseDb(pMnode, pDb); + mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); + return code; + } + + code = mndUpdateDb(pMnode, pMsg, pDb, &dbObj); + mndReleaseDb(pMnode, pDb); + + if (code != 0) { + mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); + return code; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} + +static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); + if (pTrans == NULL) { + mError("db:%s, failed to drop since %s", pDb->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); + + SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); + + SSdbRaw *pUndoRaw = mndDbActionEncode(pDb); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { + mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); + + SSdbRaw *pCommitRaw = mndDbActionEncode(pDb); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); + + if (mndTransPrepare(pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SDropDbMsg *pDrop = pMsg->rpcMsg.pCont; + + mDebug("db:%s, start to drop", pDrop->db); + + SDbObj *pDb = mndAcquireDb(pMnode, pDrop->db); + if (pDb == NULL) { + if (pDrop->ignoreNotExists) { + mDebug("db:%s, not exist, ignore not exist is set", pDrop->db); + return TSDB_CODE_SUCCESS; + } else { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); + return -1; + } + } + + int32_t code = mndDropDb(pMnode, pMsg, pDb); + mndReleaseDb(pMnode, pDb); + + if (code != 0) { + terrno = code; + mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); + return code; + } + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; +} static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SUseDbMsg *pUse = pMsg->rpcMsg.pCont; - strncpy(pMsg->db, pUse->db, TSDB_FULL_DB_NAME_LEN); - SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); if (pDb != NULL) { + strncpy(pMsg->db, pUse->db, TSDB_FULL_DB_NAME_LEN); mndReleaseDb(pMnode, pDb); return 0; } else { @@ -186,9 +626,33 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { } } -static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SSyncDbMsg *pSync = pMsg->rpcMsg.pCont; -static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) { return 0; } + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + if (pDb == NULL) { + mError("db:%s, failed to process sync db msg since %s", pMsg->db, terrstr()); + return -1; + } else { + mndReleaseDb(pMnode, pDb); + return 0; + } +} + +static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SCompactDbMsg *pCompact = pMsg->rpcMsg.pCont; + + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + if (pDb == NULL) { + mError("db:%s, failed to process compact db msg since %s", pMsg->db, terrstr()); + return -1; + } else { + mndReleaseDb(pMnode, pDb); + return 0; + } +} static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { SMnode *pMnode = pMsg->pMnode; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index c34d12c6e2..b614209e76 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -216,7 +216,7 @@ static SDnodeObj *mndAcquireDnodeByEp(SMnode *pMnode, char *pEpStr) { return NULL; } -static int32_t mndGetDnodeSize(SMnode *pMnode) { +int32_t mndGetDnodeSize(SMnode *pMnode) { SSdb *pSdb = pMnode->pSdb; return sdbGetSize(pSdb, SDB_DNODE); } diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/impl/CMakeLists.txt index 040b02d2b6..d6d267c4d4 100644 --- a/source/dnode/vnode/impl/CMakeLists.txt +++ b/source/dnode/vnode/impl/CMakeLists.txt @@ -18,6 +18,6 @@ target_link_libraries( ) # test -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) \ No newline at end of file +#if(${BUILD_TEST}) +# add_subdirectory(test) +#endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c index f328486556..3b7d1cbc29 100644 --- a/source/libs/parser/src/astGenerator.c +++ b/source/libs/parser/src/astGenerator.c @@ -972,13 +972,6 @@ void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) { memset(&pDBInfo->precision, 0, sizeof(SToken)); } -void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo) { - setDefaultCreateDbOption(pDBInfo); - - pDBInfo->dbType = TSDB_DB_TYPE_TOPIC; - pDBInfo->partitions = TSDB_DEFAULT_DB_PARTITON_OPTION; -} - // prefix show db.tables; void tSetDbName(SToken *pCpxName, SToken *pDb) { pCpxName->type = pDb->type; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c8904d6248..5110c8ba22 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -223,21 +223,29 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_EXIST, "Database not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, "Cannot delete monitor database") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_CACHE_SIZE, "Invalid database cache block size option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_TOTAL_BLOCKS, "Invalid database total blocks option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_DAYS, "Invalid database days option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP0, "Invalid database keep0 option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP1, "Invalid database keep1 option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_KEEP2, "Invalid database keep2 option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_MIN_ROWS, "Invalid database min rows option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_MAX_ROWS, "Invalid database max rows option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_COMMIT_TIME, "Invalid database commit time option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_FSYNC_PERIOD, "Invalid database fsync periodoptions") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_WAL_LEVEL, "Invalid database wal level option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_PRECISION, "Invalid database precisin option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_COMP, "Invalid database compression option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_REPLICA, "Invalid database replication option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_QUORUM, "Invalid database quorum option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_UPDATE, "Invalid database update option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_CACHE_LAST, "Invalid database cache last option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, "Database not available") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, "Database unsynced") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS, "Invalid database option: days out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP, "Invalid database option: keep2 >= keep1 >= keep0 >= days") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC, "Invalid topic name") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION, "Invalid topic option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_PARTITONS, "Invalid topic partitons num, valid range: [1, 1000]") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") From 8d77b0a7bb07c5430e3b67db4f6974e324def05c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 9 Dec 2021 16:47:23 +0800 Subject: [PATCH 11/18] TD-10431 vgroup manage --- source/dnode/mnode/impl/inc/mndDef.h | 20 +- source/dnode/mnode/impl/inc/mndInt.h | 1 + source/dnode/mnode/impl/inc/mndVgroup.h | 3 +- source/dnode/mnode/impl/src/mndDb.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 255 +++++++++++++++++++++++- 5 files changed, 263 insertions(+), 18 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 1003645143..125fed1b10 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -223,29 +223,23 @@ typedef struct SDbObj { typedef struct { int32_t dnodeId; - int8_t role; - SDnodeObj *pDnode; + ESyncState role; } SVnodeGid; typedef struct SVgObj { - uint32_t vgId; - int32_t numOfVnodes; + int32_t vgId; int64_t createdTime; int64_t updateTime; - int32_t lbDnodeId; - int32_t lbTime; + int32_t version; char dbName[TSDB_FULL_DB_NAME_LEN]; - int8_t inUse; - int8_t accessState; - int8_t status; - SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; - int32_t vgCfgVersion; - int8_t compact; int32_t numOfTables; + int32_t numOfTimeSeries; int64_t totalStorage; int64_t compStorage; int64_t pointsWritten; - SDbObj *pDb; + int8_t compact; + int8_t replica; + SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; } SVgObj; typedef struct SSTableObj { diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 8d20356e39..b44d6570fe 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -20,6 +20,7 @@ #include "sdb.h" #include "tcache.h" #include "tqueue.h" +#include "ttime.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 5509c81a2e..21aee0f0a4 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -24,7 +24,8 @@ extern "C" { int32_t mndInitVgroup(SMnode *pMnode); void mndCleanupVgroup(SMnode *pMnode); -int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); +SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); +void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 3e6caba732..4dbf532a49 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -40,7 +40,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); int32_t mndInitDb(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_USER, + SSdbTable table = {.sdbType = SDB_DB, .keyType = SDB_KEY_BINARY, .encodeFp = (SdbEncodeFp)mndDbActionEncode, .decodeFp = (SdbDecodeFp)mndDbActionDecode, diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 68ee373d8c..230514db86 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -15,17 +15,52 @@ #define _DEFAULT_SOURCE #include "mndVgroup.h" +#include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" -#include "ttime.h" +#define TSDB_VGROUP_VER_NUM 1 +#define TSDB_VGROUP_RESERVE_SIZE 64 + +static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); +static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); +static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup); + +static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg); +static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg); + +static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_VGROUP, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, + .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, + .insertFp = (SdbInsertFp)mndVgroupActionInsert, + .updateFp = (SdbUpdateFp)mndVgroupActionDelete, + .deleteFp = (SdbDeleteFp)mndVgroupActionUpdate}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_VNODE_IN_RSP, mndProcessCreateVnodeRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_VNODE_IN_RSP, mndProcessAlterVnodeRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_VNODE_IN_RSP, mndProcessDropVnodeRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SYNC_VNODE_IN_RSP, mndProcessSyncVnodeRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_COMPACT_VNODE_IN_RSP, mndProcessCompactVnodeRsp); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndGetVgroupMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndRetrieveVgroups); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VGROUP, mndCancelGetNextVgroup); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndGetVnodeMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndRetrieveVnodes); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndCancelGetNextVnode); @@ -35,7 +70,221 @@ int32_t mndInitVgroup(SMnode *pMnode) { void mndCleanupVgroup(SMnode *pMnode) {} -int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { +static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SDbObj)); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId) + SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime) + SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime) + SDB_SET_INT32(pRaw, dataPos, pVgroup->version) + SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN) + SDB_SET_INT8(pRaw, dataPos, pVgroup->replica) + for (int8_t i = 0; i < pVgroup->replica; ++i) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId) + SDB_SET_INT8(pRaw, dataPos, pVgid->role) + } + SDB_SET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE) + SDB_SET_DATALEN(pRaw, dataPos); + + return pRaw; +} + +static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_VGROUP_VER_NUM) { + mError("failed to decode vgroup since %s", terrstr()); + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + return NULL; + } + + SSdbRow *pRow = sdbAllocRow(sizeof(SDbObj)); + SVgObj *pVgroup = sdbGetRowObj(pRow); + if (pVgroup == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->vgId) + SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->updateTime) + SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->version) + SDB_GET_BINARY(pRaw, pRow, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN) + SDB_GET_INT8(pRaw, pRow, dataPos, &pVgroup->replica) + for (int8_t i = 0; i < pVgroup->replica; ++i) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SDB_GET_INT32(pRaw, pRow, dataPos, &pVgid->dnodeId) + SDB_GET_INT8(pRaw, pRow, dataPos, (int8_t *)&pVgid->role) + } + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_VGROUP_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup) { + mTrace("vgId:%d, perform insert action", pVgroup->vgId); + return 0; +} + +static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup) { + mTrace("vgId:%d, perform delete action", pVgroup->vgId); + return 0; +} + +static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) { + mTrace("vgId:%d, perform update action", pOldVgroup->vgId); + pOldVgroup->vgId = pNewVgroup->vgId; + pOldVgroup->createdTime = pNewVgroup->createdTime; + pOldVgroup->updateTime = pNewVgroup->updateTime; + pOldVgroup->version = pNewVgroup->version; + memcpy(pOldVgroup->dbName, pNewVgroup->dbName, TSDB_FULL_DB_NAME_LEN); + pOldVgroup->replica = pNewVgroup->replica; + memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); + return 0; +} + +SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_VGROUP, &vgId); +} + +void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pVgroup); +} + +static void mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { + SSdb *pSdb = pMnode->pSdb; + int8_t replica = 1; + int32_t numOfVgroups = 0; + + void *pIter = NULL; + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + if (strcmp(pVgroup->dbName, dbName) == 0) { + replica = MAX(replica, pVgroup->replica); + numOfVgroups++; + } + + sdbRelease(pSdb, pVgroup); + } + + *pReplica = replica; + *pNumOfVgroups = numOfVgroups; +} + +static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + mndGetVgroupMaxReplica(pMnode, pShow->db, &pShow->replica, &pShow->numOfRows); + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "vgId"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "tables"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + for (int32_t i = 0; i < pShow->replica; ++i) { + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + } + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SVgObj *pVgroup = NULL; + int32_t cols = 0; + char *pWrite; + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); + if (pShow->pIter == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pVgroup->vgId; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pVgroup->numOfTables; + cols++; + + for (int32_t i = 0; i < pShow->replica; ++i) { + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *) pWrite = pVgroup->vnodeGid[i].dnodeId; + cols++; + + const char *role = mndGetRoleStr(pVgroup->vnodeGid[i].role); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); + cols++; + } + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pVgroup->compact; + cols++; + + sdbRelease(pSdb, pVgroup); + numOfRows++; + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} + +static int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { if (dnodeId == 0) { return 0; } @@ -96,7 +345,7 @@ static int32_t mndRetrieveVnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); if (pShow->pIter == NULL) break; - for (int32_t i = 0; i < pVgroup->numOfVnodes && numOfRows < rows; ++i) { + for (int32_t i = 0; i < pVgroup->replica && numOfRows < rows; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; if (pVgid->dnodeId != dnodeId) continue; From c0692a3e9fb3721848448d9d2dcf80cc0a7ffff2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 9 Dec 2021 18:53:09 +0800 Subject: [PATCH 12/18] TD-10431 table manage --- include/common/taosmsg.h | 6 +- source/dnode/mnode/impl/inc/mndDef.h | 20 +- source/dnode/mnode/impl/src/mndStable.c | 308 +++++++++++++++++++++++- source/dnode/mnode/impl/src/mndVgroup.c | 16 +- 4 files changed, 331 insertions(+), 19 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 2b2383d0cd..94924e6c04 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -272,14 +272,12 @@ typedef struct { SShellSubmitRspBlock failedBlocks[]; } SShellSubmitRspMsg; -//#if 0 typedef struct SSchema { - uint8_t type; - char name[TSDB_COL_NAME_LEN]; + int8_t type; int16_t colId; int32_t bytes; + char name[TSDB_COL_NAME_LEN]; } SSchema; -//#endif typedef struct { int32_t contLen; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 125fed1b10..452385a8bf 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -47,7 +47,7 @@ typedef struct SAcctObj SAcctObj; typedef struct SUserObj SUserObj; typedef struct SDbObj SDbObj; typedef struct SVgObj SVgObj; -typedef struct SSTableObj SSTableObj; +typedef struct STableObj STableObj; typedef struct SFuncObj SFuncObj; typedef struct SOperObj SOperObj; @@ -242,15 +242,19 @@ typedef struct SVgObj { SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; } SVgObj; -typedef struct SSTableObj { - char tableId[TSDB_TABLE_NAME_LEN]; - uint64_t uid; +typedef struct SStableObj { + char name[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_FULL_DB_NAME_LEN]; int64_t createdTime; int64_t updateTime; - int32_t numOfColumns; // used by normal table - int32_t numOfTags; - SSchema *schema; -} SSTableObj; + uint64_t uid; + int32_t version; + int16_t numOfFields; + int16_t numOfTags; + SSchema *fieldSchema; + SSchema *tagSchema; + char pCont[]; +} SStableObj; typedef struct SFuncObj { char name[TSDB_FUNC_NAME_LEN]; diff --git a/source/dnode/mnode/impl/src/mndStable.c b/source/dnode/mnode/impl/src/mndStable.c index e54bb17451..aa280d8498 100644 --- a/source/dnode/mnode/impl/src/mndStable.c +++ b/source/dnode/mnode/impl/src/mndStable.c @@ -14,8 +14,308 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndStable.h" +#include "mndDnode.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndDb.h" -int32_t mndInitStable(SMnode *pMnode) { return 0; } -void mndCleanupStable(SMnode *pMnode) {} \ No newline at end of file +#define TSDB_STABLE_VER_NUM 1 +#define TSDB_STABLE_RESERVE_SIZE 64 + +static SSdbRaw *mndStableActionEncode(SStableObj *pStable); +static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw); +static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStable); +static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStable); +static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStable, SStableObj *pNewStable); +static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg); +static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg); +static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextStable(SMnode *pMnode, void *pIter); + +int32_t mndInitStable(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_STABLE, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndStableActionEncode, + .decodeFp = (SdbDecodeFp)mndStableActionDecode, + .insertFp = (SdbInsertFp)mndStableActionInsert, + .updateFp = (SdbUpdateFp)mndStableActionUpdate, + .deleteFp = (SdbDeleteFp)mndStableActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_DB, mndProcessCreateStableMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_DB, mndProcessAlterStableMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_DB, mndProcessDropStableMsg); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DB, mndGetStableMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveStables); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextStable); +} + +void mndCleanupStable(SMnode *pMnode) {} + +static SSdbRaw *mndStableActionEncode(SStableObj *pStable) { + int32_t size = sizeof(SStableObj) + (pStable->numOfFields + pStable->numOfTags) * sizeof(SSchema); + SSdbRaw *pRaw = sdbAllocRaw(SDB_STABLE, TSDB_STABLE_VER_NUM, size); + if (pRaw == NULL) return NULL; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pStable->name, TSDB_TABLE_NAME_LEN) + SDB_SET_INT64(pRaw, dataPos, pStable->createdTime) + SDB_SET_INT64(pRaw, dataPos, pStable->updateTime) + SDB_SET_INT64(pRaw, dataPos, pStable->uid) + SDB_SET_INT64(pRaw, dataPos, pStable->version) + SDB_SET_INT16(pRaw, dataPos, pStable->numOfFields) + SDB_SET_INT16(pRaw, dataPos, pStable->numOfTags) + + for (int32_t i = 0; i < pStable->numOfFields; ++i) { + SSchema *pSchema = &pStable->fieldSchema[i]; + SDB_SET_INT8(pRaw, dataPos, pSchema->type); + SDB_SET_INT16(pRaw, dataPos, pSchema->colId); + SDB_SET_INT16(pRaw, dataPos, pSchema->bytes); + SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN); + } + + for (int32_t i = 0; i < pStable->numOfTags; ++i) { + SSchema *pSchema = &pStable->tagSchema[i]; + SDB_SET_INT8(pRaw, dataPos, pSchema->type); + SDB_SET_INT32(pRaw, dataPos, pSchema->colId); + SDB_SET_INT32(pRaw, dataPos, pSchema->bytes); + SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN); + } + + SDB_SET_RESERVE(pRaw, dataPos, TSDB_STABLE_RESERVE_SIZE) + SDB_SET_DATALEN(pRaw, dataPos); + + return pRaw; +} + +static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw) { + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; + + if (sver != TSDB_STABLE_VER_NUM) { + mError("failed to decode stable since %s", terrstr()); + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + return NULL; + } + + int32_t size = sizeof(SStableObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); + SSdbRow *pRow = sdbAllocRow(size); + SStableObj *pStable = sdbGetRowObj(pRow); + if (pStable == NULL) return NULL; + + int32_t dataPos = 0; + SDB_GET_BINARY(pRaw, pRow, dataPos, pStable->name, TSDB_TABLE_NAME_LEN) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStable->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStable->updateTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStable->uid) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStable->version) + SDB_GET_INT16(pRaw, pRow, dataPos, &pStable->numOfFields) + SDB_GET_INT16(pRaw, pRow, dataPos, &pStable->numOfTags) + + pStable->fieldSchema = (SSchema *)pStable->pCont; + pStable->tagSchema = (SSchema *)(pStable->pCont + pStable->numOfFields * sizeof(SSchema)); + + for (int32_t i = 0; i < pStable->numOfFields; ++i) { + SSchema *pSchema = &pStable->fieldSchema[i]; + SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); + SDB_GET_INT16(pRaw, pRow, dataPos, &pSchema->colId); + SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); + SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); + } + + for (int32_t i = 0; i < pStable->numOfTags; ++i) { + SSchema *pSchema = &pStable->tagSchema[i]; + SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); + SDB_GET_INT16(pRaw, pRow, dataPos, &pSchema->colId); + SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); + SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); + } + + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_STABLE_RESERVE_SIZE) + + return pRow; +} + +static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStable) { + mTrace("stable:%s, perform insert action", pStable->name); + return 0; +} + +static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStable) { + mTrace("stable:%s, perform delete action", pStable->name); + return 0; +} + +static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStable, SStableObj *pNewStable) { + mTrace("stable:%s, perform update action", pOldStable->name); + memcpy(pOldStable->name, pNewStable->name, TSDB_TABLE_NAME_LEN); + pOldStable->createdTime = pNewStable->createdTime; + pOldStable->updateTime = pNewStable->updateTime; + pOldStable->uid = pNewStable->uid; + pOldStable->version = pNewStable->version; + pOldStable->numOfFields = pNewStable->numOfFields; + pOldStable->numOfTags = pNewStable->numOfTags; + pOldStable->createdTime = pNewStable->createdTime; + + memcpy(pOldStable->pCont, pNewStable, sizeof(SDbObj)); + // pStable->fieldSchema = pStable->pCont; + // pStable->tagSchema = pStable->pCont + pStable->numOfFields * sizeof(SSchema); + + return 0; +} + +static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndGetNumOfStables(SMnode *pMnode, char *dbName, int32_t *pNumOfStables) { + SSdb *pSdb = pMnode->pSdb; + + SDbObj *pDb = mndAcquireDb(pMnode, dbName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } + + int32_t numOfStables = 0; + void *pIter = NULL; + while (1) { + SStableObj *pStable = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pStable); + if (pIter == NULL) break; + + if (strcmp(pStable->db, dbName) == 0) { + numOfStables++; + } + + sdbRelease(pSdb, pStable); + } + + *pNumOfStables = numOfStables; + return 0; +} + +static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + + if (mndGetNumOfStables(pMnode, pShow->db, &pShow->numOfRows) != 0) { + return -1; + } + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "name"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "columns"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "tags"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tableFname, mndShowStr(pShow->type)); + + return 0; +} + +static void mnodeExtractTableName(char* tableId, char* name) { + int pos = -1; + int num = 0; + for (pos = 0; tableId[pos] != 0; ++pos) { + if (tableId[pos] == '.') num++; + if (num == 2) break; + } + + if (num == 2) { + strcpy(name, tableId + pos + 1); + } +} + +static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SStableObj *pStable = NULL; + int32_t cols = 0; + char *pWrite; + char prefix[64] = {0}; + + tstrncpy(prefix, pShow->db, 64); + strcat(prefix, TS_PATH_DELIMITER); + int32_t prefixLen = (int32_t)strlen(prefix); + + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_STABLE, pShow->pIter, (void **)&pStable); + if (pShow->pIter == NULL) break; + + if (strncmp(pStable->name, prefix, prefixLen) != 0) { + sdbRelease(pSdb, pStable); + continue; + } + + cols = 0; + + char stableName[TSDB_TABLE_FNAME_LEN] = {0}; + memcpy(stableName, pStable->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, stableName); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pStable->createdTime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pStable->numOfFields; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pStable->numOfTags; + cols++; + + numOfRows++; + sdbRelease(pSdb, pStable); + } + + pShow->numOfReads += numOfRows; + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + return numOfRows; +} + +static void mndCancelGetNextStable(SMnode *pMnode, void *pIter) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 230514db86..b295fe64cf 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -161,8 +161,15 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { sdbRelease(pSdb, pVgroup); } -static void mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { - SSdb *pSdb = pMnode->pSdb; +static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { + SSdb *pSdb = pMnode->pSdb; + + SDbObj *pDb = mndAcquireDb(pMnode, dbName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } + int8_t replica = 1; int32_t numOfVgroups = 0; @@ -182,13 +189,16 @@ static void mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplic *pReplica = replica; *pNumOfVgroups = numOfVgroups; + return 0; } static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { SMnode *pMnode = pMsg->pMnode; SSdb *pSdb = pMnode->pSdb; - mndGetVgroupMaxReplica(pMnode, pShow->db, &pShow->replica, &pShow->numOfRows); + if (mndGetVgroupMaxReplica(pMnode, pShow->db, &pShow->replica, &pShow->numOfRows) != 0) { + return -1; + } int32_t cols = 0; SSchema *pSchema = pMeta->schema; From a27bf43fab2d1501bd2d8416c56f9a2dc340b593 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 9 Dec 2021 19:32:45 +0800 Subject: [PATCH 13/18] minor changes --- source/dnode/mnode/impl/src/mndDb.c | 2 +- source/dnode/mnode/impl/src/mndFunc.c | 1 + source/dnode/mnode/impl/src/mndStable.c | 2 ++ source/dnode/mnode/impl/src/mndVgroup.c | 12 ++++++------ 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 4dbf532a49..d9f8cbe4db 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -59,7 +59,7 @@ int32_t mndInitDb(SMnode *pMnode) { mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb); - return 0; + return sdbSetTable(pMnode->pSdb, table); } void mndCleanupDb(SMnode *pMnode) {} diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index ab2da8e0d5..30569b96fb 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -51,6 +51,7 @@ int32_t mndInitFunc(SMnode *pMnode) { mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndGetFuncMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndRetrieveFuncs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndCancelGetNextFunc); + return sdbSetTable(pMnode->pSdb, table); } diff --git a/source/dnode/mnode/impl/src/mndStable.c b/source/dnode/mnode/impl/src/mndStable.c index aa280d8498..3aafe86a6d 100644 --- a/source/dnode/mnode/impl/src/mndStable.c +++ b/source/dnode/mnode/impl/src/mndStable.c @@ -53,6 +53,8 @@ int32_t mndInitStable(SMnode *pMnode) { mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DB, mndGetStableMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveStables); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextStable); + + return sdbSetTable(pMnode->pSdb, table); } void mndCleanupStable(SMnode *pMnode) {} diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index b295fe64cf..805748f1df 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -65,7 +65,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndRetrieveVnodes); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VNODES, mndCancelGetNextVnode); - return 0; + return sdbSetTable(pMnode->pSdb, table); } void mndCleanupVgroup(SMnode *pMnode) {} @@ -139,7 +139,7 @@ static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) { +static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) { mTrace("vgId:%d, perform update action", pOldVgroup->vgId); pOldVgroup->vgId = pNewVgroup->vgId; pOldVgroup->createdTime = pNewVgroup->createdTime; @@ -267,7 +267,7 @@ static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, for (int32_t i = 0; i < pShow->replica; ++i) { pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *) pWrite = pVgroup->vnodeGid[i].dnodeId; + *(int16_t *)pWrite = pVgroup->vnodeGid[i].dnodeId; cols++; const char *role = mndGetRoleStr(pVgroup->vnodeGid[i].role); @@ -275,11 +275,11 @@ static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); cols++; } - + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int8_t *)pWrite = pVgroup->compact; + *(int8_t *)pWrite = pVgroup->compact; cols++; - + sdbRelease(pSdb, pVgroup); numOfRows++; } From b6ed305614209ea9d823c847ecea41a32f63c35c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 10 Dec 2021 11:42:03 +0800 Subject: [PATCH 14/18] TD-10431 update sdb for stable --- include/common/taosmsg.h | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndStable.c | 60 +++++++++++++++++-------- source/dnode/mnode/impl/src/mndVgroup.c | 4 +- source/libs/parser/inc/sql.y | 4 +- 6 files changed, 48 insertions(+), 26 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 94924e6c04..3f9dd48ef4 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -159,7 +159,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_DNODE, TSDB_MGMT_TABLE_MNODE, TSDB_MGMT_TABLE_VGROUP, - TSDB_MGMT_TABLE_METRIC, + TSDB_MGMT_TABLE_STABLE, TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QUERIES, TSDB_MGMT_TABLE_STREAMS, diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 452385a8bf..b9bafddc1c 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -251,9 +251,9 @@ typedef struct SStableObj { int32_t version; int16_t numOfFields; int16_t numOfTags; + SRWLatch lock; SSchema *fieldSchema; SSchema *tagSchema; - char pCont[]; } SStableObj; typedef struct SFuncObj { diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index ac33d89517..cfecaf2775 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -270,7 +270,7 @@ char *mndShowStr(int32_t showType) { return "show mnodes"; case TSDB_MGMT_TABLE_VGROUP: return "show vgroups"; - case TSDB_MGMT_TABLE_METRIC: + case TSDB_MGMT_TABLE_STABLE: return "show stables"; case TSDB_MGMT_TABLE_MODULE: return "show modules"; diff --git a/source/dnode/mnode/impl/src/mndStable.c b/source/dnode/mnode/impl/src/mndStable.c index 3aafe86a6d..a2a10a0752 100644 --- a/source/dnode/mnode/impl/src/mndStable.c +++ b/source/dnode/mnode/impl/src/mndStable.c @@ -33,6 +33,10 @@ static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStable, SStabl static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg); static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg); +static int32_t mndProcessCreateStableInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessAlterStableInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessDropStableInRsp(SMnodeMsg *pMsg); +static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg); static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStable(SMnode *pMnode, void *pIter); @@ -46,13 +50,17 @@ int32_t mndInitStable(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndStableActionUpdate, .deleteFp = (SdbDeleteFp)mndStableActionDelete}; - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_DB, mndProcessCreateStableMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_DB, mndProcessAlterStableMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_DB, mndProcessDropStableMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STABLE, mndProcessCreateStableMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STABLE, mndProcessAlterStableMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STABLE, mndProcessDropStableMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_STABLE_IN_RSP, mndProcessCreateStableInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_STABLE_IN_RSP, mndProcessAlterStableInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_STABLE_IN_RSP, mndProcessDropStableInRsp); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_TABLE_META, mndProcessStableMetaMsg); - mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DB, mndGetStableMeta); - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveStables); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextStable); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndGetStableMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndRetrieveStables); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STABLE, mndCancelGetNextStable); return sdbSetTable(pMnode->pSdb, table); } @@ -119,8 +127,8 @@ static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, pRow, dataPos, &pStable->numOfFields) SDB_GET_INT16(pRaw, pRow, dataPos, &pStable->numOfTags) - pStable->fieldSchema = (SSchema *)pStable->pCont; - pStable->tagSchema = (SSchema *)(pStable->pCont + pStable->numOfFields * sizeof(SSchema)); + pStable->fieldSchema = calloc(pStable->numOfFields, sizeof(SSchema)); + pStable->tagSchema = calloc(pStable->numOfTags, sizeof(SSchema)); for (int32_t i = 0; i < pStable->numOfFields; ++i) { SSchema *pSchema = &pStable->fieldSchema[i]; @@ -155,28 +163,42 @@ static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStable) { static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStable, SStableObj *pNewStable) { mTrace("stable:%s, perform update action", pOldStable->name); - memcpy(pOldStable->name, pNewStable->name, TSDB_TABLE_NAME_LEN); - pOldStable->createdTime = pNewStable->createdTime; - pOldStable->updateTime = pNewStable->updateTime; - pOldStable->uid = pNewStable->uid; - pOldStable->version = pNewStable->version; - pOldStable->numOfFields = pNewStable->numOfFields; - pOldStable->numOfTags = pNewStable->numOfTags; - pOldStable->createdTime = pNewStable->createdTime; + atomic_exchange_32(&pOldStable->updateTime, pNewStable->updateTime); + atomic_exchange_32(&pOldStable->version, pNewStable->version); - memcpy(pOldStable->pCont, pNewStable, sizeof(SDbObj)); - // pStable->fieldSchema = pStable->pCont; - // pStable->tagSchema = pStable->pCont + pStable->numOfFields * sizeof(SSchema); + taosWLockLatch(&pOldStable->lock); + int16_t numOfTags = pNewStable->numOfTags; + int32_t tagSize = numOfTags * sizeof(SSchema); + int16_t numOfFields = pNewStable->numOfFields; + int32_t fieldSize = numOfFields * sizeof(SSchema); + if (pOldStable->numOfTags < numOfTags) { + pOldStable->tagSchema = malloc(tagSize); + } + if (pOldStable->numOfFields < numOfFields) { + pOldStable->fieldSchema = malloc(fieldSize); + } + + memcpy(pOldStable->tagSchema, pNewStable->tagSchema, tagSize); + memcpy(pOldStable->fieldSchema, pNewStable->fieldSchema, fieldSize); + taosWUnLockLatch(&pOldStable->lock); return 0; } static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCreateStableInRsp(SMnodeMsg *pMsg) { return 0; } + static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessAlterStableInRsp(SMnodeMsg *pMsg) { return 0; } + static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessDropStableInRsp(SMnodeMsg *pMsg) { return 0; } + +static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg) { return 0; } + static int32_t mndGetNumOfStables(SMnode *pMnode, char *dbName, int32_t *pNumOfStables) { SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 805748f1df..7bc4ce0816 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -277,9 +277,9 @@ static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, } pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int8_t *)pWrite = pVgroup->compact; + *(int8_t *)pWrite = pVgroup->compact; cols++; - + sdbRelease(pSdb, pVgroup); numOfRows++; } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 04fb3beb54..ca281c9f1c 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -112,13 +112,13 @@ cmd ::= SHOW dbPrefix(X) TABLES LIKE ids(Y). { } cmd ::= SHOW dbPrefix(X) STABLES. { - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &X, 0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &X, 0); } cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { SToken token; tSetDbName(&token, &X); - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { From 13cfafaf2c7107901525622aac0b8f9e815319fa Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 10 Dec 2021 14:12:11 +0800 Subject: [PATCH 15/18] TD-10431 retrieve stable meta --- include/common/taosmsg.h | 37 ++--- include/util/taoserror.h | 1 + .../dnode/mgmt/impl/test/cluster/cluster.cpp | 6 +- .../dnode/mgmt/impl/test/profile/profile.cpp | 28 ++-- source/dnode/mgmt/impl/test/show/show.cpp | 14 +- source/dnode/mgmt/impl/test/user/user.cpp | 8 +- source/dnode/mnode/impl/inc/mndDef.h | 7 +- source/dnode/mnode/impl/src/mndCluster.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 4 +- source/dnode/mnode/impl/src/mndFunc.c | 4 +- source/dnode/mnode/impl/src/mndMnode.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 6 +- source/dnode/mnode/impl/src/mndStable.c | 128 ++++++++++++++---- source/dnode/mnode/impl/src/mndUser.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 4 +- source/libs/parser/src/parserUtil.c | 2 +- source/libs/parser/src/sql.c | 4 +- 18 files changed, 173 insertions(+), 88 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 3f9dd48ef4..2ce6da9806 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -274,7 +274,7 @@ typedef struct { typedef struct SSchema { int8_t type; - int16_t colId; + int32_t colId; int32_t bytes; char name[TSDB_COL_NAME_LEN]; } SSchema; @@ -764,13 +764,17 @@ typedef struct { } SAuthVnodeMsg; typedef struct { - char tableFname[TSDB_TABLE_FNAME_LEN]; - int16_t createFlag; - char tags[]; + char name[TSDB_TABLE_FNAME_LEN]; +} SStableInfoMsg; + +typedef struct { + char tableFname[TSDB_TABLE_FNAME_LEN]; + int8_t createFlag; + char tags[]; } STableInfoMsg; typedef struct { - uint8_t metaClone; // create local clone of the cached table meta + int8_t metaClone; // create local clone of the cached table meta int32_t numOfVgroups; int32_t numOfTables; int32_t numOfUdfs; @@ -788,25 +792,24 @@ typedef struct { } SVgroupMsg; typedef struct { - int32_t numOfVgroups; + int32_t numOfVgroups; SVgroupMsg vgroups[]; } SVgroupsMsg, SVgroupsInfo; -typedef struct STableMetaMsg { - int32_t contLen; +typedef struct { char tableFname[TSDB_TABLE_FNAME_LEN]; // table id - int8_t numOfTags; + char stableFname[TSDB_TABLE_FNAME_LEN]; + int32_t numOfTags; + int32_t numOfColumns; int8_t precision; int8_t tableType; - int16_t numOfColumns; - int16_t sversion; - int16_t tversion; - int32_t tid; - int64_t uid; + int8_t update; + int32_t sversion; + int32_t tversion; + uint64_t tuid; + uint64_t suid; SVgroupMsg vgroup; - char sTableName[TSDB_TABLE_FNAME_LEN]; - int64_t suid; - SSchema schema[]; + SSchema pSchema[]; } STableMetaMsg; typedef struct SMultiTableMeta { diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 39efdb3d04..36301466f8 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -183,6 +183,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0348) //"Mnode already exists") #define TSDB_CODE_MND_MNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0349) //"Mnode not there") +// mnode-table #define TSDB_CODE_MND_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) //"Table already exists") #define TSDB_CODE_MND_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0361) //"Table name too long") #define TSDB_CODE_MND_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0362) //"Table does not exist") diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index d7f9bca5ed..4e71a8016c 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -93,21 +93,21 @@ TEST_F(DndTestCluster, ShowCluster) { EXPECT_EQ(pMeta->suid, 0); SSchema* pSchema = NULL; - pSchema = &pMeta->schema[0]; + pSchema = &pMeta->pSchema[0]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "id"); - pSchema = &pMeta->schema[1]; + pSchema = &pMeta->pSchema[1]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "name"); - pSchema = &pMeta->schema[2]; + pSchema = &pMeta->pSchema[2]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 14e8dda457..a4632547a2 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -152,49 +152,49 @@ TEST_F(DndTestProfile, SConnectMsg_03) { EXPECT_EQ(pMeta->suid, 0); SSchema* pSchema = NULL; - pSchema = &pMeta->schema[0]; + pSchema = &pMeta->pSchema[0]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "connId"); - pSchema = &pMeta->schema[1]; + pSchema = &pMeta->pSchema[1]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "user"); - pSchema = &pMeta->schema[2]; + pSchema = &pMeta->pSchema[2]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "program"); - pSchema = &pMeta->schema[3]; + pSchema = &pMeta->pSchema[3]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "pid"); - pSchema = &pMeta->schema[4]; + pSchema = &pMeta->pSchema[4]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "ip:port"); - pSchema = &pMeta->schema[5]; + pSchema = &pMeta->pSchema[5]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); EXPECT_EQ(pSchema->bytes, 8); EXPECT_STREQ(pSchema->name, "login_time"); - pSchema = &pMeta->schema[6]; + pSchema = &pMeta->pSchema[6]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); @@ -493,28 +493,28 @@ TEST_F(DndTestProfile, SKillQueryMsg_03) { EXPECT_EQ(pMeta->suid, 0); SSchema* pSchema = NULL; - pSchema = &pMeta->schema[0]; + pSchema = &pMeta->pSchema[0]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "queryId"); - pSchema = &pMeta->schema[1]; + pSchema = &pMeta->pSchema[1]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "connId"); - pSchema = &pMeta->schema[2]; + pSchema = &pMeta->pSchema[2]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "user"); - pSchema = &pMeta->schema[3]; + pSchema = &pMeta->pSchema[3]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); @@ -680,21 +680,21 @@ TEST_F(DndTestProfile, SKillStreamMsg_03) { EXPECT_EQ(pMeta->suid, 0); SSchema* pSchema = NULL; - pSchema = &pMeta->schema[0]; + pSchema = &pMeta->pSchema[0]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "streamId"); - pSchema = &pMeta->schema[1]; + pSchema = &pMeta->pSchema[1]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "connId"); - pSchema = &pMeta->schema[2]; + pSchema = &pMeta->pSchema[2]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index e8c8fa261c..266f5858e4 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -154,49 +154,49 @@ TEST_F(DndTestShow, SShowMsg_04) { EXPECT_EQ(pMeta->suid, 0); SSchema* pSchema = NULL; - pSchema = &pMeta->schema[0]; + pSchema = &pMeta->pSchema[0]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "connId"); - pSchema = &pMeta->schema[1]; + pSchema = &pMeta->pSchema[1]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "user"); - pSchema = &pMeta->schema[2]; + pSchema = &pMeta->pSchema[2]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "program"); - pSchema = &pMeta->schema[3]; + pSchema = &pMeta->pSchema[3]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT); EXPECT_EQ(pSchema->bytes, 4); EXPECT_STREQ(pSchema->name, "pid"); - pSchema = &pMeta->schema[4]; + pSchema = &pMeta->pSchema[4]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "ip:port"); - pSchema = &pMeta->schema[5]; + pSchema = &pMeta->pSchema[5]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); EXPECT_EQ(pSchema->bytes, 8); EXPECT_STREQ(pSchema->name, "login_time"); - pSchema = &pMeta->schema[6]; + pSchema = &pMeta->pSchema[6]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 9dc8d99596..36f801fec2 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -93,28 +93,28 @@ TEST_F(DndTestUser, ShowUser) { SSchema* pSchema = NULL; - pSchema = &pMeta->schema[0]; + pSchema = &pMeta->pSchema[0]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, TSDB_USER_LEN + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "name"); - pSchema = &pMeta->schema[1]; + pSchema = &pMeta->pSchema[1]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->bytes, 10 + VARSTR_HEADER_SIZE); EXPECT_STREQ(pSchema->name, "privilege"); - pSchema = &pMeta->schema[2]; + pSchema = &pMeta->pSchema[2]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); EXPECT_EQ(pSchema->bytes, 8); EXPECT_STREQ(pSchema->name, "create_time"); - pSchema = &pMeta->schema[3]; + pSchema = &pMeta->pSchema[3]; pSchema->bytes = htons(pSchema->bytes); EXPECT_EQ(pSchema->colId, 0); EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index b9bafddc1c..7a7bc161da 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -47,7 +47,6 @@ typedef struct SAcctObj SAcctObj; typedef struct SUserObj SUserObj; typedef struct SDbObj SDbObj; typedef struct SVgObj SVgObj; -typedef struct STableObj STableObj; typedef struct SFuncObj SFuncObj; typedef struct SOperObj SOperObj; @@ -249,10 +248,10 @@ typedef struct SStableObj { int64_t updateTime; uint64_t uid; int32_t version; - int16_t numOfFields; - int16_t numOfTags; + int32_t numOfColumns; + int32_t numOfTags; SRWLatch lock; - SSchema *fieldSchema; + SSchema *columnSchema; SSchema *tagSchema; } SStableObj; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 97baeb0a59..c5c71d8751 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -138,7 +138,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index d9f8cbe4db..7477213890 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -659,7 +659,7 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index b614209e76..dc9f54888f 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -576,7 +576,7 @@ static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pMsg) { mInfo("cfg dnode rsp static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; @@ -654,7 +654,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 30569b96fb..d2d538b973 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -51,7 +51,7 @@ int32_t mndInitFunc(SMnode *pMnode) { mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndGetFuncMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndRetrieveFuncs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndCancelGetNextFunc); - + return sdbSetTable(pMnode->pSdb, table); } @@ -375,7 +375,7 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index d295e232a3..8a05eb02e8 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -363,7 +363,7 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index ad7a6322bb..b429879789 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -467,7 +467,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * mndReleaseUser(pMnode, pUser); int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; @@ -595,7 +595,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * mndReleaseUser(pMnode, pUser); int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; @@ -811,7 +811,7 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg mndReleaseUser(pMnode, pUser); int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; diff --git a/source/dnode/mnode/impl/src/mndStable.c b/source/dnode/mnode/impl/src/mndStable.c index a2a10a0752..b603f10b1b 100644 --- a/source/dnode/mnode/impl/src/mndStable.c +++ b/source/dnode/mnode/impl/src/mndStable.c @@ -21,6 +21,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndDb.h" +#include "tname.h" #define TSDB_STABLE_VER_NUM 1 #define TSDB_STABLE_RESERVE_SIZE 64 @@ -68,7 +69,7 @@ int32_t mndInitStable(SMnode *pMnode) { void mndCleanupStable(SMnode *pMnode) {} static SSdbRaw *mndStableActionEncode(SStableObj *pStable) { - int32_t size = sizeof(SStableObj) + (pStable->numOfFields + pStable->numOfTags) * sizeof(SSchema); + int32_t size = sizeof(SStableObj) + (pStable->numOfColumns + pStable->numOfTags) * sizeof(SSchema); SSdbRaw *pRaw = sdbAllocRaw(SDB_STABLE, TSDB_STABLE_VER_NUM, size); if (pRaw == NULL) return NULL; @@ -78,14 +79,14 @@ static SSdbRaw *mndStableActionEncode(SStableObj *pStable) { SDB_SET_INT64(pRaw, dataPos, pStable->updateTime) SDB_SET_INT64(pRaw, dataPos, pStable->uid) SDB_SET_INT64(pRaw, dataPos, pStable->version) - SDB_SET_INT16(pRaw, dataPos, pStable->numOfFields) - SDB_SET_INT16(pRaw, dataPos, pStable->numOfTags) + SDB_SET_INT32(pRaw, dataPos, pStable->numOfColumns) + SDB_SET_INT32(pRaw, dataPos, pStable->numOfTags) - for (int32_t i = 0; i < pStable->numOfFields; ++i) { - SSchema *pSchema = &pStable->fieldSchema[i]; + for (int32_t i = 0; i < pStable->numOfColumns; ++i) { + SSchema *pSchema = &pStable->columnSchema[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type); - SDB_SET_INT16(pRaw, dataPos, pSchema->colId); - SDB_SET_INT16(pRaw, dataPos, pSchema->bytes); + SDB_SET_INT32(pRaw, dataPos, pSchema->colId); + SDB_SET_INT32(pRaw, dataPos, pSchema->bytes); SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN); } @@ -124,16 +125,16 @@ static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pStable->updateTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pStable->uid) SDB_GET_INT32(pRaw, pRow, dataPos, &pStable->version) - SDB_GET_INT16(pRaw, pRow, dataPos, &pStable->numOfFields) - SDB_GET_INT16(pRaw, pRow, dataPos, &pStable->numOfTags) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStable->numOfColumns) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStable->numOfTags) - pStable->fieldSchema = calloc(pStable->numOfFields, sizeof(SSchema)); + pStable->columnSchema = calloc(pStable->numOfColumns, sizeof(SSchema)); pStable->tagSchema = calloc(pStable->numOfTags, sizeof(SSchema)); - for (int32_t i = 0; i < pStable->numOfFields; ++i) { - SSchema *pSchema = &pStable->fieldSchema[i]; + for (int32_t i = 0; i < pStable->numOfColumns; ++i) { + SSchema *pSchema = &pStable->columnSchema[i]; SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); - SDB_GET_INT16(pRaw, pRow, dataPos, &pSchema->colId); + SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId); SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); } @@ -141,7 +142,7 @@ static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw) { for (int32_t i = 0; i < pStable->numOfTags; ++i) { SSchema *pSchema = &pStable->tagSchema[i]; SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); - SDB_GET_INT16(pRaw, pRow, dataPos, &pSchema->colId); + SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId); SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); } @@ -167,24 +168,34 @@ static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStable, SStable atomic_exchange_32(&pOldStable->version, pNewStable->version); taosWLockLatch(&pOldStable->lock); - int16_t numOfTags = pNewStable->numOfTags; + int32_t numOfTags = pNewStable->numOfTags; int32_t tagSize = numOfTags * sizeof(SSchema); - int16_t numOfFields = pNewStable->numOfFields; - int32_t fieldSize = numOfFields * sizeof(SSchema); + int32_t numOfColumns = pNewStable->numOfColumns; + int32_t columnSize = numOfColumns * sizeof(SSchema); if (pOldStable->numOfTags < numOfTags) { pOldStable->tagSchema = malloc(tagSize); } - if (pOldStable->numOfFields < numOfFields) { - pOldStable->fieldSchema = malloc(fieldSize); + if (pOldStable->numOfColumns < numOfColumns) { + pOldStable->columnSchema = malloc(columnSize); } memcpy(pOldStable->tagSchema, pNewStable->tagSchema, tagSize); - memcpy(pOldStable->fieldSchema, pNewStable->fieldSchema, fieldSize); + memcpy(pOldStable->columnSchema, pNewStable->columnSchema, columnSize); taosWUnLockLatch(&pOldStable->lock); return 0; } +SStableObj *mndAcquireStb(SMnode *pMnode, char *stbName) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_STABLE, stbName); +} + +void mndReleaseStb(SMnode *pMnode, SStableObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pStb); +} + static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessCreateStableInRsp(SMnodeMsg *pMsg) { return 0; } @@ -197,7 +208,78 @@ static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessDropStableInRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg) { return 0; } +static SDbObj *mndGetDbByStbName(SMnode *pMnode, char *stbName) { + SName name = {0}; + tNameFromString(&name, stbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + char db[TSDB_TABLE_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); + + return mndAcquireDb(pMnode, db); +} + +static int32_t mndProcessStableMetaMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SStableInfoMsg *pInfo = pMsg->rpcMsg.pCont; + + mDebug("stable:%s, start to retrieve meta", pInfo->name); + + SDbObj *pDb = mndGetDbByStbName(pMnode, pInfo->name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("stable:%s, failed to retrieve meta since %s", pInfo->name, terrstr()); + return -1; + } + + SStableObj *pStb = mndAcquireStb(pMnode, pInfo->name); + if (pStb == NULL) { + mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_MND_INVALID_TABLE_NAME; + mError("stable:%s, failed to get meta since %s", pInfo->name, terrstr()); + return -1; + } + + int32_t contLen = sizeof(STableMetaMsg) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); + STableMetaMsg *pMeta = rpcMallocCont(contLen); + if (pMeta == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("stable:%s, failed to get meta since %s", pInfo->name, terrstr()); + return -1; + } + + memcpy(pMeta->stableFname, pStb->name, TSDB_TABLE_FNAME_LEN); + pMeta->numOfTags = htonl(pStb->numOfTags); + pMeta->numOfColumns = htonl(pStb->numOfColumns); + pMeta->precision = pDb->cfg.precision; + pMeta->tableType = TSDB_SUPER_TABLE; + pMeta->update = pDb->cfg.update; + pMeta->sversion = htonl(pStb->version); + pMeta->suid = htonl(pStb->uid); + + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pMeta->pSchema[i]; + SSchema *pColumn = &pStb->columnSchema[i]; + memcpy(pSchema->name, pColumn->name, TSDB_COL_NAME_LEN); + pSchema->type = pColumn->type; + pSchema->colId = htonl(pColumn->colId); + pSchema->bytes = htonl(pColumn->bytes); + } + + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pMeta->pSchema[i + pStb->numOfColumns]; + SSchema *pTag = &pStb->tagSchema[i]; + memcpy(pSchema->name, pTag->name, TSDB_COL_NAME_LEN); + pSchema->type = pTag->type; + pSchema->colId = htons(pTag->colId); + pSchema->bytes = htonl(pTag->bytes); + } + + pMsg->pCont = pMeta; + pMsg->contLen = contLen; + + mDebug("stable:%s, meta is retrieved, cols:%d tags:%d", pInfo->name, pStb->numOfColumns, pStb->numOfTags); + return 0; +} static int32_t mndGetNumOfStables(SMnode *pMnode, char *dbName, int32_t *pNumOfStables) { SSdb *pSdb = pMnode->pSdb; @@ -235,7 +317,7 @@ static int32_t mndGetStableMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg } int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; @@ -323,7 +405,7 @@ static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pStable->numOfFields; + *(int16_t *)pWrite = pStable->numOfColumns; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 0be3ffc80b..fc1f441a90 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -453,7 +453,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 7bc4ce0816..a0af5f7f5e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -201,7 +201,7 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg } int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; @@ -307,7 +307,7 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->schema; + SSchema *pSchema = pMeta->pSchema; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index e98443e2a1..3e83381a76 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1480,7 +1480,7 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { pTableMeta->tversion = pChild->tversion; pTableMeta->sversion = pChild->sversion; - memcpy(pTableMeta->schema, pChild->schema, sizeof(SSchema) * total); + memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total); int32_t num = pTableMeta->tableInfo.numOfColumns; for(int32_t i = 0; i < num; ++i) { diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 7158708989..2d6ec3f864 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -2312,14 +2312,14 @@ static void yy_reduce( break; case 26: /* cmd ::= SHOW dbPrefix STABLES */ { - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &yymsp[-1].minor.yy0, 0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &yymsp[-1].minor.yy0, 0); } break; case 27: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { SToken token; tSetDbName(&token, &yymsp[-3].minor.yy0); - setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); + setShowOptions(pInfo, TSDB_MGMT_TABLE_STABLE, &token, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= SHOW dbPrefix VGROUPS */ From 17a3469a16311f1a2240bd3253448f6a38c39da9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 10 Dec 2021 14:15:19 +0800 Subject: [PATCH 16/18] TD-10431 retrieve stable meta --- source/dnode/mnode/impl/src/mndStable.c | 126 ++++++++++++------------ 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStable.c b/source/dnode/mnode/impl/src/mndStable.c index b603f10b1b..b57b05f299 100644 --- a/source/dnode/mnode/impl/src/mndStable.c +++ b/source/dnode/mnode/impl/src/mndStable.c @@ -26,11 +26,11 @@ #define TSDB_STABLE_VER_NUM 1 #define TSDB_STABLE_RESERVE_SIZE 64 -static SSdbRaw *mndStableActionEncode(SStableObj *pStable); +static SSdbRaw *mndStableActionEncode(SStableObj *pStb); static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw); -static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStable); -static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStable); -static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStable, SStableObj *pNewStable); +static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStb); +static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStb); +static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStb, SStableObj *pNewStb); static int32_t mndProcessCreateStableMsg(SMnodeMsg *pMsg); static int32_t mndProcessAlterStableMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropStableMsg(SMnodeMsg *pMsg); @@ -68,30 +68,30 @@ int32_t mndInitStable(SMnode *pMnode) { void mndCleanupStable(SMnode *pMnode) {} -static SSdbRaw *mndStableActionEncode(SStableObj *pStable) { - int32_t size = sizeof(SStableObj) + (pStable->numOfColumns + pStable->numOfTags) * sizeof(SSchema); +static SSdbRaw *mndStableActionEncode(SStableObj *pStb) { + int32_t size = sizeof(SStableObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema); SSdbRaw *pRaw = sdbAllocRaw(SDB_STABLE, TSDB_STABLE_VER_NUM, size); if (pRaw == NULL) return NULL; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pStable->name, TSDB_TABLE_NAME_LEN) - SDB_SET_INT64(pRaw, dataPos, pStable->createdTime) - SDB_SET_INT64(pRaw, dataPos, pStable->updateTime) - SDB_SET_INT64(pRaw, dataPos, pStable->uid) - SDB_SET_INT64(pRaw, dataPos, pStable->version) - SDB_SET_INT32(pRaw, dataPos, pStable->numOfColumns) - SDB_SET_INT32(pRaw, dataPos, pStable->numOfTags) + SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) + SDB_SET_INT64(pRaw, dataPos, pStb->createdTime) + SDB_SET_INT64(pRaw, dataPos, pStb->updateTime) + SDB_SET_INT64(pRaw, dataPos, pStb->uid) + SDB_SET_INT64(pRaw, dataPos, pStb->version) + SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns) + SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags) - for (int32_t i = 0; i < pStable->numOfColumns; ++i) { - SSchema *pSchema = &pStable->columnSchema[i]; + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pStb->columnSchema[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type); SDB_SET_INT32(pRaw, dataPos, pSchema->colId); SDB_SET_INT32(pRaw, dataPos, pSchema->bytes); SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN); } - for (int32_t i = 0; i < pStable->numOfTags; ++i) { - SSchema *pSchema = &pStable->tagSchema[i]; + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pStb->tagSchema[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type); SDB_SET_INT32(pRaw, dataPos, pSchema->colId); SDB_SET_INT32(pRaw, dataPos, pSchema->bytes); @@ -116,31 +116,31 @@ static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw) { int32_t size = sizeof(SStableObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); SSdbRow *pRow = sdbAllocRow(size); - SStableObj *pStable = sdbGetRowObj(pRow); - if (pStable == NULL) return NULL; + SStableObj *pStb = sdbGetRowObj(pRow); + if (pStb == NULL) return NULL; int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, pRow, dataPos, pStable->name, TSDB_TABLE_NAME_LEN) - SDB_GET_INT64(pRaw, pRow, dataPos, &pStable->createdTime) - SDB_GET_INT64(pRaw, pRow, dataPos, &pStable->updateTime) - SDB_GET_INT64(pRaw, pRow, dataPos, &pStable->uid) - SDB_GET_INT32(pRaw, pRow, dataPos, &pStable->version) - SDB_GET_INT32(pRaw, pRow, dataPos, &pStable->numOfColumns) - SDB_GET_INT32(pRaw, pRow, dataPos, &pStable->numOfTags) + SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->createdTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->updateTime) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->uid) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->version) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfColumns) + SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfTags) - pStable->columnSchema = calloc(pStable->numOfColumns, sizeof(SSchema)); - pStable->tagSchema = calloc(pStable->numOfTags, sizeof(SSchema)); + pStb->columnSchema = calloc(pStb->numOfColumns, sizeof(SSchema)); + pStb->tagSchema = calloc(pStb->numOfTags, sizeof(SSchema)); - for (int32_t i = 0; i < pStable->numOfColumns; ++i) { - SSchema *pSchema = &pStable->columnSchema[i]; + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pStb->columnSchema[i]; SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId); SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); SDB_GET_BINARY(pRaw, pRow, dataPos, pSchema->name, TSDB_COL_NAME_LEN); } - for (int32_t i = 0; i < pStable->numOfTags; ++i) { - SSchema *pSchema = &pStable->tagSchema[i]; + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pStb->tagSchema[i]; SDB_GET_INT8(pRaw, pRow, dataPos, &pSchema->type); SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->colId); SDB_GET_INT32(pRaw, pRow, dataPos, &pSchema->bytes); @@ -152,37 +152,37 @@ static SSdbRow *mndStableActionDecode(SSdbRaw *pRaw) { return pRow; } -static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStable) { - mTrace("stable:%s, perform insert action", pStable->name); +static int32_t mndStableActionInsert(SSdb *pSdb, SStableObj *pStb) { + mTrace("stable:%s, perform insert action", pStb->name); return 0; } -static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStable) { - mTrace("stable:%s, perform delete action", pStable->name); +static int32_t mndStableActionDelete(SSdb *pSdb, SStableObj *pStb) { + mTrace("stable:%s, perform delete action", pStb->name); return 0; } -static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStable, SStableObj *pNewStable) { - mTrace("stable:%s, perform update action", pOldStable->name); - atomic_exchange_32(&pOldStable->updateTime, pNewStable->updateTime); - atomic_exchange_32(&pOldStable->version, pNewStable->version); +static int32_t mndStableActionUpdate(SSdb *pSdb, SStableObj *pOldStb, SStableObj *pNewStb) { + mTrace("stable:%s, perform update action", pOldStb->name); + atomic_exchange_32(&pOldStb->updateTime, pNewStb->updateTime); + atomic_exchange_32(&pOldStb->version, pNewStb->version); - taosWLockLatch(&pOldStable->lock); - int32_t numOfTags = pNewStable->numOfTags; + taosWLockLatch(&pOldStb->lock); + int32_t numOfTags = pNewStb->numOfTags; int32_t tagSize = numOfTags * sizeof(SSchema); - int32_t numOfColumns = pNewStable->numOfColumns; + int32_t numOfColumns = pNewStb->numOfColumns; int32_t columnSize = numOfColumns * sizeof(SSchema); - if (pOldStable->numOfTags < numOfTags) { - pOldStable->tagSchema = malloc(tagSize); + if (pOldStb->numOfTags < numOfTags) { + pOldStb->tagSchema = malloc(tagSize); } - if (pOldStable->numOfColumns < numOfColumns) { - pOldStable->columnSchema = malloc(columnSize); + if (pOldStb->numOfColumns < numOfColumns) { + pOldStb->columnSchema = malloc(columnSize); } - memcpy(pOldStable->tagSchema, pNewStable->tagSchema, tagSize); - memcpy(pOldStable->columnSchema, pNewStable->columnSchema, columnSize); - taosWUnLockLatch(&pOldStable->lock); + memcpy(pOldStb->tagSchema, pNewStb->tagSchema, tagSize); + memcpy(pOldStb->columnSchema, pNewStb->columnSchema, columnSize); + taosWUnLockLatch(&pOldStb->lock); return 0; } @@ -293,15 +293,15 @@ static int32_t mndGetNumOfStables(SMnode *pMnode, char *dbName, int32_t *pNumOfS int32_t numOfStables = 0; void *pIter = NULL; while (1) { - SStableObj *pStable = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pStable); + SStableObj *pStb = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pStb); if (pIter == NULL) break; - if (strcmp(pStable->db, dbName) == 0) { + if (strcmp(pStb->db, dbName) == 0) { numOfStables++; } - sdbRelease(pSdb, pStable); + sdbRelease(pSdb, pStb); } *pNumOfStables = numOfStables; @@ -374,7 +374,7 @@ static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, SMnode *pMnode = pMsg->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; - SStableObj *pStable = NULL; + SStableObj *pStb = NULL; int32_t cols = 0; char *pWrite; char prefix[64] = {0}; @@ -384,36 +384,36 @@ static int32_t mndRetrieveStables(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t prefixLen = (int32_t)strlen(prefix); while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_STABLE, pShow->pIter, (void **)&pStable); + pShow->pIter = sdbFetch(pSdb, SDB_STABLE, pShow->pIter, (void **)&pStb); if (pShow->pIter == NULL) break; - if (strncmp(pStable->name, prefix, prefixLen) != 0) { - sdbRelease(pSdb, pStable); + if (strncmp(pStb->name, prefix, prefixLen) != 0) { + sdbRelease(pSdb, pStb); continue; } cols = 0; char stableName[TSDB_TABLE_FNAME_LEN] = {0}; - memcpy(stableName, pStable->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen); + memcpy(stableName, pStb->name + prefixLen, TSDB_TABLE_FNAME_LEN - prefixLen); pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; STR_TO_VARSTR(pWrite, stableName); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int64_t *)pWrite = pStable->createdTime; + *(int64_t *)pWrite = pStb->createdTime; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pStable->numOfColumns; + *(int16_t *)pWrite = pStb->numOfColumns; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pStable->numOfTags; + *(int16_t *)pWrite = pStb->numOfTags; cols++; numOfRows++; - sdbRelease(pSdb, pStable); + sdbRelease(pSdb, pStb); } pShow->numOfReads += numOfRows; From 29278327133ab58f2a5ad5ae8f9be1a442273854 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 10 Dec 2021 15:30:39 +0800 Subject: [PATCH 17/18] fix mem leak --- source/libs/index/src/index_fst.c | 34 +++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 403b4a9122..e7e9be5263 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -803,6 +803,7 @@ void fstBuilderDestroy(FstBuilder *b) { fstCountingWriterDestroy(b->wrt); fstUnFinishedNodesDestroy(b->unfinished); fstRegistryDestroy(b->registry); + fstSliceDestroy(&b->last); free(b); } @@ -869,13 +870,15 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { CompiledAddr addr = NONE_ADDRESS; while (istate + 1 < FST_UNFINISHED_NODES_LEN(b->unfinished)) { - FstBuilderNode *n = NULL; + FstBuilderNode *bn = NULL; if (addr == NONE_ADDRESS) { - n = fstUnFinishedNodesPopEmpty(b->unfinished); + bn = fstUnFinishedNodesPopEmpty(b->unfinished); } else { - n = fstUnFinishedNodesPopFreeze(b->unfinished, addr); + bn = fstUnFinishedNodesPopFreeze(b->unfinished, addr); } - addr = fstBuilderCompile(b, n); + addr = fstBuilderCompile(b, bn); + + fstBuilderNodeDestroy(bn); assert(addr != NONE_ADDRESS); //fstBuilderNodeDestroy(n); } @@ -910,6 +913,7 @@ void* fstBuilderInsertInner(FstBuilder *b) { fstBuilderCompileFrom(b, 0); FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished); CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); + fstBuilderNodeDestroy(rootNode); char buf64[8] = {0}; @@ -1026,7 +1030,10 @@ Fst* fstCreate(FstSlice *slice) { fst->meta->ty = type; fst->meta->len = fstLen; fst->meta->checkSum = checkSum; - fst->data = slice; + + FstSlice *s = calloc(1, sizeof(FstSlice)); + *s = fstSliceCopy(slice, 0, FST_SLICE_LEN(slice)); + fst->data = s; return fst; @@ -1038,7 +1045,8 @@ FST_CREAT_FAILED: void fstDestroy(Fst *fst) { if (fst) { free(fst->meta); - fstNodeDestroy(fst->root); + fstSliceDestroy(fst->data); + free(fst->data); } free(fst); } @@ -1048,6 +1056,9 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { Output tOut = 0; int32_t len; uint8_t *data = fstSliceData(b, &len); + + SArray *nodes = (SArray *)taosArrayInit(len, sizeof(FstNode *)); + taosArrayPush(nodes, &root); for (uint32_t i = 0; i < len; i++) { uint8_t inp = data[i]; Output res = 0; @@ -1059,12 +1070,22 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { fstNodeGetTransitionAt(root, res, &trn); tOut += trn.out; root = fstGetNode(fst, trn.addr); + taosArrayPush(nodes, &root); + //fstNodeDestroy(root); } if (!FST_NODE_IS_FINAL(root)) { return false; } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); } + for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { + FstNode **node = (FstNode **)taosArrayGet(nodes, i); + fstNodeDestroy(*node); + } + taosArrayDestroy(nodes); + fst->root = NULL; + + *out = tOut; return true; @@ -1228,6 +1249,7 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { taosArrayPush(sws->stack, &s); out += trn.out; node = fstGetNode(sws->fst, trn.addr); + fstNodeDestroy(node); } else { // This is a little tricky. We're in this case if the From f8ca5dc15b2ef809e10f5624daa956778a6735c0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 10 Dec 2021 16:23:22 +0800 Subject: [PATCH 18/18] mem leak --- source/libs/index/src/index_fst.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index e7e9be5263..6a81e888d2 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -591,14 +591,14 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { uint8_t *data = fstSliceData(&t, &len); int i = 0; for(; i < len; i++) { - //uint8_t v = slice->data[slice->start + i]; - ////slice->data[slice->start + i]; uint8_t v = data[i]; if (v == b) { + fstSliceDestroy(&t); return node->nTrans - i - 1; // bug } } if (i == len) { *null = true; } + fstSliceDestroy(&t); } } @@ -634,7 +634,7 @@ FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *slice) { } else if (st.state == OneTrans) { FstSlice data = fstSliceCopy(slice, 0, addr); PackSizes sz = fstStateSizes(&st, &data); - n->data = fstSliceCopy(slice, 0, addr); + n->data = data; n->version = version; n->state = st; n->start = addr; @@ -852,8 +852,9 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { FstSlice *input = &bs; if (fstSliceIsEmpty(&b->last)) { + fstSliceDestroy(&b->last); // deep copy or not - b->last = fstSliceCopy(&bs, input->start, input->end); + b->last = fstSliceDeepCopy(&bs, input->start, input->end); } else { int comp = fstSliceCompare(&b->last, &bs); if (comp == 0 && ckDup) { @@ -863,7 +864,7 @@ OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { } // deep copy or not fstSliceDestroy(&b->last); - b->last = fstSliceCopy(&bs, input->start, input->end); + b->last = fstSliceDeepCopy(&bs, input->start, input->end); } return Ordered; } @@ -1078,14 +1079,14 @@ bool fstGet(Fst *fst, FstSlice *b, Output *out) { } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); } + for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { FstNode **node = (FstNode **)taosArrayGet(nodes, i); fstNodeDestroy(*node); } taosArrayDestroy(nodes); + fst->root = NULL; - - *out = tOut; return true;