diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 9bae38e3a7..e14788be47 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -81,6 +81,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE, "alter-stable" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE, "drop-stable" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STABLE_VGROUP, "stable-vgroup" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_HEARTBEAT, "heartbeat" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW, "show" ) @@ -153,7 +154,8 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY9, "dummy9" ) #define TSDB_IE_TYPE_DNODE_EXT 6 #define TSDB_IE_TYPE_DNODE_STATE 7 -enum _mgmt_table { +typedef enum _mgmt_table { + TSDB_MGMT_TABLE_START, TSDB_MGMT_TABLE_ACCT, TSDB_MGMT_TABLE_USER, TSDB_MGMT_TABLE_DB, @@ -175,7 +177,7 @@ enum _mgmt_table { TSDB_MGMT_TABLE_TP, TSDB_MGMT_TABLE_FUNCTION, TSDB_MGMT_TABLE_MAX, -}; +} EShowType; #define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1 #define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2 @@ -352,11 +354,9 @@ typedef struct { } SUpdateTableTagValMsg; typedef struct { - char clientVersion[TSDB_VERSION_LEN]; - char msgVersion[TSDB_VERSION_LEN]; - char db[TSDB_TABLE_FNAME_LEN]; - char appName[TSDB_APPNAME_LEN]; int32_t pid; + char app[TSDB_APP_NAME_LEN]; + char db[TSDB_DB_NAME_LEN]; } SConnectMsg; typedef struct SEpSet { @@ -367,15 +367,14 @@ typedef struct SEpSet { } SEpSet; typedef struct { - char acctId[TSDB_ACCT_ID_LEN]; - char serverVersion[TSDB_VERSION_LEN]; - char clusterId[TSDB_CLUSTER_ID_LEN]; - int8_t writeAuth; - int8_t superAuth; - int8_t reserved1; - int8_t reserved2; - int32_t connId; - SEpSet epSet; + int32_t acctId; + int32_t clusterId; + int32_t connId; + int8_t superAuth; + int8_t readAuth; + int8_t writeAuth; + int8_t reserved[5]; + SEpSet epSet; } SConnectRsp; typedef struct { @@ -874,23 +873,23 @@ typedef struct { } SStreamDesc; typedef struct { - char clientVer[TSDB_VERSION_LEN]; - uint32_t connId; - int32_t pid; - int32_t numOfQueries; - int32_t numOfStreams; - char appName[TSDB_APPNAME_LEN]; - char pData[]; + int32_t connId; + int32_t pid; + int32_t numOfQueries; + int32_t numOfStreams; + char app[TSDB_APP_NAME_LEN]; + char pData[]; } SHeartBeatMsg; typedef struct { - uint32_t queryId; - uint32_t streamId; - uint32_t totalDnodes; - uint32_t onlineDnodes; - uint32_t connId; - int8_t killConnection; - SEpSet epSet; + int32_t connId; + int32_t queryId; + int32_t streamId; + int32_t totalDnodes; + int32_t onlineDnodes; + int8_t killConnection; + int8_t reserved[3]; + SEpSet epSet; } SHeartBeatRsp; typedef struct { diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index c7415af0d5..0a897b93f8 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -57,6 +57,7 @@ typedef struct { int32_t sver; int32_t statusInterval; int32_t mnodeEqualVnodeNum; + int32_t shellActivityTimer; char *timezone; char *locale; char *charset; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index e77540bf90..b514648bbd 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -39,6 +39,14 @@ typedef enum { } EWalType; typedef struct { + //union { + //uint32_t info; + //struct { + //uint32_t sver:3; + //uint32_t msgtype: 5; + //uint32_t reserved : 24; + //}; + //}; int8_t sver; uint8_t msgType; int8_t reserved[2]; @@ -71,13 +79,17 @@ typedef struct { #define WAL_FILESET_MAX 128 #define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) -#define WAL_CUR_POS_READ_ONLY 1 -#define WAL_CUR_FILE_READ_ONLY 2 +#define WAL_CUR_POS_WRITABLE 1 +#define WAL_CUR_FILE_WRITABLE 2 +#define WAL_CUR_FAILED 4 typedef struct SWal { // cfg int32_t vgId; int32_t fsyncPeriod; // millisecond + int32_t fsyncSeq; + int32_t rollPeriod; // second + int64_t segSize; EWalType level; //reference int64_t refId; @@ -86,7 +98,7 @@ typedef struct SWal { int64_t curIdxTfd; //current version int64_t curVersion; - int64_t curOffset; + int64_t curLogOffset; //current file version int64_t curFileFirstVersion; int64_t curFileLastVersion; @@ -94,8 +106,10 @@ typedef struct SWal { int64_t firstVersion; int64_t snapshotVersion; int64_t lastVersion; - //fsync status - int32_t fsyncSeq; + int64_t lastFileName; + //roll status + int64_t lastRollSeq; + int64_t lastFileWriteSize; //ctl int32_t curStatus; pthread_mutex_t mutex; @@ -119,12 +133,10 @@ int32_t walAlter(SWal *, SWalCfg *pCfg); void walClose(SWal *); // write -//int64_t walWriteWithMsgType(SWal*, int8_t msgType, void* body, int32_t bodyLen); int64_t walWrite(SWal *, int64_t index, uint8_t msgType, void *body, int32_t bodyLen); -//int64_t walWriteBatch(SWal *, void **bodies, int32_t *bodyLen, int32_t batchSize); +void walFsync(SWal *, bool force); // apis for lifecycle management -void walFsync(SWal *, bool force); int32_t walCommit(SWal *, int64_t ver); // truncate after int32_t walRollback(SWal *, int64_t ver); diff --git a/include/util/tdef.h b/include/util/tdef.h index 76df8887a0..897f51f5c1 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -174,7 +174,7 @@ do { \ #define TSDB_MAX_SQL_SHOW_LEN 512 #define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb -#define TSDB_APPNAME_LEN TSDB_UNI_LEN +#define TSDB_APP_NAME_LEN TSDB_UNI_LEN /** * In some scenarios uint16_t (0~65535) is used to store the row len. diff --git a/include/util/tfile.h b/include/util/tfile.h index ff62c9e341..3d0e2177ac 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -28,6 +28,7 @@ void tfCleanup(); // the same syntax as UNIX standard open/close/read/write // but FD is int64_t and will never be reused +int64_t tfOpenRead(const char *pathname); int64_t tfOpenReadWrite(const char *pathname); int64_t tfOpenCreateWrite(const char *pathname); int64_t tfOpenCreateWriteAppend(const char *pathname); diff --git a/include/util/tutil.h b/include/util/tutil.h index 8dbcb7e8d5..573dee9339 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -41,8 +41,10 @@ char * paGetToken(char *src, char **token, int32_t *tokenLen); int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); -char * taosIpStr(uint32_t ipInt); +char *taosIpStr(uint32_t ipInt); uint32_t ip2uint(const char *const ip_addr); +void taosIp2String(uint32_t ip, char *str); +void taosIpPort2String(uint32_t ip, uint16_t port, char *str); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { MD5_CTX context; diff --git a/source/dnode/mgmt/impl/CMakeLists.txt b/source/dnode/mgmt/impl/CMakeLists.txt index 3eadf24164..51131ede6a 100644 --- a/source/dnode/mgmt/impl/CMakeLists.txt +++ b/source/dnode/mgmt/impl/CMakeLists.txt @@ -14,3 +14,7 @@ target_include_directories( PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 4ec08d5fb3..4afce4655d 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -334,6 +334,7 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->sver = pDnode->opt.sver; pOption->statusInterval = pDnode->opt.statusInterval; pOption->mnodeEqualVnodeNum = pDnode->opt.mnodeEqualVnodeNum; + pOption->shellActivityTimer = pDnode->opt.shellActivityTimer; pOption->timezone = pDnode->opt.timezone; pOption->charset = pDnode->opt.charset; pOption->locale = pDnode->opt.locale; @@ -675,7 +676,7 @@ void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) { + if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pReadQ, pMsg) != 0) { SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; rpcSendResponse(&rsp); rpcFreeCont(pMsg->pCont); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 44ca1d1407..98a0b8e308 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -76,6 +76,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg; @@ -157,7 +158,7 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.label = "DND-C"; rpcInit.numOfThreads = 1; rpcInit.cfp = dndProcessResponse; - rpcInit.sessions = 8; + rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000; rpcInit.user = INTERNAL_USER; diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt new file mode 100644 index 0000000000..95ffa548b2 --- /dev/null +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(test01) \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/test01/CMakeLists.txt b/source/dnode/mgmt/impl/test/test01/CMakeLists.txt new file mode 100644 index 0000000000..c3037dfdff --- /dev/null +++ b/source/dnode/mgmt/impl/test/test01/CMakeLists.txt @@ -0,0 +1,29 @@ +add_executable(dndTest01 "") + +target_sources(dndTest01 + PRIVATE + "dndTest01.cpp" + "../util/dndTestDeploy.cpp" +) + +target_link_libraries( + dndTest01 + PUBLIC dnode + PUBLIC util + PUBLIC os + PUBLIC gtest_main +) + +target_include_directories(dndTest01 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../util" +) + +enable_testing() + +add_test( + NAME dndTest01 + COMMAND dndTest01 +) diff --git a/source/dnode/mgmt/impl/test/test01/dndTest01.cpp b/source/dnode/mgmt/impl/test/test01/dndTest01.cpp new file mode 100644 index 0000000000..70f649cc07 --- /dev/null +++ b/source/dnode/mgmt/impl/test/test01/dndTest01.cpp @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "dndTestDeploy.h" + +class DndTest01 : public ::testing::Test { + protected: + void SetUp() override { + pServer = createServer("/tmp/dndTest01"); + pClient = createClient("root", "taosdata"); + } + void TearDown() override { + dropServer(pServer); + dropClient(pClient); + } + + SServer* pServer; + SClient* pClient; +}; + +TEST_F(DndTest01, connectMsg) { + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + pReq->pid = htonl(1234); + strcpy(pReq->app, "test01"); + strcpy(pReq->db, ""); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SConnectMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; + + sendMsg(pClient, &rpcMsg); + + SConnectRsp* pRsp = (SConnectRsp*)pClient->pRsp; + ASSERT(pRsp); + pRsp->acctId = htonl(pRsp->acctId); + pRsp->clusterId = htonl(pRsp->clusterId); + pRsp->connId = htonl(pRsp->connId); + pRsp->epSet.port[0] = htonl(pRsp->epSet.port[0]); + + EXPECT_EQ(pRsp->acctId, 1); + EXPECT_GT(pRsp->clusterId, 0); + EXPECT_GT(pRsp->connId, 1); + EXPECT_EQ(pRsp->superAuth, 1); + EXPECT_EQ(pRsp->readAuth, 1); + EXPECT_EQ(pRsp->writeAuth, 1); + + EXPECT_EQ(pRsp->epSet.inUse, 0); + EXPECT_EQ(pRsp->epSet.numOfEps, 1); + EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); +} + +// TEST_F(DndTest01, heartbeatMsg) { +// SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); +// pReq->connId = htonl(1); +// pReq->pid = htonl(1234); +// pReq->numOfQueries = htonl(0); +// pReq->numOfStreams = htonl(0); +// strcpy(pReq->app, "test01"); + +// SRpcMsg rpcMsg = {0}; +// rpcMsg.pCont = pReq; +// rpcMsg.contLen = sizeof(SHeartBeatMsg); +// rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; + +// sendMsg(pClient, &rpcMsg); + +// SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pClient->pRsp; +// ASSERT(pRsp); + // pRsp->epSet.port[0] = htonl(pRsp->epSet.port[0]); + +// EXPECT_EQ(htonl(pRsp->connId), 1); +// EXPECT_GT(htonl(pRsp->queryId), 0); +// EXPECT_GT(htonl(pRsp->streamId), 1); +// EXPECT_EQ(htonl(pRsp->totalDnodes), 1); +// EXPECT_EQ(htonl(pRsp->onlineDnodes), 1); +// EXPECT_EQ(pRsp->killConnection, 0); + // EXPECT_EQ(pRsp->epSet.inUse, 0); + // EXPECT_EQ(pRsp->epSet.numOfEps, 1); + // EXPECT_EQ(pRsp->epSet.port[0], 9527); + // EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); +// } diff --git a/source/dnode/mgmt/impl/test/util/dndTestDeploy.cpp b/source/dnode/mgmt/impl/test/util/dndTestDeploy.cpp new file mode 100644 index 0000000000..4b4f0fa763 --- /dev/null +++ b/source/dnode/mgmt/impl/test/util/dndTestDeploy.cpp @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "dndTestDeploy.h" + +void initLog(char *path) { + mDebugFlag = 207; + char temp[PATH_MAX]; + snprintf(temp, PATH_MAX, "%s/taosdlog", path); + if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + printf("failed to init log file\n"); + } +} + +void* runServer(void* param) { + SServer* pServer = (SServer*)param; + while (1) { + taosMsleep(100); + pthread_testcancel(); + } +} + +void initOption(SDnodeOpt* pOption, char *path) { + pOption->sver = 1; + pOption->numOfCores = 1; + pOption->numOfSupportMnodes = 1; + pOption->numOfSupportVnodes = 1; + pOption->numOfSupportQnodes = 1; + pOption->statusInterval = 1; + pOption->mnodeEqualVnodeNum = 1; + pOption->numOfThreadsPerCore = 1; + pOption->ratioOfQueryCores = 1; + pOption->maxShellConns = 1000; + pOption->shellActivityTimer = 30; + pOption->serverPort = 9527; + strcpy(pOption->dataDir, path); + strcpy(pOption->localEp, "localhost:9527"); + strcpy(pOption->localFqdn, "localhost"); + strcpy(pOption->firstEp, "localhost:9527"); + + taosRemoveDir(path); + taosMkDir(path); +} + +SServer* createServer(char *path) { + SDnodeOpt option = {0}; + initOption(&option, path); + + SDnode* pDnode = dndInit(&option); + ASSERT(pDnode); + + SServer* pServer = (SServer*)calloc(1, sizeof(SServer)); + ASSERT(pServer); + + pServer->pDnode = pDnode; + pServer->threadId = taosCreateThread(runServer, pServer); + ASSERT(pServer->threadId); + + return pServer; +} + +void dropServer(SServer* pServer) { + if (pServer->threadId != NULL) { + taosDestoryThread(pServer->threadId); + } +} + +void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + SClient* pClient = (SClient*)parent; + pClient->pRsp = pMsg; + taosMsleep(100000); + tsem_post(&pClient->sem); +} + +SClient* createClient(char *user, char *pass) { + SClient* pClient = (SClient*)calloc(1, sizeof(SClient)); + ASSERT(pClient); + + char secretEncrypt[32] = {0}; + taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.label = "DND-C"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processClientRsp; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = 30 * 1000; + rpcInit.user = user; + rpcInit.ckey = "key"; + rpcInit.parent = pClient; + rpcInit.secret = (char*)secretEncrypt; + rpcInit.parent = pClient; + // rpcInit.spi = 1; + + pClient->clientRpc = rpcOpen(&rpcInit); + ASSERT(pClient->clientRpc); + + tsem_init(&pClient->sem, 0, 0); + + return pClient; +} + +void dropClient(SClient* pClient) { + tsem_destroy(&pClient->sem); + rpcClose(pClient->clientRpc); +} + +void sendMsg(SClient* pClient, SRpcMsg* pMsg) { + SEpSet epSet = {0}; + epSet.inUse = 0; + epSet.numOfEps = 1; + epSet.port[0] = 9527; + strcpy(epSet.fqdn[0], "localhost"); + + rpcSendRequest(pClient->clientRpc, &epSet, pMsg, NULL); + tsem_wait(&pClient->sem); +} diff --git a/source/dnode/mgmt/impl/test/util/dndTestDeploy.h b/source/dnode/mgmt/impl/test/util/dndTestDeploy.h new file mode 100644 index 0000000000..6c47b03fe7 --- /dev/null +++ b/source/dnode/mgmt/impl/test/util/dndTestDeploy.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include "os.h" + +#include "dnode.h" +#include "taosmsg.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "trpc.h" +#include "tthread.h" +#include "ulog.h" + +typedef struct { + SDnode* pDnode; + pthread_t* threadId; +} SServer; + +typedef struct { + void* clientRpc; + SRpcMsg* pRsp; + tsem_t sem; +} SClient; + +SServer* createServer(char* path); +void dropServer(SServer* pServer); +SClient* createClient(char *user, char *pass); +void dropClient(SClient* pClient); +void sendMsg(SClient* pClient, SRpcMsg* pMsg); diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index acccb62603..91f502be7d 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -24,6 +24,8 @@ extern "C" { int32_t mndInitDb(SMnode *pMnode); void mndCleanupDb(SMnode *pMnode); +SDbObj *mndAcquireDb(SMnode *pMnode, char *db); +void mndReleaseDb(SMnode *pMnode, SDbObj *pDb); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9facb8829c..aaf86c15b6 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -185,9 +185,11 @@ typedef struct SUserObj { char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; - int8_t rootAuth; + int8_t superAuth; + int8_t readAuth; + int8_t writeAuth; + int32_t acctId; SHashObj *prohibitDbHash; - SAcctObj *pAcct; } SUserObj; typedef struct { @@ -281,26 +283,30 @@ typedef struct SFuncObj { int16_t type; } SFuncObj; -typedef struct { - int8_t type; - int8_t maxReplica; - int16_t numOfColumns; - int32_t index; - int32_t rowSize; - int32_t numOfRows; - int32_t numOfReads; - uint16_t payloadLen; - void *pIter; - void *pVgIter; - void **ppShow; - char db[TSDB_FULL_DB_NAME_LEN]; - int16_t offset[TSDB_MAX_COLUMNS]; - int32_t bytes[TSDB_MAX_COLUMNS]; - char payload[]; +typedef struct SShowObj SShowObj; +typedef struct SShowObj { + int8_t type; + int8_t maxReplica; + int16_t numOfColumns; + int32_t id; + int32_t rowSize; + int32_t numOfRows; + int32_t numOfReads; + uint16_t payloadLen; + void *pIter; + void *pVgIter; + SMnode *pMnode; + SShowObj **ppShow; + char db[TSDB_FULL_DB_NAME_LEN]; + int16_t offset[TSDB_MAX_COLUMNS]; + int32_t bytes[TSDB_MAX_COLUMNS]; + char payload[]; } SShowObj; typedef struct SMnodeMsg { char user[TSDB_USER_LEN]; + char db[TSDB_FULL_DB_NAME_LEN]; + int32_t acctId; SMnode *pMnode; int16_t received; int16_t successed; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 8910ed4e63..dbca6ad057 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -18,15 +18,19 @@ #include "mndDef.h" #include "sdb.h" +#include "tcache.h" #include "tqueue.h" #ifdef __cplusplus extern "C" { #endif -typedef int32_t (*MndMsgFp)(SMnode *pMnode, SMnodeMsg *pMsg); +typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); +typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +typedef int32_t (*ShowRetrieveFp)(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef struct { const char *name; @@ -34,6 +38,19 @@ typedef struct { MndCleanupFp cleanupFp; } SMnodeStep; +typedef struct { + int32_t showId; + ShowMetaFp metaFps[TSDB_MGMT_TABLE_MAX]; + ShowRetrieveFp retrieveFps[TSDB_MGMT_TABLE_MAX]; + ShowFreeIterFp freeIterFps[TSDB_MGMT_TABLE_MAX]; + SCacheObj *cache; +} SShowMgmt; + +typedef struct { + int32_t connId; + SCacheObj *cache; +} SProfileMgmt; + typedef struct SMnode { int32_t dnodeId; int32_t clusterId; @@ -45,6 +62,8 @@ typedef struct SMnode { SSdb *pSdb; SDnode *pDnode; SArray *pSteps; + SShowMgmt showMgmt; + SProfileMgmt profileMgmt; MndMsgFp msgFp[TSDB_MSG_TYPE_MAX]; SendMsgToDnodeFp sendMsgToDnodeFp; SendMsgToMnodeFp sendMsgToMnodeFp; @@ -53,6 +72,7 @@ typedef struct SMnode { int32_t sver; int32_t statusInterval; int32_t mnodeEqualVnodeNum; + int32_t shellActivityTimer; char *timezone; char *locale; char *charset; diff --git a/source/dnode/mnode/impl/inc/mndMnode.h b/source/dnode/mnode/impl/inc/mndMnode.h index 53f7b733f2..64bbb13b60 100644 --- a/source/dnode/mnode/impl/inc/mndMnode.h +++ b/source/dnode/mnode/impl/inc/mndMnode.h @@ -25,6 +25,7 @@ extern "C" { int32_t mndInitMnode(SMnode *pMnode); void mndCleanupMnode(SMnode *pMnode); bool mndIsMnode(SMnode *pMnode, int32_t dnodeId); +void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index 06c18cb029..e7a3fcd45f 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -24,6 +24,10 @@ extern "C" { int32_t mndInitShow(SMnode *pMnode); void mndCleanupShow(SMnode *pMnode); +void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp); +void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp); +void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType msgType, ShowFreeIterFp fp); +void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 4d31a87b19..e4ecdf3283 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -24,7 +24,7 @@ extern "C" { int32_t mndInitUser(SMnode *pMnode); void mndCleanupUser(SMnode *pMnode); -SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName); +SUserObj *mndAcquireUser(SMnode *pMnode, char *userName); void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 23328506e6..e3d37cd9f9 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -14,10 +14,61 @@ */ #define _DEFAULT_SOURCE -#include "mndInt.h" +#include "mndAcct.h" +#include "mndShow.h" #define SDB_ACCT_VER 1 +static int32_t mnodeCreateDefaultAcct(SMnode *pMnode); +static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct); +static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw); +static int32_t mnodeAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); +static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); +static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pSrcAcct, SAcctObj *pDstAcct); +static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg); +static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg); +static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg); + +int32_t mndInitAcct(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_ACCT, + .keyType = SDB_KEY_BINARY, + .deployFp = mnodeCreateDefaultAcct, + .encodeFp = (SdbEncodeFp)mnodeAcctActionEncode, + .decodeFp = (SdbDecodeFp)mnodeAcctActionDecode, + .insertFp = (SdbInsertFp)mnodeAcctActionInsert, + .updateFp = (SdbUpdateFp)mnodeAcctActionUpdate, + .deleteFp = (SdbDeleteFp)mnodeAcctActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_ACCT, mndProcessCreateAcctMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_ACCT, mndProcessAlterAcctMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_ACCT, mndProcessDropAcctMsg); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupAcct(SMnode *pMnode) {} + +static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { + SAcctObj acctObj = {0}; + tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN); + acctObj.createdTime = taosGetTimestampMs(); + acctObj.updateTime = acctObj.createdTime; + acctObj.acctId = 1; + acctObj.cfg = (SAcctCfg){.maxUsers = 1024, + .maxDbs = 1024, + .maxTimeSeries = INT32_MAX, + .maxStreams = 8092, + .maxStorage = INT64_MAX, + .accessState = TSDB_VN_ALL_ACCCESS}; + + SSdbRaw *pRaw = mnodeAcctActionEncode(&acctObj); + if (pRaw == NULL) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + + mTrace("acct:%s, will be created while deploy sdb", acctObj.acct); + return sdbWrite(pMnode->pSdb, pRaw); +} + static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct) { SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj)); if (pRaw == NULL) return NULL; @@ -92,40 +143,20 @@ static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pSrcAcct, SAcctObj *p return 0; } -static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { - int32_t code = 0; - - SAcctObj acctObj = {0}; - tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN); - acctObj.createdTime = taosGetTimestampMs(); - acctObj.updateTime = acctObj.createdTime; - acctObj.acctId = 1; - acctObj.cfg = (SAcctCfg){.maxUsers = 1024, - .maxDbs = 1024, - .maxTimeSeries = INT32_MAX, - .maxStreams = 8092, - .maxStorage = INT64_MAX, - .accessState = TSDB_VN_ALL_ACCCESS}; - - SSdbRaw *pRaw = mnodeAcctActionEncode(&acctObj); - if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - - mTrace("acct:%s, will be created while deploy sdb", acctObj.acct); - return sdbWrite(pMnode->pSdb, pRaw); +static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg) { + terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + mError("failed to process create acct msg since %s", terrstr()); + return -1; } -int32_t mndInitAcct(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_ACCT, - .keyType = SDB_KEY_BINARY, - .deployFp = mnodeCreateDefaultAcct, - .encodeFp = (SdbEncodeFp)mnodeAcctActionEncode, - .decodeFp = (SdbDecodeFp)mnodeAcctActionDecode, - .insertFp = (SdbInsertFp)mnodeAcctActionInsert, - .updateFp = (SdbUpdateFp)mnodeAcctActionUpdate, - .deleteFp = (SdbDeleteFp)mnodeAcctActionDelete}; - - return sdbSetTable(pMnode->pSdb, table); +static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg) { + terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + mError("failed to process create acct msg since %s", terrstr()); + return -1; } -void mndCleanupAcct(SMnode *pMnode) {} +static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg) { + terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + mError("failed to process create acct msg since %s", terrstr()); + return -1; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 1780c88d6b..6013ac0c31 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "mndCluster.h" #include "mndTrans.h" +#include "mndShow.h" #define SDB_CLUSTER_VER 1 @@ -94,6 +95,71 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { return sdbWrite(pMnode->pSdb, pRaw); } + +// static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { +// int32_t cols = 0; +// SSchema *pSchema = pMeta->schema; + +// pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE; +// pSchema[cols].type = TSDB_DATA_TYPE_BINARY; +// strcpy(pSchema[cols].name, "clusterId"); +// pSchema[cols].bytes = htons(pShow->bytes[cols]); +// cols++; + +// pShow->bytes[cols] = 8; +// pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; +// strcpy(pSchema[cols].name, "create_time"); +// pSchema[cols].bytes = htons(pShow->bytes[cols]); +// cols++; + +// pMeta->numOfColumns = htons(cols); +// strcpy(pMeta->tableFname, "show cluster"); +// pShow->numOfColumns = cols; + +// pShow->offset[0] = 0; +// for (int32_t i = 1; i < cols; ++i) { +// pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; +// } + +// pShow->numOfRows = 1; +// pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + +// return 0; +// } + +// static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn) { +// int32_t numOfRows = 0; +// int32_t cols = 0; +// char * pWrite; +// SClusterObj *pCluster = NULL; + +// while (numOfRows < rows) { +// pShow->pIter = mnodeGetNextCluster(pShow->pIter, &pCluster); +// if (pCluster == NULL) break; + +// cols = 0; + +// pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; +// STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pCluster->uid, TSDB_CLUSTER_ID_LEN); +// cols++; + +// pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; +// *(int64_t *) pWrite = pCluster->createdTime; +// cols++; + +// mnodeDecClusterRef(pCluster); +// numOfRows++; +// } + +// mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); +// pShow->numOfReads += numOfRows; +// return numOfRows; +// } + +// static void mnodeCancelGetNextCluster(void *pIter) { +// sdbFreeIter(tsClusterSdb, pIter); +// } + int32_t mndInitCluster(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_CLUSTER, .keyType = SDB_KEY_INT32, @@ -104,6 +170,9 @@ int32_t mndInitCluster(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndClusterActionUpdate, .deleteFp = (SdbDeleteFp)mndClusterActionDelete}; + // mndAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta); + // mndAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters); + // mndAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster); return sdbSetTable(pMnode->pSdb, table); } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 3a5a221677..f01a43dfef 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -14,8 +14,39 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndDb.h" -int32_t mndInitDb(SMnode *pMnode) { return 0; } -void mndCleanupDb(SMnode *pMnode) {} \ No newline at end of file +static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg); + +int32_t mndInitDb(SMnode *pMnode) { + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_USE_DB, mnodeProcessUseMsg); + return 0; +} + +void mndCleanupDb(SMnode *pMnode) {} + +SDbObj *mndAcquireDb(SMnode *pMnode, char *db) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_DB, db); +} + +void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pDb); +} + +static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SUseDbMsg *pUse = pMsg->rpcMsg.pCont; + + strncpy(pMsg->db, pUse->db, TSDB_FULL_DB_NAME_LEN); + + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + if (pDb != NULL) { + mndReleaseDb(pMnode, pDb); + return 0; + } else { + mError("db:%s, failed to process use db msg since %s", pMsg->db, terrstr()); + return -1; + } +} diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index a6cee3efc5..24e6113161 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -226,7 +226,8 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) { pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime); } -static int32_t mndProcessStatusMsg(SMnode *pMnode, SMnodeMsg *pMsg) { +static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; SStatusMsg *pStatus = pMsg->rpcMsg.pCont; mndParseStatusMsg(pStatus); @@ -315,11 +316,11 @@ static int32_t mndProcessStatusMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessCreateDnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDropDnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessConfigDnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { return 0; } int32_t mndInitDnode(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_DNODE, diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index d2ace31a36..e3464cd327 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -103,9 +103,9 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { return sdbWrite(pMnode->pSdb, pRaw); } -static int32_t mndProcessCreateMnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDropMnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) { return 0; } int32_t mndInitMnode(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_MNODE, @@ -135,4 +135,6 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { sdbRelease(pSdb, pMnodeObj); return true; -} \ No newline at end of file +} + +void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 845e50210a..2afb5f0665 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -14,8 +14,935 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndProfile.h" +#include "mndDb.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndUser.h" -int32_t mndInitProfile(SMnode *pMnode) { return 0; } -void mndCleanupProfile(SMnode *pMnode) {} \ No newline at end of file +#define QUERY_ID_SIZE 20 +#define QUERY_OBJ_ID_SIZE 18 +#define SUBQUERY_INFO_SIZE 6 +#define QUERY_STREAM_SAVE_SIZE 20 + +typedef struct { + char user[TSDB_USER_LEN]; + char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc + int32_t pid; // pid of app that invokes taosc + int32_t connId; + int8_t killed; + int8_t align; + uint16_t port; + uint32_t ip; + int64_t stime; + int64_t lastAccess; + int32_t queryId; + int32_t streamId; + int32_t numOfQueries; + int32_t numOfStreams; + SStreamDesc *pStreams; + SQueryDesc *pQueries; +} SConnObj; + +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app); +static void mndFreeConn(SConnObj *pConn); +static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId, char *user, uint32_t ip, uint16_t port); +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); +static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn); +static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); +static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg); +static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg); +static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg); +static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg); +static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg); +static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow); +static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); +static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow); +static int32_t mndRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, SMnodeMsg *pMsg); +static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); + +int32_t mndInitProfile(SMnode *pMnode) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + int32_t connCheckTime = pMnode->shellActivityTimer * 2; + pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); + if (pMgmt->cache == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("failed to alloc profile cache since %s", terrstr()); + return -1; + } + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_HEARTBEAT, mndProcessHeartBeatMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONNECT, mndProcessConnectMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_QUERY, mndProcessKillQueryMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_STREAM, mndProcessKillStreamMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_CONN, mndProcessKillConnectionMsg); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndGetConnsMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn); + + return 0; +} + +void mndCleanupProfile(SMnode *pMnode) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + if (pMgmt->cache != NULL) { + taosCacheCleanup(pMgmt->cache); + pMgmt->cache = NULL; + } +} + +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1); + if (connId == 0) atomic_add_fetch_32(&pMgmt->connId, 1); + + SConnObj connObj = {.pid = pid, + .connId = connId, + .killed = 0, + .port = port, + .ip = ip, + .stime = taosGetTimestampMs(), + .lastAccess = 0, + .queryId = 0, + .streamId = 0, + .numOfQueries = 0, + .numOfStreams = 0, + .pStreams = NULL, + .pQueries = NULL}; + + connObj.lastAccess = connObj.stime; + tstrncpy(connObj.user, user, TSDB_USER_LEN); + tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); + + int32_t keepTime = pMnode->shellActivityTimer * 3; + SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); + + mDebug("conn:%d, is created, user:%s", connId, user); + return pConn; +} + +static void mndFreeConn(SConnObj *pConn) { + tfree(pConn->pQueries); + tfree(pConn->pStreams); + mDebug("conn:%d, is destroyed", pConn->connId); +} + +static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId, char *newUser, uint32_t newIp, uint16_t newPort) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); + if (pConn == NULL) { + mDebug("conn:%d, already destroyed, user:%s", connId, newUser); + return NULL; + } + + if (pConn->ip != newIp || pConn->port != newPort /* || strcmp(pConn->user, newUser) != 0 */) { + char oldIpStr[30]; + char newIpStr[30]; + taosIp2String(pConn->ip, oldIpStr); + taosIp2String(newIp, newIpStr); + mDebug("conn:%d, incoming conn user:%s ip:%s:%u, not match exist user:%s ip:%s:%u", connId, newUser, newIpStr, + newPort, pConn->user, oldIpStr, pConn->port); + + if (pMgmt->connId < connId) pMgmt->connId = connId + 1; + taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + return NULL; + } + + int32_t keepTime = pMnode->shellActivityTimer * 3; + pConn->lastAccess = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); + return pConn; +} + +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + if (pConn == NULL) return; + taosCacheRelease(pMgmt->cache, (void **)&pConn, false); +} + +static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + *pConn = NULL; + + pIter = taosHashIterate(pMgmt->cache->pHashTable, pIter); + if (pIter == NULL) return NULL; + + SCacheDataNode **pNode = pIter; + if (pNode == NULL || *pNode == NULL) { + taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); + return NULL; + } + + *pConn = (SConnObj *)((*pNode)->data); + return pIter; +} + +static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); +} + +static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SConnectMsg *pReq = pMsg->rpcMsg.pCont; + pReq->pid = htonl(pReq->pid); + + SRpcConnInfo info = {0}; + if (rpcGetConnInfo(pMsg->rpcMsg.handle, &info) != 0) { + mError("user:%s, failed to login while get connection info since %s", pMsg->user, terrstr()); + return -1; + } + + char ip[30]; + taosIp2String(info.clientIp, ip); + + if (pReq->db[0]) { + snprintf(pMsg->db, TSDB_FULL_DB_NAME_LEN, "%d%s%s", pMsg->acctId, TS_PATH_DELIMITER, pReq->db); + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_INVALID_DB; + mError("user:%s, failed to login from %s while use db:%s since %s", pMsg->user, ip, pReq->db, terrstr()); + return -1; + } + mndReleaseDb(pMnode, pDb); + } + + SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + if (pConn == NULL) { + mError("user:%s, failed to login from %s while create connection since %s", pMsg->user, ip, terrstr()); + return -1; + } + + SConnectRsp *pRsp = rpcMallocCont(sizeof(SConnectRsp)); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("user:%s, failed to login from %s while create rsp since %s", pMsg->user, ip, terrstr()); + return -1; + } + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser != NULL) { + pRsp->acctId = htonl(pUser->acctId); + pRsp->superAuth = pUser->superAuth; + pRsp->readAuth = pUser->readAuth; + pRsp->writeAuth = pUser->writeAuth; + mndReleaseUser(pMnode, pUser); + } + + pRsp->clusterId = htonl(pMnode->clusterId); + pRsp->connId = htonl(pConn->connId); + mndGetMnodeEpSet(pMnode, &pRsp->epSet); + + pMsg->contLen = sizeof(SConnectRsp); + pMsg->pCont = pRsp; + mDebug("user:%s, login from %s, conn:%d", info.user, ip, pConn->connId); + return 0; +} + +static int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { + pConn->numOfQueries = 0; + pConn->numOfStreams = 0; + int32_t numOfQueries = htonl(pMsg->numOfQueries); + int32_t numOfStreams = htonl(pMsg->numOfStreams); + + if (numOfQueries > 0) { + if (pConn->pQueries == NULL) { + pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_STREAM_SAVE_SIZE); + } + + pConn->numOfQueries = MIN(QUERY_STREAM_SAVE_SIZE, numOfQueries); + + int32_t saveSize = pConn->numOfQueries * sizeof(SQueryDesc); + if (saveSize > 0 && pConn->pQueries != NULL) { + memcpy(pConn->pQueries, pMsg->pData, saveSize); + } + } + + if (numOfStreams > 0) { + if (pConn->pStreams == NULL) { + pConn->pStreams = calloc(sizeof(SStreamDesc), QUERY_STREAM_SAVE_SIZE); + } + + pConn->numOfStreams = MIN(QUERY_STREAM_SAVE_SIZE, numOfStreams); + + int32_t saveSize = pConn->numOfStreams * sizeof(SStreamDesc); + if (saveSize > 0 && pConn->pStreams != NULL) { + memcpy(pConn->pStreams, pMsg->pData + numOfQueries * sizeof(SQueryDesc), saveSize); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SHeartBeatMsg *pReq = pMsg->rpcMsg.pCont; + pReq->connId = htonl(pReq->connId); + pReq->pid = htonl(pReq->pid); + + SRpcConnInfo info = {0}; + if (rpcGetConnInfo(pMsg->rpcMsg.handle, &info) != 0) { + mError("user:%s, connId:%d failed to process hb since %s", pMsg->user, pReq->connId, terrstr()); + return -1; + } + + SConnObj *pConn = mndAcquireConn(pMnode, pReq->connId, info.user, info.clientIp, info.clientPort); + if (pConn == NULL) { + pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + if (pConn == NULL) { + mError("user:%s, conn:%d is freed and failed to create new conn since %s", pMsg->user, pReq->connId, terrstr()); + return -1; + } else { + mDebug("user:%s, conn:%d is freed and create a new conn:%d", pMsg->user, pReq->connId, pConn->connId); + } + } + + SHeartBeatRsp *pRsp = rpcMallocCont(sizeof(SHeartBeatRsp)); + if (pRsp == NULL) { + mndReleaseConn(pMnode, pConn); + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("user:%s, conn:%d failed to process hb while create rsp since %s", pMsg->user, pReq->connId, terrstr()); + return -1; + } + + mnodeSaveQueryStreamList(pConn, pReq); + if (pConn->killed != 0) { + pRsp->killConnection = 1; + } + + if (pConn->streamId != 0) { + pRsp->streamId = htonl(pConn->streamId); + pConn->streamId = 0; + } + + if (pConn->queryId != 0) { + pRsp->queryId = htonl(pConn->queryId); + pConn->queryId = 0; + } + + pRsp->connId = htonl(pConn->connId); + pRsp->totalDnodes = htonl(1); + pRsp->onlineDnodes = htonl(1); + mndGetMnodeEpSet(pMnode, &pRsp->epSet); + mndReleaseConn(pMnode, pConn); + + pMsg->contLen = sizeof(SConnectRsp); + pMsg->pCont = pRsp; + return 0; +} + +static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + mndReleaseUser(pMnode, pUser); + + SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; + mInfo("kill query msg is received, queryId:%s", pKill->queryId); + + const char delim = ':'; + char *connIdStr = strtok(pKill->queryId, &delim); + char *queryIdStr = strtok(NULL, &delim); + + if (queryIdStr == NULL || connIdStr == NULL) { + mError("failed to kill query, queryId:%s", pKill->queryId); + terrno = TSDB_CODE_MND_INVALID_QUERY_ID; + return -1; + } + + int32_t queryId = (int32_t)strtol(queryIdStr, NULL, 10); + + int32_t connId = atoi(connIdStr); + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); + if (pConn == NULL) { + mError("connId:%s, failed to kill queryId:%d, conn not exist", connIdStr, queryId); + terrno = TSDB_CODE_MND_INVALID_CONN_ID; + return -1; + } else { + mInfo("connId:%s, queryId:%d is killed by user:%s", connIdStr, queryId, pMsg->user); + pConn->queryId = queryId; + taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + return 0; + } +} + +static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + mndReleaseUser(pMnode, pUser); + + SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; + mInfo("kill stream msg is received, streamId:%s", pKill->queryId); + + const char delim = ':'; + char *connIdStr = strtok(pKill->queryId, &delim); + char *streamIdStr = strtok(NULL, &delim); + + if (streamIdStr == NULL || connIdStr == NULL) { + mError("failed to kill stream, streamId:%s", pKill->queryId); + terrno = TSDB_CODE_MND_INVALID_STREAM_ID; + return -1; + } + + int32_t streamId = (int32_t)strtol(streamIdStr, NULL, 10); + int32_t connId = atoi(connIdStr); + + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); + if (pConn == NULL) { + mError("connId:%s, failed to kill streamId:%d, conn not exist", connIdStr, streamId); + terrno = TSDB_CODE_MND_INVALID_CONN_ID; + return -1; + } else { + mInfo("connId:%s, streamId:%d is killed by user:%s", connIdStr, streamId, pMsg->user); + pConn->streamId = streamId; + taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + return TSDB_CODE_SUCCESS; + } +} + +static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + mndReleaseUser(pMnode, pUser); + + SKillConnMsg *pKill = pMsg->rpcMsg.pCont; + int32_t connId = atoi(pKill->queryId); + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); + if (pConn == NULL) { + mError("connId:%s, failed to kill, conn not exist", pKill->queryId); + terrno = TSDB_CODE_MND_INVALID_CONN_ID; + return -1; + } else { + mInfo("connId:%s, is killed by user:%s", pKill->queryId, pMsg->user); + pConn->killed = 1; + taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + return TSDB_CODE_SUCCESS; + } +} + +static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + mndReleaseUser(pMnode, pUser); + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "connId"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "user"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + // app name + pShow->bytes[cols] = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "program"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + // app pid + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "pid"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "ip:port"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "login_time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "last_access"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + + return 0; +} + +static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + int32_t numOfRows = 0; + SConnObj *pConnObj = NULL; + int32_t cols = 0; + char *pWrite; + char ipStr[TSDB_IPv4ADDR_LEN + 6]; + + while (numOfRows < rows) { + pShow->pIter = mndGetNextConn(pMnode, pShow->pIter, &pConnObj); + if (pConnObj == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pConnObj->connId; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]); + cols++; + + // app name + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->app, pShow->bytes[cols]); + cols++; + + // app pid + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pConnObj->pid; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + taosIpPort2String(pConnObj->ip, pConnObj->port, ipStr); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pConnObj->stime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + if (pConnObj->lastAccess < pConnObj->stime) pConnObj->lastAccess = pConnObj->stime; + *(int64_t *)pWrite = pConnObj->lastAccess; + cols++; + + numOfRows++; + } + + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + mndReleaseUser(pMnode, pUser); + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = QUERY_ID_SIZE + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "query_id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "user"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "ip:port"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 24; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "qid"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "created_time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; + strcpy(pSchema[cols].name, "time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "sql_obj_id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "pid"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "ep"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_BOOL; + strcpy(pSchema[cols].name, "stable_query"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "sub_queries"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "sub_query_info"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "sql"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = 1000000; + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + + return 0; +} + +static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + int32_t numOfRows = 0; + SConnObj *pConnObj = NULL; + int32_t cols = 0; + char *pWrite; + void *pIter; + char str[TSDB_IPv4ADDR_LEN + 6] = {0}; + + while (numOfRows < rows) { + pIter = mndGetNextConn(pMnode, pShow->pIter, &pConnObj); + if (pConnObj == NULL) { + pShow->pIter = pIter; + break; + } + + if (numOfRows + pConnObj->numOfQueries >= rows) { + mndCancelGetNextConn(pMnode, pIter); + break; + } + + pShow->pIter = pIter; + for (int32_t i = 0; i < pConnObj->numOfQueries; ++i) { + SQueryDesc *pDesc = pConnObj->pQueries + i; + cols = 0; + + snprintf(str, QUERY_ID_SIZE + 1, "%u:%u", pConnObj->connId, htonl(pDesc->queryId)); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + snprintf(str, tListLen(str), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]); + cols++; + + char handleBuf[24] = {0}; + snprintf(handleBuf, tListLen(handleBuf), "%" PRIu64, htobe64(pDesc->qId)); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, handleBuf, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = htobe64(pDesc->stime); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = htobe64(pDesc->useconds); + cols++; + + snprintf(str, tListLen(str), "0x%" PRIx64, htobe64(pDesc->sqlObjId)); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = htonl(pDesc->pid); + cols++; + + char epBuf[TSDB_EP_LEN + 1] = {0}; + snprintf(epBuf, tListLen(epBuf), "%s:%u", pDesc->fqdn, pConnObj->port); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, epBuf, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(bool *)pWrite = pDesc->stableQuery; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = htonl(pDesc->numOfSub); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->subSqlInfo, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->bytes[cols]); + cols++; + + numOfRows++; + } + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); +} + +static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + mndReleaseUser(pMnode, pUser); + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = QUERY_ID_SIZE + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "streamId"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "user"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "dest table"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "ip:port"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "created time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "exec time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; + strcpy(pSchema[cols].name, "time(us)"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "sql"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "cycles"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = 1000000; + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + + return 0; +} + +static int32_t mndRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + int32_t numOfRows = 0; + SConnObj *pConnObj = NULL; + int32_t cols = 0; + char *pWrite; + void *pIter; + char ipStr[TSDB_IPv4ADDR_LEN + 6]; + + while (numOfRows < rows) { + pIter = mndGetNextConn(pMnode, pShow->pIter, &pConnObj); + if (pConnObj == NULL) { + pShow->pIter = pIter; + break; + } + + if (numOfRows + pConnObj->numOfStreams >= rows) { + mndCancelGetNextConn(pMnode, pIter); + break; + } + + pShow->pIter = pIter; + for (int32_t i = 0; i < pConnObj->numOfStreams; ++i) { + SStreamDesc *pDesc = pConnObj->pStreams + i; + cols = 0; + + snprintf(ipStr, QUERY_ID_SIZE + 1, "%u:%u", pConnObj->connId, htonl(pDesc->streamId)); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->dstTable, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = htobe64(pDesc->ctime); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = htobe64(pDesc->stime); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = htobe64(pDesc->useconds); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = (int32_t)htobe64(pDesc->num); + cols++; + + numOfRows++; + } + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 9938e95a73..e652f94957 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -14,8 +14,322 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndShow.h" -int32_t mndInitShow(SMnode *pMnode) { return 0; } -void mndCleanupShow(SMnode *pMnode) {} \ No newline at end of file +static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg); +static int32_t mndProcessRetrieveMsg( SMnodeMsg *pMsg); +static bool mndCheckRetrieveFinished(SShowObj *pShow); +static int32_t mndAcquireShowObj(SMnode *pMnode, SShowObj *pShow); +static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); +static int32_t mndPutShowObj(SMnode *pMnode, SShowObj *pShow); +static void mndFreeShowObj(void *ppShow); +static char *mndShowStr(int32_t showType); + +int32_t mndInitShow(SMnode *pMnode) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + + pMgmt->cache = taosCacheInit(TSDB_CACHE_PTR_KEY, 5, true, mndFreeShowObj, "show"); + if (pMgmt->cache == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("failed to alloc show cache since %s", terrstr()); + return -1; + } + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SHOW, mndProcessShowMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SHOW_RETRIEVE, mndProcessRetrieveMsg); + return 0; +} + +void mndCleanupShow(SMnode *pMnode) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + if (pMgmt->cache != NULL) { + taosCacheCleanup(pMgmt->cache); + pMgmt->cache = NULL; + } +} + +static int32_t mndAcquireShowObj(SMnode *pMnode, SShowObj *pShow) { + TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pShow; + + SShowMgmt *pMgmt = &pMnode->showMgmt; + SShowObj **ppShow = taosCacheAcquireByKey(pMgmt->cache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE)); + if (ppShow) { + mTrace("show:%d, data:%p acquired from cache", pShow->id, ppShow); + return 0; + } + + return -1; +} + +static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { + SMnode *pMnode = pShow->pMnode; + SShowMgmt *pMgmt = &pMnode->showMgmt; + SShowObj **ppShow = (SShowObj **)pShow->ppShow; + taosCacheRelease(pMgmt->cache, (void **)(&ppShow), forceRemove); + mDebug("show:%d, data:%p released from cache, force:%d", pShow->id, ppShow, forceRemove); +} + +static int32_t mndPutShowObj(SMnode *pMnode, SShowObj *pShow) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + int32_t lifeSpan = pMnode->shellActivityTimer * 6 * 1000; + + TSDB_CACHE_PTR_TYPE val = (TSDB_CACHE_PTR_TYPE)pShow; + pShow->id = atomic_add_fetch_32(&pMgmt->showId, 1); + SShowObj **ppShow = + taosCachePut(pMgmt->cache, &val, sizeof(TSDB_CACHE_PTR_TYPE), &pShow, sizeof(TSDB_CACHE_PTR_TYPE), lifeSpan); + if (ppShow == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("show:%d, failed to put into cache", pShow->id); + return -1; + } + + mTrace("show:%d, data:%p put into cache", pShow->id, ppShow); + return 0; +} + +static void mndFreeShowObj(void *ppShow) { + SShowObj *pShow = *(SShowObj **)ppShow; + SMnode *pMnode = pShow->pMnode; + SShowMgmt *pMgmt = &pMnode->showMgmt; + + ShowFreeIterFp freeFp = pMgmt->freeIterFps[pShow->type]; + if (freeFp != NULL) { + if (pShow->pVgIter != NULL) { + // only used in 'show vnodes "ep"' + (*freeFp)(pMnode, pShow->pVgIter); + } + if (pShow->pIter != NULL) { + (*freeFp)(pMnode, pShow->pIter); + } + } + + mDebug("show:%d, data:%p destroyed", pShow->id, ppShow); + tfree(pShow); +} + +static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { + SMnode *pMnode = pMnodeMsg->pMnode; + SShowMgmt *pMgmt = &pMnode->showMgmt; + SShowMsg *pMsg = pMnodeMsg->rpcMsg.pCont; + int8_t type = pMsg->type; + uint16_t payloadLen = htonl(pMsg->payloadLen); + + if (type <= TSDB_MGMT_TABLE_START || type >= TSDB_MGMT_TABLE_MAX) { + terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; + mError("failed to process show msg since %s", terrstr()); + return -1; + } + + ShowMetaFp metaFp = pMgmt->metaFps[type]; + if (metaFp == NULL) { + terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; + mError("failed to process show-meta msg:%s since no message handle", mndShowStr(type)); + return -1; + } + + int32_t size = sizeof(SShowObj) + payloadLen; + SShowObj *pShow = calloc(1, size); + if (pShow != NULL) { + pShow->pMnode = pMnode; + pShow->type = type; + pShow->payloadLen = payloadLen; + memcpy(pShow->db, pMsg->db, TSDB_FULL_DB_NAME_LEN); + memcpy(pShow->payload, pMsg->payload, payloadLen); + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr()); + return -1; + } + + if (mndPutShowObj(pMnode, pShow) == 0) { + mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr()); + free(pShow); + return -1; + } + + size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; + SShowRsp *pRsp = rpcMallocCont(size); + if (pRsp == NULL) { + mndReleaseShowObj(pShow, true); + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("show:%d, failed to process show-meta msg:%s since malloc rsp error", pShow->id, mndShowStr(type)); + return -1; + } + + pRsp->qhandle = htobe64((uint64_t)pShow); + + int32_t code = (*metaFp)(pMnodeMsg,pShow, &pRsp->tableMeta); + mDebug("show:%d, type:%s, get meta finished, numOfRows:%d cols:%d result:%s", pShow->id, mndShowStr(type), + pShow->numOfRows, pShow->numOfColumns, tstrerror(code)); + + if (code == TSDB_CODE_SUCCESS) { + pMnodeMsg->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; + pMnodeMsg->pCont = pRsp; + mndReleaseShowObj(pShow, false); + return TSDB_CODE_SUCCESS; + } else { + rpcFreeCont(pRsp); + mndReleaseShowObj(pShow, true); + return code; + } +} + +static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { + SMnode *pMnode = pMnodeMsg->pMnode; + SShowMgmt *pMgmt = &pMnode->showMgmt; + int32_t rowsToRead = 0; + int32_t size = 0; + int32_t rowsRead = 0; + + SRetrieveTableMsg *pRetrieve = pMnodeMsg->rpcMsg.pCont; + pRetrieve->qhandle = htobe64(pRetrieve->qhandle); + SShowObj *pShow = (SShowObj *)pRetrieve->qhandle; + + /* + * in case of server restart, apps may hold qhandle created by server before + * restart, which is actually invalid, therefore, signature check is required. + */ + if (mndAcquireShowObj(pMnode, pShow) != 0) { + terrno = TSDB_CODE_MND_INVALID_SHOWOBJ; + mError("failed to process show-retrieve msg:%p since %s", pShow, terrstr()); + return -1; + } + + ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; + if (retrieveFp == NULL) { + mndReleaseShowObj(pShow, false); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr()); + return -1; + } + + mDebug("show:%d, type:%s, start retrieve data, numOfReads:%d numOfRows:%d", pShow->id, mndShowStr(pShow->type), + pShow->numOfReads, pShow->numOfRows); + + if (mndCheckRetrieveFinished(pShow)) { + mDebug("show:%d, read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads, pShow->numOfRows); + pShow->numOfReads = pShow->numOfRows; + } + + if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { + rowsToRead = pShow->numOfRows - pShow->numOfReads; + } + + /* return no more than 100 tables in one round trip */ + if (rowsToRead > 100) rowsToRead = 100; + + /* + * the actual number of table may be larger than the value of pShow->numOfRows, if a query is + * issued during a continuous create table operation. Therefore, rowToRead may be less than 0. + */ + if (rowsToRead < 0) rowsToRead = 0; + size = pShow->rowSize * rowsToRead; + + size += 100; + SRetrieveTableRsp *pRsp = rpcMallocCont(size); + if (pRsp == NULL) { + mndReleaseShowObj(pShow, false); + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr()); + return -1; + } + + // if free flag is set, client wants to clean the resources + if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { + rowsRead = (*retrieveFp)(pMnodeMsg, pShow, pRsp->data, rowsToRead); + } + + mDebug("show:%d, stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); + + pRsp->numOfRows = htonl(rowsRead); + pRsp->precision = (int16_t)htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision + + pMnodeMsg->pCont = pRsp; + pMnodeMsg->contLen = size; + + if (rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { + pRsp->completed = 1; + mDebug("%p, retrieve completed", pShow); + mndReleaseShowObj(pShow, true); + } else { + mDebug("%p, retrieve not completed yet", pShow); + mndReleaseShowObj(pShow, false); + } + + return TSDB_CODE_SUCCESS; +} + +static char *mndShowStr(int32_t showType) { + switch (showType) { + case TSDB_MGMT_TABLE_ACCT: + return "show accounts"; + case TSDB_MGMT_TABLE_USER: + return "show users"; + case TSDB_MGMT_TABLE_DB: + return "show databases"; + case TSDB_MGMT_TABLE_TABLE: + return "show tables"; + case TSDB_MGMT_TABLE_DNODE: + return "show dnodes"; + case TSDB_MGMT_TABLE_MNODE: + return "show mnodes"; + case TSDB_MGMT_TABLE_VGROUP: + return "show vgroups"; + case TSDB_MGMT_TABLE_METRIC: + return "show stables"; + case TSDB_MGMT_TABLE_MODULE: + return "show modules"; + case TSDB_MGMT_TABLE_QUERIES: + return "show queries"; + case TSDB_MGMT_TABLE_STREAMS: + return "show streams"; + case TSDB_MGMT_TABLE_VARIABLES: + return "show configs"; + case TSDB_MGMT_TABLE_CONNS: + return "show connections"; + case TSDB_MGMT_TABLE_SCORES: + return "show scores"; + case TSDB_MGMT_TABLE_GRANTS: + return "show grants"; + case TSDB_MGMT_TABLE_VNODES: + return "show vnodes"; + case TSDB_MGMT_TABLE_CLUSTER: + return "show clusters"; + case TSDB_MGMT_TABLE_STREAMTABLES: + return "show streamtables"; + case TSDB_MGMT_TABLE_TP: + return "show topics"; + default: + return "undefined"; + } +} + +static bool mndCheckRetrieveFinished(SShowObj *pShow) { + if (pShow->pIter == NULL && pShow->numOfReads != 0) { + return true; + } + return false; +} + +void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { + if (rows < capacity) { + for (int32_t i = 0; i < numOfCols; ++i) { + memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows); + } + } +} + +void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + pMgmt->metaFps[showType] = fp; +} + +void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + pMgmt->retrieveFps[showType] = fp; +} + +void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType showType, ShowFreeIterFp fp) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + pMgmt->freeIterFps[showType] = fp; +} diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index abbe41a60d..f59ef8f8e3 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -14,12 +14,76 @@ */ #define _DEFAULT_SOURCE +#include "mndUser.h" +#include "mndShow.h" #include "mndSync.h" #include "mndTrans.h" #include "tkey.h" #define SDB_USER_VER 1 +static int32_t mndCreateDefaultUsers(SMnode *pMnode); +static SSdbRaw *mndUserActionEncode(SUserObj *pUser); +static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); +static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); +static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); +static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDstUser); +static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pMsg); +static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg); +static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg); + +int32_t mndInitUser(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_USER, + .keyType = SDB_KEY_BINARY, + .deployFp = (SdbDeployFp)mndCreateDefaultUsers, + .encodeFp = (SdbEncodeFp)mndUserActionEncode, + .decodeFp = (SdbDecodeFp)mndUserActionDecode, + .insertFp = (SdbInsertFp)mndUserActionInsert, + .updateFp = (SdbUpdateFp)mndUserActionUpdate, + .deleteFp = (SdbDeleteFp)mndUserActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_USER, mndProcessCreateUserMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_USER, mndProcessAlterUserMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_USER, mndProcessDropUserMsg); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupUser(SMnode *pMnode) {} + +static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) { + SUserObj userObj = {0}; + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + + if (strcmp(user, TSDB_DEFAULT_USER) == 0) { + userObj.superAuth = 1; + } + + SSdbRaw *pRaw = mndUserActionEncode(&userObj); + if (pRaw == NULL) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + + mTrace("user:%s, will be created while deploy sdb", userObj.user); + return sdbWrite(pMnode->pSdb, pRaw); +} + +static int32_t mndCreateDefaultUsers(SMnode *pMnode) { + if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { + return -1; + } + + if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, "_" TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { + return -1; + } + + return 0; +} + static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, SDB_USER_VER, sizeof(SUserObj)); if (pRaw == NULL) return NULL; @@ -30,7 +94,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN) SDB_SET_INT64(pRaw, dataPos, pUser->createdTime) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime) - SDB_SET_INT8(pRaw, dataPos, pUser->rootAuth) + SDB_SET_INT8(pRaw, dataPos, pUser->superAuth) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -56,7 +120,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->acct, TSDB_USER_LEN) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->rootAuth) + SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superAuth) return pRow; } @@ -70,12 +134,14 @@ static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser) { return -1; } - pUser->pAcct = sdbAcquire(pSdb, SDB_ACCT, pUser->acct); - if (pUser->pAcct == NULL) { + SAcctObj *pAcct = sdbAcquire(pSdb, SDB_ACCT, pUser->acct); + if (pAcct == NULL) { terrno = TSDB_CODE_MND_ACCT_NOT_EXIST; mError("user:%s, failed to perform insert action since %s", pUser->user, terrstr()); return -1; } + pUser->acctId = pAcct->acctId; + sdbRelease(pSdb, pAcct); return 0; } @@ -87,11 +153,6 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { pUser->prohibitDbHash = NULL; } - if (pUser->pAcct != NULL) { - sdbRelease(pSdb, pUser->pAcct); - pUser->pAcct = NULL; - } - return 0; } @@ -102,40 +163,18 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDs memcpy(pSrcUser->acct, pDstUser->acct, TSDB_USER_LEN); pSrcUser->createdTime = pDstUser->createdTime; pSrcUser->updateTime = pDstUser->updateTime; - pSrcUser->rootAuth = pDstUser->rootAuth; + pSrcUser->superAuth = pDstUser->superAuth; return 0; } -static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) { - SUserObj userObj = {0}; - tstrncpy(userObj.user, user, TSDB_USER_LEN); - tstrncpy(userObj.acct, acct, TSDB_USER_LEN); - taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); - userObj.createdTime = taosGetTimestampMs(); - userObj.updateTime = userObj.createdTime; - - if (strcmp(user, TSDB_DEFAULT_USER) == 0) { - userObj.rootAuth = 1; - } - - SSdbRaw *pRaw = mndUserActionEncode(&userObj); - if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - - mTrace("user:%s, will be created while deploy sdb", userObj.user); - return sdbWrite(pMnode->pSdb, pRaw); +SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_USER, userName); } -static int32_t mndCreateDefaultUsers(SMnode *pMnode) { - if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { - return -1; - } - - if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, "_" TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { - return -1; - } - - return 0; +void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pUser); } static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pMsg) { @@ -145,7 +184,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.rootAuth = 0; + userObj.superAuth = 0; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) return -1; @@ -183,7 +222,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, return 0; } -static int32_t mndProcessCreateUserMsg(SMnode *pMnode, SMnodeMsg *pMsg) { +static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; if (pCreate->user[0] == 0) { @@ -224,30 +264,14 @@ static int32_t mndProcessCreateUserMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -int32_t mndInitUser(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_USER, - .keyType = SDB_KEY_BINARY, - .deployFp = (SdbDeployFp)mndCreateDefaultUsers, - .encodeFp = (SdbEncodeFp)mndUserActionEncode, - .decodeFp = (SdbDecodeFp)mndUserActionDecode, - .insertFp = (SdbInsertFp)mndUserActionInsert, - .updateFp = (SdbUpdateFp)mndUserActionUpdate, - .deleteFp = (SdbDeleteFp)mndUserActionDelete}; - - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_USER, mndProcessCreateUserMsg); - - return sdbSetTable(pMnode->pSdb, table); -} - -void mndCleanupUser(SMnode *pMnode) {} - -SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_USER, &userName); -} - -void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { - SSdb *pSdb = pMnode->pSdb; - sdbRelease(pSdb, pUser); +static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { + terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + mError("failed to process alter user msg since %s", terrstr()); + return -1; } +static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg) { + terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + mError("failed to process drop user msg since %s", terrstr()); + return -1; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index e390deda6d..b99de4a019 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -206,6 +206,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->sver = pOption->sver; pMnode->statusInterval = pOption->statusInterval; pMnode->mnodeEqualVnodeNum = pOption->mnodeEqualVnodeNum; + pMnode->shellActivityTimer = pOption->shellActivityTimer; pMnode->timezone = strdup(pOption->timezone); pMnode->locale = strdup(pOption->locale); pMnode->charset = strdup(pOption->charset); @@ -386,7 +387,7 @@ static void mndProcessRpcMsg(SMnodeMsg *pMsg) { goto PROCESS_RPC_END; } - code = (*fp)(pMnode, pMsg); + code = (*fp)(pMsg); if (code != 0) { code = terrno; mError("msg:%p, app:%p failed to process since %s", pMsg, ahandle, terrstr()); diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index bcb7070755..eb288e0aa2 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -16,6 +16,9 @@ #ifndef __INDEX_FST_H__ #define __INDEX_FST_H__ +#ifdef __cplusplus +extern "C" { +#endif #include "tarray.h" #include "index_fst_util.h" @@ -34,6 +37,7 @@ typedef struct FstRange { } FstRange; +typedef enum {GE, GT, LE, LT} RangeType; typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal} State; typedef enum {Ordered, OutOfOrdered, DuplicateKey} OrderType; @@ -85,16 +89,19 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, typedef struct FstBuilder { FstCountingWriter *wrt; // The FST raw data is written directly to `wtr`. FstUnFinishedNodes *unfinished; // The stack of unfinished nodes - FstRegistry* registry; // A map of finished nodes. - FstSlice last; // The last word added + FstRegistry* registry; // A map of finished nodes. + FstSlice last; // The last word added CompiledAddr lastAddr; // The address of the last compiled node uint64_t len; // num of keys added } FstBuilder; FstBuilder *fstBuilderCreate(void *w, FstType ty); + + void fstBuilderDestroy(FstBuilder *b); void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in); +bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in); OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup); void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate); CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn); @@ -169,11 +176,6 @@ uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null - - - - - #define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) #define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) #define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) @@ -272,18 +274,13 @@ FstNode* fstGetNode(Fst *fst, CompiledAddr); FstNode* fstGetRoot(Fst *fst); FstType fstGetType(Fst *fst); CompiledAddr fstGetRootAddr(Fst *fst); - -Output fstEmptyFinalOutput(Fst *fst, bool *null); -bool fstVerify(Fst *fst); +Output fstEmptyFinalOutput(Fst *fst, bool *null); +bool fstVerify(Fst *fst); //refactor this function bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); - - - - typedef struct StreamState { FstNode *node; uint64_t trans; @@ -310,10 +307,30 @@ typedef struct StreamWithStateResult { } StreamWithStateResult; StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state); +void swsResultDestroy(StreamWithStateResult *result); typedef void* (*StreamCallback)(void *); StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoundWithData *min, FstBoundWithData *max) ; void streamWithStateDestroy(StreamWithState *sws); bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); + +typedef struct FstStreamBuilder { + Fst *fst; + Automation *aut; + FstBoundWithData *min; + FstBoundWithData *max; +} FstStreamBuilder; + +FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut); +// set up bound range +// refator, simple code by marco + +FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type); + + +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index c9df9c219e..480d3110c4 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -15,6 +15,10 @@ #ifndef __INDEX_FST_AUTAOMATION_H__ #define __INDEX_FST_AUTAOMATION_H__ +#ifdef __cplusplus +extern "C" { +#endif + typedef struct AutomationCtx AutomationCtx; typedef struct StartWith { @@ -42,6 +46,8 @@ typedef struct Automation { void *data; } Automation; - +#ifdef __cplusplus +} +#endif #endif diff --git a/source/libs/index/inc/index_fst_common.h b/source/libs/index/inc/index_fst_common.h index b261f4090c..9c802faa33 100644 --- a/source/libs/index/inc/index_fst_common.h +++ b/source/libs/index/inc/index_fst_common.h @@ -1,7 +1,16 @@ #ifndef __INDEX_FST_COMM_H__ #define __INDEX_FST_COMM_H__ + extern const uint8_t COMMON_INPUTS[]; extern char const COMMON_INPUTS_INV[]; +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index 4650804034..9280461780 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -16,6 +16,38 @@ #ifndef __INDEX_FST_COUNTING_WRITER_H__ #define __INDEX_FST_COUNTING_WRITER_H__ +#ifdef __cplusplus +extern "C" { +#endif + +#include "tfile.h" + + +#define DefaultMem 1024*1024 + +static char tmpFile[] = "/tmp/index"; +typedef enum WriterType {TMemory, TFile} WriterType; + +typedef struct WriterCtx { + int (*write)(struct WriterCtx *ctx, uint8_t *buf, int len); + int (*read)(struct WriterCtx *ctx, uint8_t *buf, int len); + int (*flush)(struct WriterCtx *ctx); + WriterType type; + union { + int fd; + void *mem; + }; + int32_t offset; + int32_t limit; +} WriterCtx; + +static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len); +static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); +static int writeCtxDoFlush(WriterCtx *ctx); + +WriterCtx* writerCtxCreate(WriterType type); +void writerCtxDestroy(WriterCtx *w); + typedef uint32_t CheckSummer; @@ -25,7 +57,7 @@ typedef struct FstCountingWriter { CheckSummer summer; } FstCountingWriter; -uint64_t fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen); +int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen); int fstCountingWriterFlush(FstCountingWriter *write); @@ -44,6 +76,10 @@ uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n); #define FST_WRITER_INTER_WRITER(writer) (writer->wtr) #define FST_WRITE_CHECK_SUMMER(writer) (writer->summer) +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/index/inc/index_fst_node.h b/source/libs/index/inc/index_fst_node.h index 0645aa1158..87eb7cb746 100644 --- a/source/libs/index/inc/index_fst_node.h +++ b/source/libs/index/inc/index_fst_node.h @@ -16,6 +16,10 @@ #ifndef __INDEX_FST_NODE_H__ #define __INDEX_FST_NODE_H__ +#ifdef __cplusplus +extern "C" { +#endif + #include "index_fst_util.h" #include "index_fst_counting_writer.h" @@ -42,7 +46,12 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src); void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src); //bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); +bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2); void fstBuilderNodeDestroy(FstBuilderNode *node); +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/index/inc/index_fst_registry.h b/source/libs/index/inc/index_fst_registry.h index 2504d7ccff..1d89e57e52 100644 --- a/source/libs/index/inc/index_fst_registry.h +++ b/source/libs/index/inc/index_fst_registry.h @@ -15,6 +15,10 @@ #ifndef __FST_REGISTRY_H__ #define __FST_REGISTRY_H__ +#ifdef __cplusplus +extern "C" { +#endif + #include "index_fst_util.h" #include "tarray.h" #include "index_fst_node.h" @@ -59,4 +63,8 @@ void fstRegistryDestroy(FstRegistry *registry); FstRegistryEntry* fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode); void fstRegistryEntryDestroy(FstRegistryEntry *entry); +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/index/inc/index_fst_util.h b/source/libs/index/inc/index_fst_util.h index ff0946063d..4af885816f 100644 --- a/source/libs/index/inc/index_fst_util.h +++ b/source/libs/index/inc/index_fst_util.h @@ -17,6 +17,10 @@ #ifndef __INDEX_FST_UTIL_H__ #define __INDEX_FST_UTIL_H__ +#ifdef __cplusplus +extern "C" { +#endif + #include "tarray.h" #include "index_fst_common.h" @@ -67,20 +71,30 @@ uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr); CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr); +typedef struct FstString { + uint8_t *data; + uint32_t len; + int32_t ref; +} FstString; typedef struct FstSlice { - uint8_t *data; - uint64_t dLen; - int32_t start; - int32_t end; + FstString *str; + int32_t start; + int32_t end; } FstSlice; -FstSlice fstSliceCopy(FstSlice *slice, int32_t start, int32_t end); -FstSlice fstSliceCreate(uint8_t *data, uint64_t dLen); -bool fstSliceEmpty(FstSlice *slice); -int fstSliceCompare(FstSlice *a, FstSlice *b); +FstSlice fstSliceCreate(uint8_t *data, uint64_t len); +FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end); +FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end); +bool fstSliceIsEmpty(FstSlice *s); +int fstSliceCompare(FstSlice *s1, FstSlice *s2); +void fstSliceDestroy(FstSlice *s); +uint8_t *fstSliceData(FstSlice *s, int32_t *sz); -#define FST_SLICE_LEN(s) ((s)->end - (s)->start + 1) +#define FST_SLICE_LEN(s) (s->end - s->start + 1) +#ifdef __cplusplus +} +#endif #endif diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 315253d907..418d1f2bda 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -52,7 +52,7 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) { FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); node->isFinal = isFinal; node->finalOutput = 0; - node->trans = NULL; + node->trans = taosArrayInit(16, sizeof(FstTransition)); FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; taosArrayPush(nodes->stack, &un); @@ -62,7 +62,7 @@ FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes) { assert(taosArrayGetSize(nodes->stack) == 1); FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); - assert(un->last == NULL); + //assert(un->last == NULL); return un->node; } @@ -70,6 +70,7 @@ FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledA FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); fstBuilderNodeUnfinishedLastCompiled(un, addr); free(un->last); // TODO add func FstLastTransitionFree() + un->last = NULL; return un->node; } @@ -92,30 +93,30 @@ void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *nodes, CompiledAddr add } void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output out) { FstSlice *s = &bs; - if (s->data == NULL || s->dLen == 0 || s->start > s->end) { + if (fstSliceIsEmpty(s)) { return; } size_t sz = taosArrayGetSize(nodes->stack) - 1; FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz); assert(un->last == NULL); - - //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); //trn->inp = s->data[s->start]; //trn->out = out; - un->last = fstLastTransitionCreate(s->data[s->start], out); + int32_t len = 0; + uint8_t *data = fstSliceData(s, &len); + un->last = fstLastTransitionCreate(data[0], out); - for (uint64_t i = s->start; i <= s->end; i++) { + for (uint64_t i = 1; i < len; i++) { FstBuilderNode *n = malloc(sizeof(FstBuilderNode)); n->isFinal = false; n->finalOutput = 0; - n->trans = NULL; + n->trans = taosArrayInit(16, sizeof(FstTransition)); //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); //trn->inp = s->data[i]; //trn->out = out; - FstLastTransition *trn = fstLastTransitionCreate(s->data[i], out); + FstLastTransition *trn = fstLastTransitionCreate(data[i], 0); FstBuilderNodeUnfinished un = {.node = n, .last = trn}; taosArrayPush(nodes->stack, &un); @@ -127,13 +128,13 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) { FstSlice *s = &bs; - size_t lsz = (size_t)(s->end - s->start + 1); // data len size_t ssz = taosArrayGetSize(node->stack); // stack size - uint64_t count = 0; + int32_t lsz; // data len + uint8_t *data = fstSliceData(s, &lsz); for (size_t i = 0; i < ssz && i < lsz; i++) { FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); - if (un->last->inp == s->data[s->start + i]) { + if (un->last->inp == data[i]) { count++; } else { break; @@ -153,7 +154,8 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstLastTransition *t = un->last; uint64_t addPrefix = 0; - if (t && t->inp == s->data[s->start + i]) { + uint8_t *data = fstSliceData(s, NULL); + if (t && t->inp == data[i]) { uint64_t commPrefix = MIN(t->out, *out); uint64_t tAddPrefix = t->out - commPrefix; (*out) = (*out) - commPrefix; @@ -164,7 +166,6 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, } if (addPrefix != 0) { fstBuilderNodeUnfinishedAddOutputPrefix(un, addPrefix); - } } return i; @@ -176,7 +177,9 @@ FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { if (addr == EMPTY_ADDRESS) { return fs; } - uint8_t v = slice->data[addr]; + + uint8_t *data = fstSliceData(slice, NULL); + uint8_t v = data[addr]; uint8_t t = (v & 0b11000000) >> 6; if (t == 0b11) { fs.state = OneTransNext; @@ -376,7 +379,8 @@ uint8_t fstStateInput(FstState *s, FstNode *node) { FstSlice *slice = &node->data; bool null = false; uint8_t inp = fstStateCommInput(s, &null); - return null == false ? inp : slice->data[slice->start - 1]; + uint8_t *data = fstSliceData(slice, NULL); + return null == false ? inp : data[-1]; } uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { assert(s->state == AnyTrans); @@ -388,7 +392,9 @@ uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size - return slice->data[at]; + + uint8_t *data = fstSliceData(slice, NULL); + return data[at]; } // trans_addr @@ -406,7 +412,8 @@ CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) { - tSizes; // refactor error logic - return unpackDelta(slice->data + slice->start + i, tSizes, node->end); + uint8_t *data = fstSliceData(slice, NULL); + return unpackDelta(data +i, tSizes, node->end); } } CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i) { @@ -421,7 +428,8 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i - node->nTrans - (i * tSizes) - tSizes; - return unpackDelta(slice->data + slice->start + at, tSizes, node->end); + uint8_t *data = fstSliceData(slice, NULL); + return unpackDelta(data + at, tSizes, node->end); } // sizes @@ -434,7 +442,8 @@ PackSizes fstStateSizes(FstState *s, FstSlice *slice) { i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1; } - return (PackSizes)(slice->data[slice->start + i]); + uint8_t *data = fstSliceData(slice, NULL); + return (PackSizes)(*(data +i)); } // Output Output fstStateOutput(FstState *s, FstNode *node) { @@ -452,7 +461,8 @@ Output fstStateOutput(FstState *s, FstNode *node) { - 1 - tSizes - oSizes; - return unpackUint64(slice->data + slice->start + i, oSizes); + uint8_t *data = fstSliceData(slice, NULL); + return unpackUint64(data + i, oSizes); } Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { @@ -469,7 +479,9 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes; - return unpackUint64(slice->data + slice->start + at, oSizes); + + uint8_t *data = fstSliceData(slice, NULL); + return unpackUint64(data + at, oSizes); } // anyTrans specify function @@ -523,7 +535,10 @@ uint64_t fstStateNtrans(FstState *s, FstSlice *slice) { if (null != true) { return n; } - n = slice->data[slice->end - 1]; // data[data.len() - 2] + int32_t len; + uint8_t *data = fstSliceData(slice, &len); + n = data[len - 2]; + //n = data[slice->end - 1]; // data[data.len() - 2] return n == 1 ? 256: n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, then it is encoded in the state byte } Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, PackSizes sizes, uint64_t nTrans) { @@ -538,7 +553,8 @@ Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, Pack - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; - return unpackUint64(slice->data + slice->start + at, (uint8_t)oSizes); + uint8_t *data = fstSliceData(slice, NULL); + return unpackUint64(data + at, (uint8_t)oSizes); } uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { @@ -549,7 +565,10 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { - fstStateNtransLen(s) - 1 // pack size - fstStateTransIndexSize(s, node->version, node->nTrans); - uint64_t i = slice->data[slice->start + at + b]; + int32_t dlen = 0; + uint8_t *data = fstSliceData(slice, &dlen); + uint64_t i = data[at + b]; + //uint64_t i = slice->data[slice->start + at + b]; if (i >= node->nTrans) { *null = true; } @@ -561,8 +580,13 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { - node->nTrans; uint64_t end = start + node->nTrans; uint64_t len = end - start; + int32_t dlen = 0; + uint8_t *data = fstSliceData(slice, &dlen); for(int i = 0; i < len; i++) { - uint8_t v = slice->data[slice->start + i]; + //uint8_t v = slice->data[slice->start + i]; + ////slice->data[slice->start + i]; + uint8_t v = data[i]; + if (v == b) { return node->nTrans - i - 1; // bug } @@ -635,6 +659,7 @@ static const char *fstNodeState(FstNode *node) { void fstNodeDestroy(FstNode *node) { + fstSliceDestroy(&node->data); free(node); } FstTransitions* fstNodeTransitions(FstNode *node) { @@ -774,18 +799,18 @@ bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in) { void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { FstSlice *s = &bs; - if (fstSliceEmpty(s)) { + if (fstSliceIsEmpty(s)) { b->len = 1; fstUnFinishedNodesSetRootOutput(b->unfinished, in); return; } - Output out; //if (in != 0) { //if let Some(in) = in // prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); //} else { // prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs); // out = 0; //} + Output out; uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); if (prefixLen == FST_SLICE_LEN(s)) { @@ -798,12 +823,13 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { FstSlice sub = fstSliceCopy(s, prefixLen, s->end); fstUnFinishedNodesAddSuffix(b->unfinished, sub, out); + fstSliceDestroy(&sub); return; } OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { FstSlice *input = &bs; - if (fstSliceEmpty(&b->last)) { + if (fstSliceIsEmpty(&b->last)) { // deep copy or not b->last = fstSliceCopy(&bs, input->start, input->end); } else { @@ -827,9 +853,9 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { } else { n = fstUnFinishedNodesPopFreeze(b->unfinished, addr); } - addr = fstBuilderCompile(b, n); + addr = fstBuilderCompile(b, n); assert(addr != NONE_ADDRESS); - fstBuilderNodeDestroy(n); + //fstBuilderNodeDestroy(n); } fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); return; @@ -863,22 +889,26 @@ void* fstBuilderInsertInner(FstBuilder *b) { FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished); CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); - uint8_t buf64[8] = {0}; + char buf64[8] = {0}; - taosEncodeFixedU64((void **)&buf64, b->len); + void *pBuf64 = buf64; + taosEncodeFixedU64(&pBuf64, b->len); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - taosEncodeFixedU64((void **)&buf64, rootAddr); + pBuf64 = buf64; + taosEncodeFixedU64(&pBuf64, rootAddr); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - uint8_t buf32[4] = {0}; + char buf32[4] = {0}; + void *pBuf32 = buf32; uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt); - taosEncodeFixedU32((void **)&buf32, sum); + taosEncodeFixedU32(&pBuf32, sum); fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); fstCountingWriterFlush(b->wrt); + //fstCountingWriterDestroy(b->wrt); + //b->wrt = NULL; return b->wrt; - } void fstBuilderFinish(FstBuilder *b) { fstBuilderInsertInner(b); @@ -888,7 +918,7 @@ void fstBuilderFinish(FstBuilder *b) { FstSlice fstNodeAsSlice(FstNode *node) { FstSlice *slice = &node->data; - FstSlice s = fstSliceCopy(slice, slice->end, slice->dLen - 1); + FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); return s; } @@ -929,12 +959,13 @@ void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *unNode, O } Fst* fstCreate(FstSlice *slice) { - char *buf = slice->data; - uint64_t skip = 0; - uint64_t len = slice->dLen; - if (len < 36) { + int32_t slen; + char *buf = fstSliceData(slice, &slen); + if (slen < 36) { return NULL; } + uint64_t len = slen; + uint64_t skip = 0; uint64_t version; taosDecodeFixedU64(buf, &version); @@ -992,8 +1023,10 @@ void fstDestroy(Fst *fst) { bool fstGet(Fst *fst, FstSlice *b, Output *out) { FstNode *root = fstGetRoot(fst); Output tOut = 0; - for (uint32_t i = 0; i < b->dLen; i++) { - uint8_t inp = b->data[i]; + int32_t len; + uint8_t *data = fstSliceData(b, &len); + for (uint32_t i = 0; i < len; i++) { + uint8_t inp = data[i]; Output res = 0; bool null = fstNodeFindInput(root, inp, &res); if (null) { return false; } @@ -1046,9 +1079,10 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null) { bool fstVerify(Fst *fst) { uint32_t checkSum = fst->meta->checkSum; - FstSlice *data = fst->data; + int32_t len; + uint8_t *data = fstSliceData(fst->data, &len); TSCKSUM initSum = 0; - if (!taosCheckChecksumWhole(data->data, data->dLen)) { + if (!taosCheckChecksumWhole(data, len)) { return false; } return true; @@ -1058,9 +1092,14 @@ bool fstVerify(Fst *fst) { FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data) { FstBoundWithData *b = calloc(1, sizeof(FstBoundWithData)); if (b == NULL) { return NULL; } - + + if (data != NULL) { + b->data = fstSliceCopy(data, data->start, data->end); + } else { + b->data = fstSliceCreate(NULL, 0); + } b->type = type; - b->data = fstSliceCopy(data, data->start, data->end); + return b; } @@ -1078,7 +1117,7 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { if (bound->type == Unbounded) { return true; } else { - return fstSliceEmpty(&bound->data); + return fstSliceIsEmpty(&bound->data); } } @@ -1145,8 +1184,10 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { Output out = 0; void* autState = sws->aut->start(); - for (uint32_t i = 0; i < key->dLen; i++) { - uint8_t b = key->data[i]; + int32_t len; + uint8_t *data = fstSliceData(key, &len); + for (uint32_t i = 0; i < len; i++) { + uint8_t b = data[i]; uint64_t res = 0; bool null = fstNodeFindInput(node, b, &res); if (null == false) { @@ -1262,12 +1303,16 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { - FstOutput fOutput = {.null = false, out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; - return swsResultCreate(&slice, fOutput , tState); + FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; + StreamWithStateResult *result = swsResultCreate(&slice, fOutput , tState); + fstSliceDestroy(&slice); + return result; } + fstSliceDestroy(&slice); } return NULL; @@ -1277,14 +1322,19 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult)); if (result == NULL) { return NULL; } - FstSlice slice = fstSliceCopy(data, 0, data->dLen - 1); - result->data = slice; + result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); result->out = fOut; result->state = state; return result; - } +void swsResultDestroy(StreamWithStateResult *result) { + if (NULL == result) { return; } + + fstSliceDestroy(&result->data); + free(result); +} + void streamStateDestroy(void *s) { if (NULL == s) { return; } StreamState *ss = (StreamState *)s; @@ -1293,5 +1343,44 @@ void streamStateDestroy(void *s) { //free(s->autoState); } +FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) { + FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder)); + if (NULL == b) { return NULL; } + + b->fst = fst; + b->aut = aut; + b->min = fstBoundStateCreate(Unbounded, NULL); + b->max = fstBoundStateCreate(Unbounded, NULL); + return b; +} +void fstStreamBuilderDestroy(FstStreamBuilder *b) { + fstSliceDestroy(&b->min->data); + fstSliceDestroy(&b->max->data); + free(b); +} +FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { + if (b == NULL) { return NULL; } + + if (type == GE) { + b->min->type = Included; + fstSliceDestroy(&(b->min->data)); + b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); + } else if (type == GT) { + b->min->type = Excluded; + fstSliceDestroy(&(b->min->data)); + b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); + } else if (type == LE) { + b->max->type = Included; + fstSliceDestroy(&(b->max->data)); + b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); + } else if (type == LT) { + b->max->type = Excluded; + fstSliceDestroy(&(b->max->data)); + b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); + } + return b; +} + + diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index f2f48bbc8a..3d5efd30f3 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -12,3 +12,4 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index a0a2c380f1..6820292e65 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -16,24 +16,91 @@ #include "index_fst_util.h" #include "index_fst_counting_writer.h" +static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { + if (ctx->offset + len > ctx->limit) { + return -1; + } + + if (ctx->type == TFile) { + assert(len != tfWrite(ctx->fd, buf, len)); + } else { + memcpy(ctx->mem + ctx->offset, buf, len); + } + ctx->offset += len; + return len; +} +static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { + if (ctx->type == TFile) { + tfRead(ctx->fd, buf, len); + } else { + memcpy(buf, ctx->mem + ctx->offset, len); + } + ctx->offset += len; + + return 1; +} +static int writeCtxDoFlush(WriterCtx *ctx) { + if (ctx->type == TFile) { + //tfFlush(ctx->fd); + } else { + // do nothing + } + return 1; +} + +WriterCtx* writerCtxCreate(WriterType type) { + WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); + if (ctx == NULL) { return NULL; } + + ctx->type = type; + if (ctx->type == TFile) { + ctx->fd = tfOpenCreateWriteAppend(tmpFile); + if (ctx->fd < 0) { + + } + } else if (ctx->type == TMemory) { + ctx->mem = calloc(1, DefaultMem * sizeof(uint8_t)); + } + ctx->write = writeCtxDoWrite; + ctx->read = writeCtxDoRead; + ctx->flush = writeCtxDoFlush; + + ctx->offset = 0; + ctx->limit = DefaultMem; + + return ctx; +} +void writerCtxDestroy(WriterCtx *ctx) { + if (ctx->type == TMemory) { + free(ctx->mem); + } else { + tfClose(ctx->fd); + } + free(ctx); +} + + FstCountingWriter *fstCountingWriterCreate(void *wrt) { FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); if (cw == NULL) { return NULL; } - - cw->wrt = wrt; + + cw->wrt = (void *)(writerCtxCreate(TFile)); return cw; } void fstCountingWriterDestroy(FstCountingWriter *cw) { // free wrt object: close fd or free mem + writerCtxDestroy((WriterCtx *)(cw->wrt)); free(cw); } -uint64_t fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen) { +int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen) { if (write == NULL) { return 0; } // update checksum // write data to file/socket or mem - - write->count += bufLen; + WriterCtx *ctx = write->wrt; + + int nWrite = ctx->write(ctx, buf, bufLen); + write->count += nWrite; return bufLen; } @@ -41,6 +108,8 @@ uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { return 0; } int fstCountingWriterFlush(FstCountingWriter *write) { + WriterCtx *ctx = write->wrt; + ctx->flush(ctx); //write->wtr->flush return 1; } diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c index b33b8e4428..5abe8ad5a0 100644 --- a/source/libs/index/src/index_fst_node.c +++ b/source/libs/index/src/index_fst_node.c @@ -18,7 +18,7 @@ FstBuilderNode *fstBuilderNodeDefault() { FstBuilderNode *bn = malloc(sizeof(FstBuilderNode)); bn->isFinal = false; bn->finalOutput = 0; - bn->trans = NULL; + bn->trans = taosArrayInit(16, sizeof(FstTransition)); return bn; } void fstBuilderNodeDestroy(FstBuilderNode *node) { @@ -27,6 +27,31 @@ void fstBuilderNodeDestroy(FstBuilderNode *node) { taosArrayDestroy(node->trans); free(node); } + +bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) { + if (n1 == n2) { return true; } + if (n1 == NULL || n2 == NULL ) { + return false; + } + + if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { + return false; + } + size_t s1 = n1->trans? taosArrayGetSize(n1->trans): 0; + size_t s2 = n2->trans? taosArrayGetSize(n2->trans): 0; + if (s1 != s2) { + return false; + } + for (size_t i = 0; i < s1; i++) { + FstTransition *t1 = taosArrayGet(n1->trans, i); + FstTransition *t2 = taosArrayGet(n2->trans, i); + if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { + return false; + } + } + + return true; +} FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); if (node == NULL) { return NULL; } @@ -53,12 +78,17 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { dst->isFinal = src->isFinal; dst->finalOutput = src->finalOutput; - // avoid mem leak + //release free avoid mem leak taosArrayDestroy(dst->trans); - dst->trans = src->trans; - src->trans = NULL; + size_t sz = taosArrayGetSize(src->trans); + dst->trans = taosArrayInit(sz, sizeof(FstTransition)); + for (size_t i = 0; i < sz; i++) { + FstTransition *trn = taosArrayGet(src->trans, i); + taosArrayPush(dst->trans, trn); + } } + //bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { //size_t sz = taosArrayGetSize(b->trans); diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c index b25964e0e7..8fb0dbfcaa 100644 --- a/source/libs/index/src/index_fst_registry.c +++ b/source/libs/index/src/index_fst_registry.c @@ -112,7 +112,7 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo if (end - start == 1) { FstRegistryCell *cell = taosArrayGet(registry->table, start); //cell->isNode && - if (cell->addr != NONE_ADDRESS && cell->node == bNode) { + if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { entry->state = FOUND; entry->addr = cell->addr ; return entry; @@ -123,13 +123,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo } } else if (end - start == 2) { FstRegistryCell *cell1 = taosArrayGet(registry->table, start); - if (cell1->addr != NONE_ADDRESS && cell1->node == bNode) { + if (cell1->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell1->node, bNode)) { entry->state = FOUND; entry->addr = cell1->addr; return entry; } FstRegistryCell *cell2 = taosArrayGet(registry->table, start + 1); - if (cell2->addr != NONE_ADDRESS && cell2->node == bNode) { + if (cell2->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell2->node, bNode)) { entry->state = FOUND; entry->addr = cell2->addr; // must swap here @@ -147,7 +147,7 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo uint32_t i = start; for (; i < end; i++) { FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, i); - if (cell->addr != NONE_ADDRESS && cell->node == bNode) { + if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { entry->state = FOUND; entry->addr = cell->addr; fstRegistryCellPromote(registry->table, i, start); diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index 952c710676..532b0b8ac3 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -91,42 +91,87 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { } // fst slice func -FstSlice fstSliceCreate(uint8_t *data, uint64_t dLen) { - FstSlice slice = {.data = data, .dLen = dLen, .start = 0, .end = dLen - 1}; - return slice; +// + +FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { + FstString *str = (FstString *)malloc(sizeof(FstString)); + str->ref = 1; + str->len = len; + str->data = malloc(len * sizeof(uint8_t)); + memcpy(str->data, data, len); + + FstSlice s = {.str = str, .start = 0, .end = len - 1}; + return s; } // just shallow copy -FstSlice fstSliceCopy(FstSlice *slice, int32_t start, int32_t end) { - FstSlice t; - if (start >= slice->dLen || end >= slice->dLen || start > end) { - t.data = NULL; - return t; - }; - - t.data = slice->data; - t.dLen = slice->dLen; - t.start = start; - t.end = end; +FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { + FstString *str = s->str; + str->ref++; + //uint8_t *buf = fstSliceData(s, &alen); + //start = buf + start - (buf - s->start); + //end = buf + end - (buf - s->start); + + FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start}; return t; } -bool fstSliceEmpty(FstSlice *slice) { - return slice->data == NULL || slice->dLen <= 0; +FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { + + int32_t tlen = end - start + 1; + int32_t slen; + uint8_t *data = fstSliceData(s, &slen); + assert(tlen <= slen); + + uint8_t *buf = malloc(sizeof(uint8_t) * tlen); + memcpy(buf, data + start, tlen); + + FstString *str = malloc(sizeof(FstString)); + str->data = buf; + str->len = tlen; + str->ref = 1; + + FstSlice ans; + ans.str = str; + ans.start = 0; + ans.end = tlen - 1; + return ans; +} +bool fstSliceIsEmpty(FstSlice *s) { + return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; +} + +uint8_t *fstSliceData(FstSlice *s, int32_t *size) { + FstString *str = s->str; + if (size != NULL) { + *size = s->end - s->start + 1; + } + return str->data + s->start; +} +void fstSliceDestroy(FstSlice *s) { + FstString *str = s->str; + str->ref--; + if (str->ref <= 0) { + free(str->data); + free(str); + s->str = NULL; + } } int fstSliceCompare(FstSlice *a, FstSlice *b) { - int32_t aLen = (a->end - a->start + 1); - int32_t bLen = (b->end - b->start + 1); - int32_t mLen = (aLen < bLen ? aLen : bLen); - for (int i = 0; i < mLen; i++) { - uint8_t x = a->data[i + a->start]; - uint8_t y = b->data[i + b->start]; - if (x == y) { continue; } + int32_t alen, blen; + uint8_t *aBuf = fstSliceData(a, &alen); + uint8_t *bBuf = fstSliceData(b, &blen); + + uint32_t i, j; + for (i = 0, j = 0; i < alen && j < blen; i++, j++) { + uint8_t x = aBuf[i]; + uint8_t y = bBuf[j]; + if (x == y) { continue;} else if (x < y) { return -1; } - else { return 1; } - } - if (aLen == bLen) { return 0; } - else if (aLen < bLen) { return -1; } - else { return 1; } + else { return 1; }; + } + if (i < alen) { return 1; } + else if (j < blen) { return -1; } + else { return 0; } } diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index f2a7442a5a..f84f874a23 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -1,7 +1,6 @@ add_executable(indexTest "") target_sources(indexTest PRIVATE - "../src/index.c" "indexTests.cpp" ) target_include_directories ( indexTest diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 857060ce5e..14ff2caf12 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -3,58 +3,84 @@ #include #include "index.h" #include "indexInt.h" +#include "index_fst.h" +#include "index_fst_util.h" +#include "index_fst_counting_writer.h" -TEST(IndexTest, index_create_test) { - SIndexOpts *opts = indexOptsCreate(); - SIndex *index = indexOpen(opts, "./test"); - if (index == NULL) { - std::cout << "index open failed" << std::endl; - } +//TEST(IndexTest, index_create_test) { +// SIndexOpts *opts = indexOptsCreate(); +// SIndex *index = indexOpen(opts, "./test"); +// if (index == NULL) { +// std::cout << "index open failed" << std::endl; +// } +// +// +// // write +// for (int i = 0; i < 100000; i++) { +// SIndexMultiTerm* terms = indexMultiTermCreate(); +// std::string val = "field"; +// +// indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size()); +// +// val.append(std::to_string(i)); +// indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size()); +// +// val.insert(0, std::to_string(i)); +// indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size()); +// +// val.append("const"); +// indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size()); +// +// +// indexPut(index, terms, i); +// indexMultiTermDestroy(terms); +// } +// +// +// // query +// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST); +// +// indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX); +// indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM); +// +// SArray *result = (SArray *)taosArrayInit(10, sizeof(int)); +// indexSearch(index, multiQuery, result); +// +// std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl; +// for (int i = 0; i < taosArrayGetSize(result); i++) { +// int *v = (int *)taosArrayGet(result, i); +// std::cout << "value --->" << *v << std::endl; +// } +// // add more test case +// indexMultiTermQueryDestroy(multiQuery); +// +// indexOptsDestroy(opts); +// indexClose(index); +// // +//} - - // write - for (int i = 0; i < 100000; i++) { - SIndexMultiTerm* terms = indexMultiTermCreate(); - std::string val = "field"; +int main(int argc, char** argv) { + std::string str("abc"); + FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); + Output val = 1; - indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size()); - - val.append(std::to_string(i)); - indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size()); - - val.insert(0, std::to_string(i)); - indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size()); - - val.append("const"); - indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size()); - - - indexPut(index, terms, i); - indexMultiTermDestroy(terms); - } - - - // query - SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST); - - indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX); - indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM); - - SArray *result = (SArray *)taosArrayInit(10, sizeof(int)); - indexSearch(index, multiQuery, result); - - std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl; - for (int i = 0; i < taosArrayGetSize(result); i++) { - int *v = (int *)taosArrayGet(result, i); - std::cout << "value --->" << *v << std::endl; - } - // add more test case - indexMultiTermQueryDestroy(multiQuery); - - indexOptsDestroy(opts); - indexClose(index); - // + //std::string str1("bcd"); + //FstSlice key1 = fstSliceCreate((uint8_t *)str1.c_str(), str1.size()); + //Output val2 = 10; + FstBuilder *b = fstBuilderCreate(NULL, 1); + fstBuilderInsert(b, key, val); + //fstBuilderInsert(b, key1, val2); + fstBuilderFinish(b); + fstBuilderDestroy(b); + fstSliceDestroy(&key); + return 1; } + +//TEST(IndexFstBuilder, IndexFstInput) { +// +//} + + diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index a2041c76fc..e392351366 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -507,7 +507,6 @@ void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) { } int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { -#if 0 SRpcConn *pConn = (SRpcConn *)thandle; if (pConn->user[0] == 0) return -1; @@ -516,9 +515,6 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { // pInfo->serverIp = pConn->destIp; tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user)); -#else - strcpy(pInfo->user, "root"); -#endif return 0; } diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 42ede49c6b..285d7e2576 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -17,14 +17,16 @@ #define _TD_WAL_INT_H_ #include "wal.h" +#include "compare.h" #ifdef __cplusplus extern "C" { #endif -int walRotate(SWal* pWal); int walGetFile(SWal* pWal, int32_t version); +int64_t walGetSeq(); + #ifdef __cplusplus } #endif diff --git a/source/libs/wal/src/walIndex.c b/source/libs/wal/src/walIndex.c index 2569af841f..1aa64b34b5 100644 --- a/source/libs/wal/src/walIndex.c +++ b/source/libs/wal/src/walIndex.c @@ -20,9 +20,38 @@ #include "tfile.h" #include "walInt.h" -int walSeekVerImpl(SWal *pWal, int64_t ver) { - //close old file +static int walSeekFilePos(SWal* pWal, int64_t ver) { int code = 0; + + int64_t idxTfd = pWal->curIdxTfd; + int64_t logTfd = pWal->curLogTfd; + + //seek position + int64_t offset = (ver - pWal->curFileFirstVersion) * WAL_IDX_ENTRY_SIZE; + code = tfLseek(idxTfd, offset, SEEK_SET); + if(code != 0) { + + } + int64_t readBuf[2]; + code = tfRead(idxTfd, readBuf, sizeof(readBuf)); + if(code != 0) { + + } + //TODO:deserialize + ASSERT(readBuf[0] == ver); + code = tfLseek(logTfd, readBuf[1], SEEK_CUR); + if (code != 0) { + + } + pWal->curLogOffset = readBuf[1]; + pWal->curVersion = ver; + return code; +} + +static int walChangeFile(SWal *pWal, int64_t ver) { + int code = 0; + int64_t idxTfd, logTfd; + char fnameStr[WAL_FILE_LEN]; code = tfClose(pWal->curLogTfd); if(code != 0) { //TODO @@ -32,29 +61,36 @@ int walSeekVerImpl(SWal *pWal, int64_t ver) { //TODO } //bsearch in fileSet - int fName = 0;//TODO - //open the right file - char fNameStr[WAL_FILE_LEN]; - sprintf(fNameStr, "%d."WAL_INDEX_SUFFIX, fName); - bool closed = 1; //TODO:read only - int64_t idxTfd = tfOpenReadWrite(fNameStr); - sprintf(fNameStr, "%d."WAL_LOG_SUFFIX, fName); - int64_t logTfd = tfOpenReadWrite(fNameStr); - //seek position - int64_t offset = (ver - fName) * WAL_IDX_ENTRY_SIZE; - tfLseek(idxTfd, offset, SEEK_SET); - //set cur version, cur file version and cur status - pWal->curFileFirstVersion = fName; - pWal->curFileLastVersion = 1;//TODO + int64_t* pRet = taosArraySearch(pWal->fileSet, &ver, compareInt64Val, TD_LE); + ASSERT(pRet != NULL); + int64_t fname = *pRet; + if(fname < pWal->lastFileName) { + pWal->curStatus &= ~WAL_CUR_FILE_WRITABLE; + pWal->curFileLastVersion = pRet[1]-1; + sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname); + idxTfd = tfOpenRead(fnameStr); + sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname); + logTfd = tfOpenRead(fnameStr); + } else { + pWal->curStatus |= WAL_CUR_FILE_WRITABLE; + pWal->curFileLastVersion = -1; + sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname); + idxTfd = tfOpenReadWrite(fnameStr); + sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname); + logTfd = tfOpenReadWrite(fnameStr); + } + + pWal->curFileFirstVersion = fname; pWal->curLogTfd = logTfd; pWal->curIdxTfd = idxTfd; - pWal->curVersion = ver; - pWal->curOffset = offset; - pWal->curStatus = 0;//TODO return code; } int walSeekVer(SWal *pWal, int64_t ver) { + if((!(pWal->curStatus & WAL_CUR_FAILED)) + && ver == pWal->curVersion) { + return 0; + } if(ver > pWal->lastVersion) { //TODO: some records are skipped return -1; @@ -64,54 +100,13 @@ int walSeekVer(SWal *pWal, int64_t ver) { return -1; } if(ver < pWal->snapshotVersion) { - //TODO: seek snapshotted log + //TODO: seek snapshotted log, invalid in some cases } - if(ver >= pWal->curFileFirstVersion - && ((pWal->curFileLastVersion == -1 && ver <= pWal->lastVersion) || (ver <= pWal->curFileLastVersion))) { - - } - if(ver < pWal->curFileFirstVersion || (pWal->curFileLastVersion != -1 && ver > pWal->curFileLastVersion)) { - int index = 0; - index = 1; - //back up to avoid inconsistency - int64_t curVersion = pWal->curVersion; - int64_t curOffset = pWal->curOffset; - int64_t curFileFirstVersion = pWal->curFileFirstVersion; - int64_t curFileLastVersion = pWal->curFileLastVersion; - if(walSeekVerImpl(pWal, ver) < 0) { - //TODO: errno - pWal->curVersion = curVersion; - pWal->curOffset = curOffset; - pWal->curFileFirstVersion = curFileFirstVersion; - pWal->curFileLastVersion = curFileLastVersion; - return -1; - } + if(ver < pWal->curFileFirstVersion || + (pWal->curFileLastVersion != -1 && ver > pWal->curFileLastVersion)) { + walChangeFile(pWal, ver); } + walSeekFilePos(pWal, ver); return 0; } - -int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { - int code = 0; - //get index file - if(!tfValid(pWal->curIdxTfd)) { - code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno)); - } - if(pWal->curVersion != ver) { - if(walSeekVer(pWal, ver) != 0) { - //TODO: some records are skipped - return -1; - } - } - //check file checksum - //append index - return 0; -} - -int walRotateIndex(SWal *pWal) { - //check file checksum - //create new file - //switch file - return 0; -} diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index d60cdfe118..bc2e687069 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -26,57 +26,44 @@ int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId); int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId); int32_t walGetNewFile(SWal *pWal, int64_t *newFileId); -static pthread_mutex_t walInitLock = PTHREAD_MUTEX_INITIALIZER; -static int8_t walInited = 0; - typedef struct { int32_t refSetId; - int32_t seq; + uint32_t seq; int8_t stop; + int8_t inited; pthread_t thread; - pthread_mutex_t mutex; } SWalMgmt; -static SWalMgmt tsWal = {0}; +static SWalMgmt tsWal = {0, .seq = 1}; static int32_t walCreateThread(); static void walStopThread(); static int32_t walInitObj(SWal *pWal); static void walFreeObj(void *pWal); -int32_t walInit() { - //TODO: change to atomic - pthread_mutex_lock(&walInitLock); - if(walInited) { - pthread_mutex_unlock(&walInitLock); - return 0; - } else { - walInited = 1; - pthread_mutex_unlock(&walInitLock); - } +int64_t walGetSeq() { + return (int64_t)atomic_load_32(&tsWal.seq); +} + +int32_t walInit() { + int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1); + if(old == 1) return 0; - int32_t code = 0; tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj); - code = pthread_mutex_init(&tsWal.mutex, NULL); - if (code != 0) { - wError("failed to init wal mutex since %s", tstrerror(code)); - return code; - } - - code = walCreateThread(); + int code = walCreateThread(); if (code != 0) { wError("failed to init wal module since %s", tstrerror(code)); + atomic_store_8(&tsWal.inited, 0); return code; } wInfo("wal module is initialized, rsetId:%d", tsWal.refSetId); - return code; + return 0; } void walCleanUp() { walStopThread(); taosCloseRef(tsWal.refSetId); - pthread_mutex_destroy(&tsWal.mutex); wInfo("wal module is cleaned up"); } @@ -92,7 +79,7 @@ static int walLoadFileset(SWal *pWal) { char *name = ent->d_name; name[WAL_NOSUFFIX_LEN] = 0; //validate file name by regex matching - if(1 /* regex match */) { + if(1 /* TODO:regex match */) { int64_t fnameInt64 = atoll(name); taosArrayPush(pWal->fileSet, &fnameInt64); } @@ -133,6 +120,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { walFreeObj(pWal); return NULL; } + walLoadFileset(pWal); wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod); @@ -164,6 +152,9 @@ void walClose(SWal *pWal) { pthread_mutex_lock(&pWal->mutex); tfClose(pWal->curLogTfd); + tfClose(pWal->curIdxTfd); + taosArrayDestroy(pWal->fileSet); + pWal->fileSet = NULL; pthread_mutex_unlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); } @@ -188,6 +179,9 @@ static void walFreeObj(void *wal) { wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal); tfClose(pWal->curLogTfd); + tfClose(pWal->curIdxTfd); + taosArrayDestroy(pWal->fileSet); + pWal->fileSet = NULL; pthread_mutex_destroy(&pWal->mutex); tfree(pWal); } @@ -197,7 +191,7 @@ static bool walNeedFsync(SWal *pWal) { return false; } - if (tsWal.seq % pWal->fsyncSeq == 0) { + if (atomic_load_32(&tsWal.seq) % pWal->fsyncSeq == 0) { return true; } @@ -206,16 +200,14 @@ static bool walNeedFsync(SWal *pWal) { static void walUpdateSeq() { taosMsleep(WAL_REFRESH_MS); - if (++tsWal.seq <= 0) { - tsWal.seq = 1; - } + atomic_add_fetch_32(&tsWal.seq, 1); } static void walFsyncAll() { SWal *pWal = taosIterateRef(tsWal.refSetId, 0); while (pWal) { if (walNeedFsync(pWal)) { - wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq); + wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); int32_t code = tfFsync(pWal->curLogTfd); if (code != 0) { wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(code)); @@ -226,16 +218,12 @@ static void walFsyncAll() { } static void *walThreadFunc(void *param) { - int stop = 0; setThreadName("wal"); while (1) { walUpdateSeq(); walFsyncAll(); - pthread_mutex_lock(&tsWal.mutex); - stop = tsWal.stop; - pthread_mutex_unlock(&tsWal.mutex); - if (stop) break; + if (atomic_load_8(&tsWal.stop)) break; } return NULL; @@ -258,9 +246,7 @@ static int32_t walCreateThread() { } static void walStopThread() { - pthread_mutex_lock(&tsWal.mutex); - tsWal.stop = 1; - pthread_mutex_unlock(&tsWal.mutex); + atomic_store_8(&tsWal.stop, 1); if (taosCheckPthreadValid(tsWal.thread)) { pthread_join(tsWal.thread, NULL); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 7563ec02c7..69c83a9912 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -26,10 +26,24 @@ int32_t walCommit(SWal *pWal, int64_t ver) { } int32_t walRollback(SWal *pWal, int64_t ver) { + //TODO: ftruncate return 0; } int32_t walTakeSnapshot(SWal *pWal, int64_t ver) { + pWal->snapshotVersion = ver; + + //mark files safe to delete + int64_t* pRet = taosArraySearch(pWal->fileSet, &ver, compareInt64Val, TD_LE); + if(pRet != pWal->fileSet->pData) { + //delete files until less than retention size + + //find first file that exceeds retention time + + } + + //delete files living longer than retention limit + //remove file from fileset return 0; } @@ -124,13 +138,102 @@ void walRemoveAllOldFiles(void *handle) { } #endif +static int walRoll(SWal *pWal) { + int code = 0; + code = tfClose(pWal->curIdxTfd); + if(code != 0) { + return code; + } + code = tfClose(pWal->curLogTfd); + if(code != 0) { + return code; + } + int64_t idxTfd, logTfd; + //create new file + int64_t newFileFirstVersion = pWal->lastVersion + 1; + char fnameStr[WAL_FILE_LEN]; + sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, newFileFirstVersion); + idxTfd = tfOpenCreateWrite(fnameStr); + sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, newFileFirstVersion); + logTfd = tfOpenCreateWrite(fnameStr); + + taosArrayPush(pWal->fileSet, &newFileFirstVersion); + + //switch file + pWal->curIdxTfd = idxTfd; + pWal->curLogTfd = logTfd; + //change status + pWal->curFileLastVersion = -1; + pWal->curFileFirstVersion = newFileFirstVersion; + pWal->curVersion = newFileFirstVersion; + pWal->curLogOffset = 0; + pWal->curStatus = WAL_CUR_FILE_WRITABLE & WAL_CUR_POS_WRITABLE; + + pWal->lastFileName = newFileFirstVersion; + pWal->lastFileWriteSize = 0; + pWal->lastRollSeq = walGetSeq(); + return 0; +} + +int walChangeFileToLast(SWal *pWal) { + int64_t idxTfd, logTfd; + int64_t* pRet = taosArrayGetLast(pWal->fileSet); + ASSERT(pRet != NULL); + int64_t fname = *pRet; + + char fnameStr[WAL_FILE_LEN]; + sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname); + idxTfd = tfOpenReadWrite(fnameStr); + sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname); + logTfd = tfOpenReadWrite(fnameStr); + //switch file + pWal->curIdxTfd = idxTfd; + pWal->curLogTfd = logTfd; + //change status + pWal->curFileLastVersion = -1; + pWal->curFileFirstVersion = fname; + pWal->curVersion = fname; + pWal->curLogOffset = 0; + pWal->curStatus = WAL_CUR_FILE_WRITABLE; + return 0; +} + +int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { + int code = 0; + //get index file + if(!tfValid(pWal->curIdxTfd)) { + code = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno)); + } + int64_t writeBuf[2] = { ver, offset }; + int size = tfWrite(pWal->curIdxTfd, writeBuf, sizeof(writeBuf)); + if(size != sizeof(writeBuf)) { + //TODO: + } + return 0; +} + int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, void *body, int32_t bodyLen) { if (pWal == NULL) return -1; // no wal - if (!tfValid(pWal->curLogTfd)) return 0; if (pWal->level == TAOS_WAL_NOLOG) return 0; - if (index > pWal->lastVersion + 1) return -1; + + if (index == pWal->lastVersion + 1) { + int64_t passed = walGetSeq() - pWal->lastRollSeq; + if(passed > pWal->rollPeriod) { + walRoll(pWal); + } else if(pWal->lastFileWriteSize > pWal->segSize) { + walRoll(pWal); + } else { + walChangeFileToLast(pWal); + } + } else { + //reject skip log or rewrite log + //must truncate explicitly first + return -1; + } + if (!tfValid(pWal->curLogTfd)) return 0; pWal->head.version = index; int32_t code = 0; @@ -155,8 +258,12 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, void *body, int32_t code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno)); } - //TODO:write idx + walWriteIndex(pWal, index, pWal->curLogOffset); + pWal->curLogOffset += sizeof(SWalHead) + bodyLen; + //set status + pWal->lastVersion = index; + pthread_mutex_unlock(&pWal->mutex); return code; diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 5b61186b12..5d4789aae6 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -53,6 +53,11 @@ static int64_t tfOpenImp(int32_t fd) { return rid; } +int64_t tfOpenRead(const char *pathname, int32_t flags) { + int32_t fd = taosOpenFileRead(pathname); + return tfOpenImp(fd); +} + int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { int32_t fd = taosOpenFileReadWrite(pathname); return tfOpenImp(fd); diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index ee524e4448..b9d2da6939 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -410,3 +410,10 @@ char *taosIpStr(uint32_t ipInt) { return ipStr; } +void taosIp2String(uint32_t ip, char *str) { + sprintf(str, "%u.%u.%u.%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24)); +} + +void taosIpPort2String(uint32_t ip, uint16_t port, char *str) { + sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port); +} \ No newline at end of file