From 53ed71959b49fc493db90a0bc55fb9f04e2b2d7b Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 13 Sep 2023 11:30:04 +0800 Subject: [PATCH 01/16] enh: fetch cluster info for grant periodically --- include/common/tglobal.h | 1 + include/common/tmsgdef.h | 1 + source/common/src/tglobal.c | 1 + source/dnode/mnode/impl/src/mndGrant.c | 2 ++ source/dnode/mnode/impl/src/mndMain.c | 15 +++++++++++++++ 5 files changed, 20 insertions(+) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 1c5510ba0d..1b37c8c855 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -198,6 +198,7 @@ extern int32_t tsTtlPushIntervalSec; extern int32_t tsTtlBatchDropNum; extern int32_t tsTrimVDbIntervalSec; extern int32_t tsGrantHBInterval; +extern int32_t tsGrantFetchInterval; extern int32_t tsUptimeInterval; extern int32_t tsRpcRetryLimit; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index fb2c780724..13811fae83 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -177,6 +177,7 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_UPTIME_TIMER, "uptime-timer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, "lost-consumer-clear", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_HEARTBEAT, "stream-heartbeat", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_GRANT_FETCH_TIMER, "grant-fetch-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_BALANCE_VGROUP_LEADER, "balance-vgroup-leader", NULL, NULL) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 90812a66b2..a9c712c3d0 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -247,6 +247,7 @@ int32_t tsTtlUnit = 86400; int32_t tsTtlPushIntervalSec = 10; int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups int32_t tsGrantHBInterval = 60; +int32_t tsGrantFetchInterval = 2; int32_t tsUptimeInterval = 300; // seconds char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits char tsUdfdLdLibPath[512] = ""; diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index 1b46e16961..fbe987975a 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -117,10 +117,12 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl } static int32_t mndProcessGrantHB(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; } +static int32_t mndProcessGrantFetch(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; } int32_t mndInitGrant(SMnode *pMnode) { mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS, mndRetrieveGrant); mndSetMsgHandle(pMnode, TDMT_MND_GRANT_HB_TIMER, mndProcessGrantHB); + mndSetMsgHandle(pMnode, TDMT_MND_GRANT_FETCH_TIMER, mndProcessGrantFetch); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 1c87cde78a..fb97d688d6 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -175,6 +175,17 @@ static void mndPullupGrant(SMnode *pMnode) { } } +static void mndFetchGrant(SMnode *pMnode) { + mTrace("fetch grant msg"); + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + if (pReq != NULL) { + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_GRANT_FETCH_TIMER, .pCont = pReq, .contLen = contLen, .info.ahandle = (void *)0x9529}; + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); + } +} + static void mndIncreaseUpTime(SMnode *pMnode) { mTrace("increate uptime"); int32_t contLen = 0; @@ -293,6 +304,10 @@ static void *mndThreadFp(void *param) { mndPullupTelem(pMnode); } + if (sec % tsGrantFetchInterval == 0) { + mndFetchGrant(pMnode); + } + if (sec % tsGrantHBInterval == 0) { mndPullupGrant(pMnode); } From 62c4113f82c90981752680530727757c81e7f744 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 13 Sep 2023 11:38:18 +0800 Subject: [PATCH 02/16] chore: update ts grant fetch interval --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a9c712c3d0..5afd0f10d3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -247,7 +247,7 @@ int32_t tsTtlUnit = 86400; int32_t tsTtlPushIntervalSec = 10; int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups int32_t tsGrantHBInterval = 60; -int32_t tsGrantFetchInterval = 2; +int32_t tsGrantFetchInterval = 5; int32_t tsUptimeInterval = 300; // seconds char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits char tsUdfdLdLibPath[512] = ""; From 449092c5dc6b8dfd2e9c29e8f0bf1ed7b860e292 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 14 Sep 2023 20:16:02 +0800 Subject: [PATCH 03/16] enh: timeseries update optimize --- include/common/tmsgdef.h | 1 - source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 5 +++-- source/dnode/mnode/impl/src/mndDnode.c | 10 ++++++++++ source/dnode/mnode/impl/src/mndGrant.c | 2 -- source/dnode/mnode/impl/src/mndMain.c | 15 --------------- source/dnode/vnode/src/meta/metaQuery.c | 2 +- source/dnode/vnode/src/meta/metaTable.c | 6 ++++++ 7 files changed, 20 insertions(+), 21 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 13811fae83..fb2c780724 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -177,7 +177,6 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_UPTIME_TIMER, "uptime-timer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, "lost-consumer-clear", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_HEARTBEAT, "stream-heartbeat", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_GRANT_FETCH_TIMER, "grant-fetch-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_BALANCE_VGROUP_LEADER, "balance-vgroup-leader", NULL, NULL) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 76cb65b53a..5c08bc4dbc 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -17,6 +17,7 @@ #include "dmInt.h" #include "thttp.h" +int8_t tsNeedUpdStatus = 0; static void *dmStatusThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); @@ -28,13 +29,13 @@ static void *dmStatusThreadFp(void *param) { int64_t upTime = 0; while (1) { - taosMsleep(200); + taosMsleep(100); if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); if (curTime < lastTime) lastTime = curTime; float interval = (curTime - lastTime) / 1000.0f; - if (interval >= tsStatusInterval) { + if (atomic_val_compare_exchange_8(&tsNeedUpdStatus, 1, 0) == 1 || interval >= tsStatusInterval) { dmSendStatusReq(pMgmt); lastTime = curTime; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index a11bd2556e..7997b31e76 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -51,6 +51,7 @@ enum { DND_CONN_ACTIVE_CODE, }; +extern int32_t mndUpdateClusterInfo(SRpcMsg *pReq); static int32_t mndCreateDefaultDnode(SMnode *pMnode); static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); @@ -525,6 +526,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { bool reboot = (pDnode->rebootTime != statusReq.rebootTime); bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes; bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged; + int64_t nDiffTimeSeries = 0; const STraceId *trace = &pReq->info.traceId; mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, @@ -535,6 +537,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVload->vgId); if (pVgroup != NULL) { + nDiffTimeSeries = pVload->numOfTimeSeries - pVgroup->numOfTimeSeries; if (pVload->syncState == TAOS_SYNC_STATE_LEADER) { pVgroup->cacheUsage = pVload->cacheUsage; pVgroup->numOfCachedTables = pVload->numOfCachedTables; @@ -662,9 +665,16 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { _OVER: mndReleaseDnode(pMnode, pDnode); taosArrayDestroy(statusReq.pVloads); + if (nDiffTimeSeries > 0) { + mndUpdateClusterInfo(pReq); + } return code; } +#ifndef TD_ENTERPRISE +int32_t mndUpdateClusterInfo(SRpcMsg *pReq) { return 0; } +#endif + static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pCreate) { int32_t code = -1; SSdbRaw *pRaw = NULL; diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index fbe987975a..1b46e16961 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -117,12 +117,10 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl } static int32_t mndProcessGrantHB(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; } -static int32_t mndProcessGrantFetch(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; } int32_t mndInitGrant(SMnode *pMnode) { mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS, mndRetrieveGrant); mndSetMsgHandle(pMnode, TDMT_MND_GRANT_HB_TIMER, mndProcessGrantHB); - mndSetMsgHandle(pMnode, TDMT_MND_GRANT_FETCH_TIMER, mndProcessGrantFetch); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index fb97d688d6..1c87cde78a 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -175,17 +175,6 @@ static void mndPullupGrant(SMnode *pMnode) { } } -static void mndFetchGrant(SMnode *pMnode) { - mTrace("fetch grant msg"); - int32_t contLen = 0; - void *pReq = mndBuildTimerMsg(&contLen); - if (pReq != NULL) { - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_GRANT_FETCH_TIMER, .pCont = pReq, .contLen = contLen, .info.ahandle = (void *)0x9529}; - tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); - } -} - static void mndIncreaseUpTime(SMnode *pMnode) { mTrace("increate uptime"); int32_t contLen = 0; @@ -304,10 +293,6 @@ static void *mndThreadFp(void *param) { mndPullupTelem(pMnode); } - if (sec % tsGrantFetchInterval == 0) { - mndFetchGrant(pMnode); - } - if (sec % tsGrantHBInterval == 0) { mndPullupGrant(pMnode); } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index c74f36eaa2..903f7d4cce 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -700,7 +700,7 @@ int64_t metaGetTbNum(SMeta *pMeta) { int64_t metaGetTimeSeriesNum(SMeta *pMeta) { // sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column) int64_t nTables = metaGetTbNum(pMeta); - if (nTables - pMeta->pVnode->config.vndStats.numOfCmprTables > 100 || + if (nTables - pMeta->pVnode->config.vndStats.numOfCmprTables > 0 || pMeta->pVnode->config.vndStats.numOfTimeSeries <= 0 || ++pMeta->pVnode->config.vndStats.itvTimeSeries % (60 * 5) == 0) { int64_t num = 0; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 9a298a4bb7..8a71327919 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -15,6 +15,8 @@ #include "meta.h" +extern int8_t tsNeedUpdStatus; + static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME); @@ -788,6 +790,8 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs if (metaHandleEntry(pMeta, &me) < 0) goto _err; + atomic_val_compare_exchange_8(&tsNeedUpdStatus, 0, 1); + if (pMetaRsp) { *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); @@ -1094,6 +1098,8 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { --pMeta->pVnode->config.vndStats.numOfSTables; } + atomic_val_compare_exchange_8(&tsNeedUpdStatus, 0, 1); + metaCacheDrop(pMeta, uid); tDecoderClear(&dc); From 7b29d62f04a4a91333c3850e83ae5a49fcafe7da Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 14 Sep 2023 20:18:13 +0800 Subject: [PATCH 04/16] enh: timeseries update optimize --- include/common/tglobal.h | 1 - source/common/src/tglobal.c | 1 - 2 files changed, 2 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 1b37c8c855..1c5510ba0d 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -198,7 +198,6 @@ extern int32_t tsTtlPushIntervalSec; extern int32_t tsTtlBatchDropNum; extern int32_t tsTrimVDbIntervalSec; extern int32_t tsGrantHBInterval; -extern int32_t tsGrantFetchInterval; extern int32_t tsUptimeInterval; extern int32_t tsRpcRetryLimit; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5afd0f10d3..90812a66b2 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -247,7 +247,6 @@ int32_t tsTtlUnit = 86400; int32_t tsTtlPushIntervalSec = 10; int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups int32_t tsGrantHBInterval = 60; -int32_t tsGrantFetchInterval = 5; int32_t tsUptimeInterval = 300; // seconds char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits char tsUdfdLdLibPath[512] = ""; From 8505fb2aeabfd644ac87d4b6db3003e193668743 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 15 Sep 2023 08:49:45 +0800 Subject: [PATCH 05/16] chore: code refactor for mndDnode --- source/dnode/mnode/impl/src/mndDnode.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 7997b31e76..cfa851d35d 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -472,6 +472,10 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) { return stateChanged; } +#ifndef TD_ENTERPRISE +int32_t mndUpdateClusterInfo(SRpcMsg *pReq) { return 0; } +#endif + static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStatusReq statusReq = {0}; @@ -671,10 +675,6 @@ _OVER: return code; } -#ifndef TD_ENTERPRISE -int32_t mndUpdateClusterInfo(SRpcMsg *pReq) { return 0; } -#endif - static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pCreate) { int32_t code = -1; SSdbRaw *pRaw = NULL; From 6b95013ba01a17d4146b6a53a56c022db4d35726 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 15 Sep 2023 10:28:07 +0800 Subject: [PATCH 06/16] chore: code refactor for mndDnode --- source/dnode/mnode/impl/src/mndDnode.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index cfa851d35d..c27778a413 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -51,7 +51,6 @@ enum { DND_CONN_ACTIVE_CODE, }; -extern int32_t mndUpdateClusterInfo(SRpcMsg *pReq); static int32_t mndCreateDefaultDnode(SMnode *pMnode); static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); @@ -473,7 +472,9 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) { } #ifndef TD_ENTERPRISE -int32_t mndUpdateClusterInfo(SRpcMsg *pReq) { return 0; } +static int32_t mndUpdateClusterInfo(SRpcMsg *pReq) { return 0; } +#else +int32_t mndUpdateClusterInfo(SRpcMsg *pReq); #endif static int32_t mndProcessStatusReq(SRpcMsg *pReq) { From 03aea454d7838018576225bffd37e5d358726671 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 15 Sep 2023 14:33:35 +0800 Subject: [PATCH 07/16] chore: more code --- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 3 +++ source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 14 ++++++++++++++ source/dnode/mnode/impl/inc/mndGrant.h | 2 ++ source/dnode/mnode/impl/src/mndDnode.c | 10 +++++----- source/dnode/mnode/impl/src/mndMain.c | 11 +++++++---- source/dnode/vnode/inc/vnode.h | 1 - source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 7 ++----- source/dnode/vnode/src/vnd/vnodeQuery.c | 2 +- 9 files changed, 35 insertions(+), 17 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 09783a5ea9..ab430fa60c 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -26,6 +26,9 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { if (dmStartCrashReportThread(pMgmt) != 0) { return -1; } + if (dmStartNotifyThread(pMgmt) != 0) { + return -1; + } return 0; } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 5c08bc4dbc..08816749bf 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -134,6 +134,20 @@ static void *dmCrashReportThreadFp(void *param) { } +int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusThreadFp, pMgmt) != 0) { + dError("failed to create status thread since %s", strerror(errno)); + return -1; + } + + taosThreadAttrDestroy(&thAttr); + tmsgReportStartup("dnode-status", "initialized"); + return 0; +} + int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index 88f118cb8f..a036bf6d7e 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -22,6 +22,8 @@ #include "mndInt.h" +#define GRANT_HB_INTERVAL 300 // 300 seconds + int32_t mndInitGrant(SMnode * pMnode); void mndCleanupGrant(); void grantParseParameter(); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 7997b31e76..3e8d64b2a5 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -51,7 +51,7 @@ enum { DND_CONN_ACTIVE_CODE, }; -extern int32_t mndUpdateClusterInfo(SRpcMsg *pReq); +int32_t mndUpdateClusterInfo(SRpcMsg *pReq); static int32_t mndCreateDefaultDnode(SMnode *pMnode); static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); @@ -472,6 +472,10 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) { return stateChanged; } +#ifndef TD_ENTERPRISE +int32_t mndUpdateClusterInfo(SRpcMsg *pReq) { return 0; } +#endif + static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStatusReq statusReq = {0}; @@ -671,10 +675,6 @@ _OVER: return code; } -#ifndef TD_ENTERPRISE -int32_t mndUpdateClusterInfo(SRpcMsg *pReq) { return 0; } -#endif - static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pCreate) { int32_t code = -1; SSdbRaw *pRaw = NULL; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 1c87cde78a..e00af5375d 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -262,6 +262,13 @@ static void *mndThreadFp(void *param) { lastTime++; taosMsleep(100); if (mndGetStop(pMnode)) break; + + if (atomic_val_compare_exchange_32(&tsGrantHBInterval, -GRANT_HB_INTERVAL, GRANT_HB_INTERVAL) == + -GRANT_HB_INTERVAL || + (lastTime % (tsGrantHBInterval * 10) == 0)) { + mndPullupGrant(pMnode); + } + if (lastTime % 10 != 0) continue; int64_t sec = lastTime / 10; @@ -293,10 +300,6 @@ static void *mndThreadFp(void *param) { mndPullupTelem(pMnode); } - if (sec % tsGrantHBInterval == 0) { - mndPullupGrant(pMnode); - } - if (sec % tsUptimeInterval == 0) { mndIncreaseUpTime(pMnode); } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e15f5f911d..5ae257aef8 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -288,7 +288,6 @@ typedef struct { int64_t numOfSTables; int64_t numOfCTables; int64_t numOfNTables; - int64_t numOfCmprTables; int64_t numOfNTimeSeries; int64_t numOfTimeSeries; int64_t itvTimeSeries; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 536273c044..6eda33424f 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -168,7 +168,7 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); -int64_t metaGetTimeSeriesNum(SMeta* pMeta); +int64_t metaGetTimeSeriesNum(SMeta* pMeta, int type); SMCtbCursor* metaOpenCtbCursor(void* pVnode, tb_uid_t uid, int lock); int32_t metaResumeCtbCursor(SMCtbCursor* pCtbCur, int8_t first); void metaPauseCtbCursor(SMCtbCursor* pCtbCur); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 903f7d4cce..c7b7ce9686 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -697,18 +697,15 @@ int64_t metaGetTbNum(SMeta *pMeta) { } // N.B. Called by statusReq per second -int64_t metaGetTimeSeriesNum(SMeta *pMeta) { +int64_t metaGetTimeSeriesNum(SMeta *pMeta, int type) { // sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column) - int64_t nTables = metaGetTbNum(pMeta); - if (nTables - pMeta->pVnode->config.vndStats.numOfCmprTables > 0 || - pMeta->pVnode->config.vndStats.numOfTimeSeries <= 0 || + if (type || pMeta->pVnode->config.vndStats.numOfTimeSeries <= 0 || ++pMeta->pVnode->config.vndStats.itvTimeSeries % (60 * 5) == 0) { int64_t num = 0; vnodeGetTimeSeriesNum(pMeta->pVnode, &num); pMeta->pVnode->config.vndStats.numOfTimeSeries = num; pMeta->pVnode->config.vndStats.itvTimeSeries = (TD_VID(pMeta->pVnode) % 100) * 2; - pMeta->pVnode->config.vndStats.numOfCmprTables = nTables; } return pMeta->pVnode->config.vndStats.numOfTimeSeries + pMeta->pVnode->config.vndStats.numOfNTimeSeries; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 01dd062866..16a21ca405 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -388,7 +388,7 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->cacheUsage = tsdbCacheGetUsage(pVnode); pLoad->numOfCachedTables = tsdbCacheGetElems(pVnode); pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); - pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta); + pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta, 0); pLoad->totalStorage = (int64_t)3 * 1073741824; pLoad->compStorage = (int64_t)2 * 1073741824; pLoad->pointsWritten = 100; From 6698861c8bd0fc795022619ba97b802e02ef92ee Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 19 Sep 2023 08:29:51 +0800 Subject: [PATCH 08/16] chore: timeseries distribute --- include/common/tmsg.h | 16 +++ include/common/tmsgdef.h | 2 + include/dnode/mgmt/dnode.h | 1 + source/common/src/tmsg.c | 46 +++++++++ source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 2 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 105 +++++++++++++++++++- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 2 +- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 18 +--- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mgmt/node_util/inc/dmUtil.h | 2 + source/dnode/mnode/impl/src/mndDnode.c | 16 +-- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/meta/metaTable.c | 17 +++- source/dnode/vnode/src/vnd/vnodeQuery.c | 2 +- 14 files changed, 201 insertions(+), 30 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9d1a8a9189..e64314c723 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1511,6 +1511,22 @@ int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq); +typedef struct { + int32_t vgId; + int64_t nTimeSeries; +} SDndNotifyInfo; + +int32_t dmProcessNotifyReq(SDndNotifyInfo* pInfo); + +typedef struct { + int32_t nVgroup; + SDndNotifyInfo* payload; +} SNotifyReq; + +int32_t tSerializeSNotifyReq(void* buf, int32_t bufLen, SNotifyReq* pReq); +int32_t tDeserializeSNotifyReq(void* buf, int32_t bufLen, SNotifyReq* pReq); +void tFreeSNotifyReq(SNotifyReq* pReq); + typedef struct { int32_t dnodeId; int64_t clusterId; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index dd9a1af67c..39a36a248b 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -179,6 +179,8 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_STREAM_HEARTBEAT, "stream-heartbeat", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_IP_WHITE, "retrieve-ip-white", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_USER_WHITELIST, "get-user-whitelist", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_NOTIFY, "notify", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_BALANCE_VGROUP_LEADER, "balance-vgroup-leader", NULL, NULL) diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 82823e3f57..c9f2339d11 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -44,6 +44,7 @@ int32_t dmRun(); */ void dmStop(); + #ifdef __cplusplus } #endif diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 2285d0df23..457f450945 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1035,6 +1035,52 @@ int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextR return 0; } +int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeI32(&encoder, pReq->nVgroup) < 0) return -1; + for (int32_t i = 0; i < pReq->nVgroup; ++i) { + if (tEncodeI32(&encoder, (pReq->payload + i)->vgId) < 0) return -1; + if (tEncodeI64(&encoder, (pReq->payload + i)->nTimeSeries) < 0) return -1; + } + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeI32(&decoder, &pReq->nVgroup) < 0) return -1; + + pReq->payload = taosMemoryMalloc(pReq->nVgroup * (sizeof(SDndNotifyInfo))); + if (!pReq->payload) return -1; + + for (int32_t i = 0; i < pReq->nVgroup; ++i) { + if (tDecodeI32(&decoder, &((pReq->payload + i)->vgId)) < 0) return -1; + if (tDecodeI64(&decoder, &((pReq->payload + i)->nTimeSeries)) < 0) return -1; + } + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + +void tFreeSNotifyReq(SNotifyReq *pReq) { + if (pReq) { + taosMemoryFreeClear(pReq->payload); + } +} + int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 35947c001b..0cf2c4f7b8 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -50,6 +50,8 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg); +int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg); +int32_t dmStartNotify(SDnodeMgmt *pMgmt); // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index a56387079f..bc53827222 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -130,11 +130,11 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { taosThreadRwlockUnlock(&pMgmt->pData->lock); SMonVloadInfo vinfo = {0}; - (*pMgmt->getVnodeLoadsFp)(&vinfo); + (*pMgmt->getVnodeLoadsFp)(&vinfo); req.pVloads = vinfo.pVloads; SMonMloadInfo minfo = {0}; - (*pMgmt->getMnodeLoadsFp)(&minfo); + (*pMgmt->getMnodeLoadsFp)(&minfo); req.mload = minfo.load; (*pMgmt->getQnodeLoadsFp)(&req.qload); @@ -198,7 +198,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SServerStatusRsp statusRsp = {0}; SMonMloadInfo minfo = {0}; - (*pMgmt->getMnodeLoadsFp)(&minfo); + (*pMgmt->getMnodeLoadsFp)(&minfo); if (minfo.isMnode && (minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; @@ -207,7 +207,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { } SMonVloadInfo vinfo = {0}; - (*pMgmt->getVnodeLoadsFp)(&vinfo); + (*pMgmt->getVnodeLoadsFp)(&vinfo); for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) { SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i); if (pLoad->syncState == TAOS_SYNC_STATE_ERROR || pLoad->syncState == TAOS_SYNC_STATE_OFFLINE) { @@ -395,6 +395,7 @@ SArray *dmGetMsgHandles() { // Requests handled by MNODE if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_NOTIFY, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH_RSP, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; code = 0; @@ -407,3 +408,99 @@ _OVER: return pArray; } } + +#ifndef TD_ENTERPRISE +int32_t dmStartNotify(SDnodeMgmt *pMgmt) { return 0; } +int32_t dmProcessNotifyReq(SDndNotifyInfo *pInfo) { return 0; } +#else +static SHashObj *tsDndNotifyInfo = NULL; +static int64_t tsTimeSeries = 0; +static int8_t tsDndNotifyInitLock = 0; +static int8_t tsDndNotifyLock = 0; +static SDnodeMgmt *pSDnodeMgmt = NULL; + +int32_t dmStartNotify(SDnodeMgmt *pMgmt) { + pSDnodeMgmt = pMgmt; + return 0; +} + +int32_t dmProcessNotifyReq(SDndNotifyInfo *pInfo) { + if (atomic_load_8(&tsDndNotifyInitLock) != 2) { + if (atomic_val_compare_exchange_8(&tsDndNotifyInitLock, 0, 1) == 0) { + tsDndNotifyInfo = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + + // fetch the latest tsTimeSeries of vgroups in current dnode + atomic_store_8(&tsDndNotifyInitLock, 2); + } else { + int32_t nLoops = 0; + while (atomic_load_8(&tsDndNotifyInitLock) != 2) { + if (++nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + } + } + + if (atomic_val_compare_exchange_8(&tsDndNotifyLock, 0, 1) != 0) { + return 0; + } + + int64_t lastTimeSeries = atomic_load_64(&tsTimeSeries); + + int64_t *val = NULL; + if ((val = taosHashGet(tsDndNotifyInfo, &pInfo->vgId, sizeof(pInfo->vgId)))) { + if (*val != pInfo->nTimeSeries) { + assert(*val > 0); + atomic_add_fetch_64(&tsTimeSeries, pInfo->nTimeSeries - *val); + taosHashPut(tsDndNotifyInfo, &pInfo->vgId, sizeof(pInfo->vgId), &pInfo->nTimeSeries, sizeof(pInfo->nTimeSeries)); + } + } else { + atomic_add_fetch_64(&tsTimeSeries, pInfo->nTimeSeries); + taosHashPut(tsDndNotifyInfo, &pInfo->vgId, sizeof(pInfo->vgId), &pInfo->nTimeSeries, sizeof(pInfo->nTimeSeries)); + } + + if (atomic_load_64(&tsTimeSeries) - lastTimeSeries > 100) { + int32_t hashSize = taosHashGetSize(tsDndNotifyInfo); + SNotifyReq req = {.nVgroup = hashSize}; + req.payload = taosMemoryMalloc(sizeof(SDndNotifyInfo) * hashSize); + + int64_t *nVal = NULL; + int32_t iter = 0; + + int64_t nnn = 0; + size_t kLen = sizeof(int32_t); + while ((nVal = taosHashIterate(tsDndNotifyInfo, nVal)) && iter < hashSize) { + int32_t *nVgId = taosHashGetKey(nVal, &kLen); + (req.payload + iter)->vgId = *nVgId; + (req.payload + iter)->nTimeSeries = *nVal; + ++iter; + } + // assert(nnn < 1000000); + + int32_t contLen = tSerializeSNotifyReq(NULL, 0, &req); + if (contLen <= 0) return -1; + void *pHead = rpcMallocCont(contLen); + if (!pHead) { + tFreeSNotifyReq(&req); + atomic_store_8(&tsDndNotifyLock, 0); + return -1; + } + tSerializeSNotifyReq(pHead, contLen, &req); + tFreeSNotifyReq(&req); + + SRpcMsg rpcMsg = {.pCont = pHead, + .contLen = contLen, + .msgType = TDMT_MND_NOTIFY, + .info.ahandle = NULL, + .info.refId = 0, + .info.noResp = 1}; + + SEpSet epSet = {0}; + dmGetMnodeEpSet(pSDnodeMgmt->pData, &epSet); + rpcSendRequest(pSDnodeMgmt->msgCb.clientRpc, &epSet, &rpcMsg, NULL); + } + atomic_store_8(&tsDndNotifyLock, 0); + return 0; +} +#endif diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index cd692090c4..2f232a995f 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -26,7 +26,7 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { if (dmStartCrashReportThread(pMgmt) != 0) { return -1; } - if (dmStartNotifyThread(pMgmt) != 0) { + if(dmStartNotify(pMgmt) != 0) { return -1; } return 0; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 08816749bf..9ddc64358a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -133,21 +133,6 @@ static void *dmCrashReportThreadFp(void *param) { return NULL; } - -int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { - TdThreadAttr thAttr; - taosThreadAttrInit(&thAttr); - taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (taosThreadCreate(&pMgmt->statusThread, &thAttr, dmStatusThreadFp, pMgmt) != 0) { - dError("failed to create status thread since %s", strerror(errno)); - return -1; - } - - taosThreadAttrDestroy(&thAttr); - tmsgReportStartup("dnode-status", "initialized"); - return 0; -} - int32_t dmStartStatusThread(SDnodeMgmt *pMgmt) { TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); @@ -266,6 +251,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_MND_GRANT: code = dmProcessGrantReq(&pMgmt->pData->clusterId, pMsg); break; + case TDMT_MND_GRANT_NOTIFY: + code = dmProcessGrantNotify(NULL, pMsg); + break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; dGError("msg:%p, not processed in mgmt queue", pMsg); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 42eb89b5fb..3fc3ca4cea 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -178,6 +178,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_NOTIFY, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index ad87bc91c6..8d506563ec 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -92,6 +92,8 @@ typedef void (*GetMnodeLoadsFp)(SMonMloadInfo *pInfo); typedef void (*GetQnodeLoadsFp)(SQnodeLoad *pInfo); typedef int32_t (*ProcessAlterNodeTypeFp)(EDndNodeType ntype, SRpcMsg *pMsg); +int32_t dmProcessNotifyReq(SDndNotifyInfo *pInfo); + typedef struct { int32_t dnodeId; int64_t clusterId; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index ec6d547795..8f9d37187b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -79,6 +79,14 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t opLen, int32_t *pOutValue); +#ifndef TD_ENTERPRISE +static int32_t mndUpdateClusterInfo(SRpcMsg *pReq) { return 0; } +static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { return 0; } +#else +int32_t mndUpdateClusterInfo(SRpcMsg *pReq); +int32_t mndProcessNotifyReq(SRpcMsg *pReq); +#endif + int32_t mndInitDnode(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_DNODE, @@ -96,6 +104,7 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeReq); mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusReq); + mndSetMsgHandle(pMnode, TDMT_MND_NOTIFY, mndProcessNotifyReq); mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); mndSetMsgHandle(pMnode, TDMT_MND_SHOW_VARIABLES, mndProcessShowVariablesReq); mndSetMsgHandle(pMnode, TDMT_MND_RESTORE_DNODE, mndProcessRestoreDnodeReq); @@ -480,12 +489,6 @@ static bool mndUpdateMnodeState(SMnodeObj *pObj, SMnodeLoad *pMload) { return stateChanged; } -#ifndef TD_ENTERPRISE -static int32_t mndUpdateClusterInfo(SRpcMsg *pReq) { return 0; } -#else -int32_t mndUpdateClusterInfo(SRpcMsg *pReq); -#endif - static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStatusReq statusReq = {0}; @@ -537,6 +540,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE); int64_t curMs = taosGetTimestampMs(); + int64_t nDiffTimeSeries = 0; bool online = mndIsDnodeOnline(pDnode, curMs); bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer); bool reboot = (pDnode->rebootTime != statusReq.rebootTime); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 5ae257aef8..9d422d8068 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -288,6 +288,7 @@ typedef struct { int64_t numOfSTables; int64_t numOfCTables; int64_t numOfNTables; + int64_t numOfCmprTimeSeries; int64_t numOfNTimeSeries; int64_t numOfTimeSeries; int64_t itvTimeSeries; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 8a71327919..4102fa4312 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -36,6 +36,7 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); + static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { pInfo->uid = pEntry->uid; pInfo->version = pEntry->version; @@ -736,6 +737,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs metaReaderClear(&mr); // build SMetaEntry + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; me.version = ver; me.type = pReq->type; me.uid = pReq->uid; @@ -769,7 +771,9 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs } #endif - ++pMeta->pVnode->config.vndStats.numOfCTables; + ++pStats->numOfCTables; + + pStats->numOfTimeSeries += 2; // 2 cols for test. metaWLock(pMeta); metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1); @@ -784,11 +788,18 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs me.ntbEntry.schemaRow = pReq->ntb.schemaRow; me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1; - ++pMeta->pVnode->config.vndStats.numOfNTables; - pMeta->pVnode->config.vndStats.numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1; + ++pStats->numOfNTables; + pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1; } if (metaHandleEntry(pMeta, &me) < 0) goto _err; + // assert(pStats->numOfTimeSeries + pStats->numOfNTimeSeries < 200000); + // if (pStats->numOfTimeSeries + pStats->numOfNTimeSeries - pStats->numOfCmprTimeSeries > 100) { + // pStats->numOfCmprTimeSeries = pStats->numOfTimeSeries + pStats->numOfNTimeSeries; + // SDndNotifyInfo dNotifyInfo = {.vgId = pMeta->pVnode->config.vgId, + // .nTimeSeries = pStats->numOfTimeSeries + pStats->numOfNTimeSeries}; + // dmProcessNotifyReq(&dNotifyInfo); + // } atomic_val_compare_exchange_8(&tsNeedUpdStatus, 0, 1); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 16a21ca405..be4941d444 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -388,7 +388,7 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->cacheUsage = tsdbCacheGetUsage(pVnode); pLoad->numOfCachedTables = tsdbCacheGetElems(pVnode); pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); - pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta, 0); + pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta, 1); pLoad->totalStorage = (int64_t)3 * 1073741824; pLoad->compStorage = (int64_t)2 * 1073741824; pLoad->pointsWritten = 100; From 8f828e052ba35e660bb3a758c4740703d40673fb Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 22 Sep 2023 20:07:51 +0800 Subject: [PATCH 09/16] enh: timeseries calculation optimization --- include/common/tglobal.h | 1 + include/common/tmsg.h | 17 ++- include/common/tmsgdef.h | 5 +- include/libs/executor/storageapi.h | 5 +- include/libs/monitor/monitor.h | 2 +- include/os/osThread.h | 2 +- source/common/src/tglobal.c | 5 + source/common/src/tmsg.c | 47 +++++--- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 5 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 126 +++++--------------- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 9 +- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 48 +++++++- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 22 ++++ source/dnode/mgmt/node_mgmt/inc/dmMgmt.h | 1 + source/dnode/mgmt/node_mgmt/inc/dmNodes.h | 1 + source/dnode/mgmt/node_mgmt/src/dmEnv.c | 1 + source/dnode/mgmt/node_mgmt/src/dmMonitor.c | 11 ++ source/dnode/mgmt/node_util/inc/dmUtil.h | 3 +- source/dnode/mnode/impl/src/mndDnode.c | 45 +++++-- source/dnode/vnode/inc/vnode.h | 8 +- source/dnode/vnode/src/inc/meta.h | 2 +- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/meta/metaQuery.c | 52 +++++--- source/dnode/vnode/src/meta/metaTable.c | 50 +++++--- source/dnode/vnode/src/vnd/vnodeQuery.c | 17 ++- source/libs/executor/src/scanoperator.c | 4 +- 26 files changed, 300 insertions(+), 190 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 1c5510ba0d..bef581a31b 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -145,6 +145,7 @@ extern bool tsUseAdapter; extern int32_t tsMetaCacheMaxSize; extern int32_t tsSlowLogThreshold; extern int32_t tsSlowLogScope; +extern int32_t tsTimeSeriesThreshold; // client extern int32_t tsMinSlidingTime; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1217f3f6b4..d1d38ca2f8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1464,6 +1464,11 @@ typedef struct { int32_t learnerProgress; // use one reservered } SVnodeLoad; +typedef struct { + int32_t vgId; + int64_t nTimeSeries; +} SVnodeLoadLite; + typedef struct { int8_t syncState; int64_t syncTerm; @@ -1512,15 +1517,9 @@ int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq); typedef struct { - int32_t vgId; - int64_t nTimeSeries; -} SDndNotifyInfo; - -int32_t dmProcessNotifyReq(SDndNotifyInfo* pInfo); - -typedef struct { - int32_t nVgroup; - SDndNotifyInfo* payload; + int32_t dnodeId; + int64_t clusterId; + SArray* pVloads; } SNotifyReq; int32_t tSerializeSNotifyReq(void* buf, int32_t bufLen, SNotifyReq* pReq); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 39a36a248b..7e030bef3f 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -180,9 +180,6 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_IP_WHITE, "retrieve-ip-white", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_USER_WHITELIST, "get-user-whitelist", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_NOTIFY, "notify", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_BALANCE_VGROUP_LEADER, "balance-vgroup-leader", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_RESTORE_DNODE, "restore-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_PAUSE_STREAM, "pause-stream", NULL, NULL) @@ -191,6 +188,8 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_MSG) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 0a240dd8f5..d21fb8c459 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -142,6 +142,7 @@ typedef struct SSnapContext { typedef struct { int64_t uid; int64_t ctbNum; + int32_t colNum; } SMetaStbStats; // void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); @@ -285,8 +286,8 @@ typedef struct SStoreMeta { // db name, vgId, numOfTables, numOfSTables int32_t (*getNumOfChildTables)( - void* pVnode, int64_t uid, - int64_t* numOfTables); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); + void* pVnode, int64_t uid, int64_t* numOfTables, + int32_t* numOfCols); // int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, int64_t* numOfNormalTables); // vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId) & // metaGetTbNum(SMeta *pMeta) & metaGetNtbNum(SMeta *pMeta); diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 607179c446..708953f45e 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -191,7 +191,7 @@ typedef struct { } SMonBmInfo; typedef struct { - SArray *pVloads; // SVnodeLoad + SArray *pVloads; // SVnodeLoad/SVnodeLoadLite } SMonVloadInfo; typedef struct { diff --git a/include/os/osThread.h b/include/os/osThread.h index 3fdafe2c34..f0b79ac2c9 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -46,7 +46,7 @@ typedef HANDLE TdThreadMutexAttr; // windows api typedef struct { SRWLOCK lock; int8_t excl; -} TdThreadRwlock; // pthread api +} TdThreadRwlock; // windows api typedef pthread_attr_t TdThreadAttr; // pthread api typedef pthread_once_t TdThreadOnce; // pthread api typedef HANDLE TdThreadRwlockAttr; // windows api diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 03285edcf7..251919568f 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -144,6 +144,7 @@ bool tsUseAdapter = false; int32_t tsMetaCacheMaxSize = -1; // MB int32_t tsSlowLogThreshold = 3; // seconds int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL; +int32_t tsTimeSeriesThreshold = 50; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -630,6 +631,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER) != 0) @@ -1036,6 +1038,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsTrimVDbIntervalSec = cfgGetItem(pCfg, "trimVDbIntervalSec")->i32; tsUptimeInterval = cfgGetItem(pCfg, "uptimeInterval")->i32; tsQueryRsmaTolerance = cfgGetItem(pCfg, "queryRsmaTolerance")->i32; + tsTimeSeriesThreshold = cfgGetItem(pCfg, "timeseriesThreshold")->i32; tsWalFsyncDataSizeLimit = cfgGetItem(pCfg, "walFsyncDataSizeLimit")->i64; @@ -1448,6 +1451,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; } else if (strcasecmp("ttlFlushThreshold", name) == 0) { tsTtlFlushThreshold = cfgGetItem(pCfg, "ttlFlushThreshold")->i32; + } else if (strcasecmp("timeseriesThreshold", name) == 0) { + tsTimeSeriesThreshold = cfgGetItem(pCfg, "timeseriesThreshold")->i32; } break; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c805053a5c..e1ea76f463 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1041,10 +1041,15 @@ int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) { if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pReq->nVgroup) < 0) return -1; - for (int32_t i = 0; i < pReq->nVgroup; ++i) { - if (tEncodeI32(&encoder, (pReq->payload + i)->vgId) < 0) return -1; - if (tEncodeI64(&encoder, (pReq->payload + i)->nTimeSeries) < 0) return -1; + if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->clusterId) < 0) return -1; + + int32_t nVgroup = taosArrayGetSize(pReq->pVloads); + if (tEncodeI32(&encoder, nVgroup) < 0) return -1; + for (int32_t i = 0; i < nVgroup; ++i) { + SVnodeLoadLite *vload = TARRAY_GET_ELEM(pReq->pVloads, i); + if (tEncodeI32(&encoder, vload->vgId) < 0) return -1; + if (tEncodeI64(&encoder, vload->nTimeSeries) < 0) return -1; } tEndEncode(&encoder); @@ -1055,29 +1060,39 @@ int32_t tSerializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) { } int32_t tDeserializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) { + int32_t code = TSDB_CODE_INVALID_MSG; SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; - - if (tDecodeI32(&decoder, &pReq->nVgroup) < 0) return -1; - - pReq->payload = taosMemoryMalloc(pReq->nVgroup * (sizeof(SDndNotifyInfo))); - if (!pReq->payload) return -1; - - for (int32_t i = 0; i < pReq->nVgroup; ++i) { - if (tDecodeI32(&decoder, &((pReq->payload + i)->vgId)) < 0) return -1; - if (tDecodeI64(&decoder, &((pReq->payload + i)->nTimeSeries)) < 0) return -1; + if (tStartDecode(&decoder) < 0) goto _exit; + if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) goto _exit; + if (tDecodeI64(&decoder, &pReq->clusterId) < 0) goto _exit; + int32_t nVgroup = 0; + if (tDecodeI32(&decoder, &nVgroup) < 0) goto _exit; + if (nVgroup > 0) { + pReq->pVloads = taosArrayInit(nVgroup, sizeof(SVnodeLoadLite)); + if (!pReq->pVloads) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + for (int32_t i = 0; i < nVgroup; ++i) { + SVnodeLoadLite *vload = TARRAY_GET_ELEM(pReq->pVloads, i); + if (tDecodeI32(&decoder, &(vload->vgId)) < 0) goto _exit; + if (tDecodeI64(&decoder, &(vload->nTimeSeries)) < 0) goto _exit; + } } + code = 0; + +_exit: tEndDecode(&decoder); tDecoderClear(&decoder); - return 0; + return code; } void tFreeSNotifyReq(SNotifyReq *pReq) { if (pReq) { - taosMemoryFreeClear(pReq->payload); + taosArrayDestroy(pReq->pVloads); } } diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 0cf2c4f7b8..93e2513f4a 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -28,6 +28,7 @@ typedef struct SDnodeMgmt { const char *path; const char *name; TdThread statusThread; + TdThread notifyThread; TdThread monitorThread; TdThread crashReportThread; SSingleWorker mgmtWorker; @@ -36,6 +37,7 @@ typedef struct SDnodeMgmt { ProcessDropNodeFp processDropNodeFp; SendMonitorReportFp sendMonitorReportFp; GetVnodeLoadsFp getVnodeLoadsFp; + GetVnodeLoadsFp getVnodeLoadsLiteFp; GetMnodeLoadsFp getMnodeLoadsFp; GetQnodeLoadsFp getQnodeLoadsFp; int32_t statusSeq; @@ -44,6 +46,7 @@ typedef struct SDnodeMgmt { // dmHandle.c SArray *dmGetMsgHandles(); void dmSendStatusReq(SDnodeMgmt *pMgmt); +void dmSendNotifyReq(SDnodeMgmt *pMgmt); int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); @@ -51,11 +54,11 @@ int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg); int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg); -int32_t dmStartNotify(SDnodeMgmt *pMgmt); // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); +int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt); void dmStopMonitorThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index bc53827222..d6a4b76964 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -170,6 +170,36 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { dmProcessStatusRsp(pMgmt, &rpcRsp); } +void dmSendNotifyReq(SDnodeMgmt *pMgmt) { + SNotifyReq req = {0}; + + // taosThreadRwlockRdlock(&pMgmt->pData->lock); + // req.dnodeId = pMgmt->pData->dnodeId; + // taosThreadRwlockUnlock(&pMgmt->pData->lock); + + // req.clusterId = pMgmt->pData->clusterId; + + SMonVloadInfo vinfo = {0}; + (*pMgmt->getVnodeLoadsLiteFp)(&vinfo); + req.pVloads = vinfo.pVloads; + + int32_t contLen = tSerializeSNotifyReq(NULL, 0, &req); + void *pHead = rpcMallocCont(contLen); + tSerializeSNotifyReq(pHead, contLen, &req); + tFreeSNotifyReq(&req); + + SRpcMsg rpcMsg = {.pCont = pHead, + .contLen = contLen, + .msgType = TDMT_MND_NOTIFY, + .info.ahandle = (void *)0x9527, + .info.refId = 0, + .info.noResp = 1}; + + SEpSet epSet = {0}; + dmGetMnodeEpSet(pMgmt->pData, &epSet); + rpcSendRequest(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, NULL); +} + int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { dError("auth rsp is received, but not supported yet"); return 0; @@ -408,99 +438,3 @@ _OVER: return pArray; } } - -#ifndef TD_ENTERPRISE -int32_t dmStartNotify(SDnodeMgmt *pMgmt) { return 0; } -int32_t dmProcessNotifyReq(SDndNotifyInfo *pInfo) { return 0; } -#else -static SHashObj *tsDndNotifyInfo = NULL; -static int64_t tsTimeSeries = 0; -static int8_t tsDndNotifyInitLock = 0; -static int8_t tsDndNotifyLock = 0; -static SDnodeMgmt *pSDnodeMgmt = NULL; - -int32_t dmStartNotify(SDnodeMgmt *pMgmt) { - pSDnodeMgmt = pMgmt; - return 0; -} - -int32_t dmProcessNotifyReq(SDndNotifyInfo *pInfo) { - if (atomic_load_8(&tsDndNotifyInitLock) != 2) { - if (atomic_val_compare_exchange_8(&tsDndNotifyInitLock, 0, 1) == 0) { - tsDndNotifyInfo = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - - // fetch the latest tsTimeSeries of vgroups in current dnode - atomic_store_8(&tsDndNotifyInitLock, 2); - } else { - int32_t nLoops = 0; - while (atomic_load_8(&tsDndNotifyInitLock) != 2) { - if (++nLoops > 1000) { - sched_yield(); - nLoops = 0; - } - } - } - } - - if (atomic_val_compare_exchange_8(&tsDndNotifyLock, 0, 1) != 0) { - return 0; - } - - int64_t lastTimeSeries = atomic_load_64(&tsTimeSeries); - - int64_t *val = NULL; - if ((val = taosHashGet(tsDndNotifyInfo, &pInfo->vgId, sizeof(pInfo->vgId)))) { - if (*val != pInfo->nTimeSeries) { - assert(*val > 0); - atomic_add_fetch_64(&tsTimeSeries, pInfo->nTimeSeries - *val); - taosHashPut(tsDndNotifyInfo, &pInfo->vgId, sizeof(pInfo->vgId), &pInfo->nTimeSeries, sizeof(pInfo->nTimeSeries)); - } - } else { - atomic_add_fetch_64(&tsTimeSeries, pInfo->nTimeSeries); - taosHashPut(tsDndNotifyInfo, &pInfo->vgId, sizeof(pInfo->vgId), &pInfo->nTimeSeries, sizeof(pInfo->nTimeSeries)); - } - - if (atomic_load_64(&tsTimeSeries) - lastTimeSeries > 100) { - int32_t hashSize = taosHashGetSize(tsDndNotifyInfo); - SNotifyReq req = {.nVgroup = hashSize}; - req.payload = taosMemoryMalloc(sizeof(SDndNotifyInfo) * hashSize); - - int64_t *nVal = NULL; - int32_t iter = 0; - - int64_t nnn = 0; - size_t kLen = sizeof(int32_t); - while ((nVal = taosHashIterate(tsDndNotifyInfo, nVal)) && iter < hashSize) { - int32_t *nVgId = taosHashGetKey(nVal, &kLen); - (req.payload + iter)->vgId = *nVgId; - (req.payload + iter)->nTimeSeries = *nVal; - ++iter; - } - // assert(nnn < 1000000); - - int32_t contLen = tSerializeSNotifyReq(NULL, 0, &req); - if (contLen <= 0) return -1; - void *pHead = rpcMallocCont(contLen); - if (!pHead) { - tFreeSNotifyReq(&req); - atomic_store_8(&tsDndNotifyLock, 0); - return -1; - } - tSerializeSNotifyReq(pHead, contLen, &req); - tFreeSNotifyReq(&req); - - SRpcMsg rpcMsg = {.pCont = pHead, - .contLen = contLen, - .msgType = TDMT_MND_NOTIFY, - .info.ahandle = NULL, - .info.refId = 0, - .info.noResp = 1}; - - SEpSet epSet = {0}; - dmGetMnodeEpSet(pSDnodeMgmt->pData, &epSet); - rpcSendRequest(pSDnodeMgmt->msgCb.clientRpc, &epSet, &rpcMsg, NULL); - } - atomic_store_8(&tsDndNotifyLock, 0); - return 0; -} -#endif diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 2f232a995f..1e8ad2f7c2 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -20,15 +20,17 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) { if (dmStartStatusThread(pMgmt) != 0) { return -1; } +#ifdef TD_ENTERPRISE + if (dmStartNotifyThread(pMgmt) != 0) { + return -1; + } +#endif if (dmStartMonitorThread(pMgmt) != 0) { return -1; } if (dmStartCrashReportThread(pMgmt) != 0) { return -1; } - if(dmStartNotify(pMgmt) != 0) { - return -1; - } return 0; } @@ -55,6 +57,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->processDropNodeFp = pInput->processDropNodeFp; pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp; pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp; + pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp; pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp; pMgmt->getQnodeLoadsFp = pInput->getQnodeLoadsFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 9ddc64358a..4cd7a9f951 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -17,7 +17,6 @@ #include "dmInt.h" #include "thttp.h" -int8_t tsNeedUpdStatus = 0; static void *dmStatusThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); @@ -29,13 +28,13 @@ static void *dmStatusThreadFp(void *param) { int64_t upTime = 0; while (1) { - taosMsleep(100); + taosMsleep(200); if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; int64_t curTime = taosGetTimestampMs(); if (curTime < lastTime) lastTime = curTime; float interval = (curTime - lastTime) / 1000.0f; - if (atomic_val_compare_exchange_8(&tsNeedUpdStatus, 1, 0) == 1 || interval >= tsStatusInterval) { + if (interval >= tsStatusInterval) { dmSendStatusReq(pMgmt); lastTime = curTime; @@ -54,6 +53,26 @@ static void *dmStatusThreadFp(void *param) { return NULL; } +tsem_t dmNotifySem; +static void *dmNotifyThreadFp(void *param) { + SDnodeMgmt *pMgmt = param; + int64_t lastTime = taosGetTimestampMs(); + setThreadName("dnode-notify"); + + if (tsem_init(&dmNotifySem, 0, 0) != 0) { + return NULL; + } + + while (1) { + if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; + + tsem_wait(&dmNotifySem); + dmSendNotifyReq(pMgmt); + } + + return NULL; +} + static void *dmMonitorThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); @@ -154,6 +173,29 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) { } } +int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) { + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pMgmt->notifyThread, &thAttr, dmNotifyThreadFp, pMgmt) != 0) { + dError("failed to create notify thread since %s", strerror(errno)); + return -1; + } + + taosThreadAttrDestroy(&thAttr); + tmsgReportStartup("dnode-notify", "initialized"); + return 0; +} + +void dmStopNotifyThread(SDnodeMgmt *pMgmt) { + if (taosCheckPthreadValid(pMgmt->notifyThread)) { + tsem_post(&dmNotifySem); + taosThreadJoin(pMgmt->notifyThread, NULL); + taosThreadClear(&pMgmt->notifyThread); + } + tsem_destroy(&dmNotifySem); +} + int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) { TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 0251b9b636..a13c8fcc57 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -40,6 +40,28 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) { taosThreadRwlockUnlock(&pMgmt->lock); } +void vmGetVnodeLoadsLite(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) { + pInfo->pVloads = taosArrayInit(pMgmt->state.totalVnodes, sizeof(SVnodeLoadLite)); + if (!pInfo->pVloads) return; + + taosThreadRwlockRdlock(&pMgmt->lock); + + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + if (ppVnode == NULL || *ppVnode == NULL) continue; + + SVnodeObj *pVnode = *ppVnode; + SVnodeLoadLite vload = {0}; + if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) { + taosArrayPush(pInfo->pVloads, &vload); + } + pIter = taosHashIterate(pMgmt->hash, pIter); + } + + taosThreadRwlockUnlock(&pMgmt->lock); +} + void vmGetMonitorInfo(SVnodeMgmt *pMgmt, SMonVmInfo *pInfo) { SMonVloadInfo vloads = {0}; vmGetVnodeLoads(pMgmt, &vloads, true); diff --git a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h index 98489433b9..1c6625ba1a 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmMgmt.h @@ -119,6 +119,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); // dmMonitor.c void dmSendMonitorReport(); void dmGetVnodeLoads(SMonVloadInfo *pInfo); +void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo); void dmGetMnodeLoads(SMonMloadInfo *pInfo); void dmGetQnodeLoads(SQnodeLoad *pInfo); diff --git a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h index 5f36309038..7d635c6bdc 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h @@ -35,6 +35,7 @@ void smGetMonitorInfo(void *pMgmt, SMonSmInfo *pInfo); void bmGetMonitorInfo(void *pMgmt, SMonBmInfo *pInfo); void vmGetVnodeLoads(void *pMgmt, SMonVloadInfo *pInfo, bool isReset); +void vmGetVnodeLoadsLite(void *pMgmt, SMonVloadInfo *pInfo); void mmGetMnodeLoads(void *pMgmt, SMonMloadInfo *pInfo); void qmGetQnodeLoads(void *pMgmt, SQnodeLoad *pInfo); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 320c9db37d..8815647047 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -419,6 +419,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { .processDropNodeFp = dmProcessDropNodeReq, .sendMonitorReportFp = dmSendMonitorReport, .getVnodeLoadsFp = dmGetVnodeLoads, + .getVnodeLoadsLiteFp = dmGetVnodeLoadsLite, .getMnodeLoadsFp = dmGetMnodeLoads, .getQnodeLoadsFp = dmGetQnodeLoads, }; diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index 4ab56ed682..b3db7c3058 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -119,6 +119,17 @@ void dmGetVnodeLoads(SMonVloadInfo *pInfo) { } } +void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo) { + SDnode *pDnode = dmInstance(); + SMgmtWrapper *pWrapper = &pDnode->wrappers[VNODE]; + if (dmMarkWrapper(pWrapper) == 0) { + if (pWrapper->pMgmt != NULL) { + vmGetVnodeLoadsLite(pWrapper->pMgmt, pInfo); + } + dmReleaseWrapper(pWrapper); + } +} + void dmGetMnodeLoads(SMonMloadInfo *pInfo) { SDnode *pDnode = dmInstance(); SMgmtWrapper *pWrapper = &pDnode->wrappers[MNODE]; diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 8d506563ec..9d97e6ae9f 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -92,8 +92,6 @@ typedef void (*GetMnodeLoadsFp)(SMonMloadInfo *pInfo); typedef void (*GetQnodeLoadsFp)(SQnodeLoad *pInfo); typedef int32_t (*ProcessAlterNodeTypeFp)(EDndNodeType ntype, SRpcMsg *pMsg); -int32_t dmProcessNotifyReq(SDndNotifyInfo *pInfo); - typedef struct { int32_t dnodeId; int64_t clusterId; @@ -123,6 +121,7 @@ typedef struct { ProcessDropNodeFp processDropNodeFp; SendMonitorReportFp sendMonitorReportFp; GetVnodeLoadsFp getVnodeLoadsFp; + GetVnodeLoadsFp getVnodeLoadsLiteFp; GetMnodeLoadsFp getMnodeLoadsFp; GetQnodeLoadsFp getQnodeLoadsFp; } SMgmtInputOpt; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 8f9d37187b..300bd6fbb9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -70,6 +70,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq); static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp); static int32_t mndProcessStatusReq(SRpcMsg *pReq); +static int32_t mndProcessNotifyReq(SRpcMsg *pReq); static int32_t mndProcessRestoreDnodeReq(SRpcMsg *pReq); static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -80,11 +81,9 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t opLen, int32_t *pOutValue); #ifndef TD_ENTERPRISE -static int32_t mndUpdateClusterInfo(SRpcMsg *pReq) { return 0; } -static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { return 0; } +static int32_t mndUpdClusterInfo(SRpcMsg *pReq) { return 0; } #else -int32_t mndUpdateClusterInfo(SRpcMsg *pReq); -int32_t mndProcessNotifyReq(SRpcMsg *pReq); +int32_t mndUpdClusterInfo(SRpcMsg *pReq); #endif int32_t mndInitDnode(SMnode *pMnode) { @@ -540,7 +539,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE); int64_t curMs = taosGetTimestampMs(); - int64_t nDiffTimeSeries = 0; bool online = mndIsDnodeOnline(pDnode, curMs); bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer); bool reboot = (pDnode->rebootTime != statusReq.rebootTime); @@ -557,7 +555,6 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVload->vgId); if (pVgroup != NULL) { - nDiffTimeSeries = pVload->numOfTimeSeries - pVgroup->numOfTimeSeries; if (pVload->syncState == TAOS_SYNC_STATE_LEADER) { pVgroup->cacheUsage = pVload->cacheUsage; pVgroup->numOfCachedTables = pVload->numOfCachedTables; @@ -686,9 +683,41 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { _OVER: mndReleaseDnode(pMnode, pDnode); taosArrayDestroy(statusReq.pVloads); - if (nDiffTimeSeries > 0) { - mndUpdateClusterInfo(pReq); + mndUpdClusterInfo(pReq); + return code; +} + +static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SNotifyReq notifyReq = {0}; + int32_t code = 0; + + if ((code = tDeserializeSNotifyReq(pReq->pCont, pReq->contLen, ¬ifyReq)) != 0) { + terrno = code; + goto _OVER; } + + // int64_t clusterid = mndGetClusterId(pMnode); + // if (notifyReq.clusterId != 0 && notifyReq.clusterId != clusterid) { + // code = TSDB_CODE_MND_DNODE_DIFF_CLUSTER; + // mWarn("dnode:%d, its clusterid:%" PRId64 " differ from current cluster:%" PRId64 " since %s", notifyReq.dnodeId, + // notifyReq.clusterId, clusterid, tstrerror(code)); + // goto _OVER; + // } + + int32_t nVgroup = taosArrayGetSize(notifyReq.pVloads); + for (int32_t v = 0; v < nVgroup; ++v) { + SVnodeLoadLite *pVload = taosArrayGet(notifyReq.pVloads, v); + + SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVload->vgId); + if (pVgroup != NULL) { + pVgroup->numOfTimeSeries = pVload->nTimeSeries; + mndReleaseVgroup(pMnode, pVgroup); + } + } +_OVER: + mndUpdClusterInfo(pReq); + tFreeSNotifyReq(¬ifyReq); return code; } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 9d422d8068..727d790b91 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -86,11 +86,13 @@ void *vnodeGetIdx(void *pVnode); void *vnodeGetIvtIdx(void *pVnode); int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num); +int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num); int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num); int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num); void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); +int32_t vnodeGetLoadLite(SVnode *pVnode, SVnodeLoadLite *pLoad); int32_t vnodeValidateTableHash(SVnode *pVnode, char *tableFName); int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg); @@ -134,7 +136,7 @@ bool metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type); int32_t metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type); int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); -int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); +int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols); // tsdb typedef struct STsdbReader STsdbReader; @@ -288,10 +290,10 @@ typedef struct { int64_t numOfSTables; int64_t numOfCTables; int64_t numOfNTables; - int64_t numOfCmprTimeSeries; + int64_t numOfReportedTimeSeries; int64_t numOfNTimeSeries; int64_t numOfTimeSeries; - int64_t itvTimeSeries; + // int64_t itvTimeSeries; int64_t pointsWritten; int64_t totalStorage; int64_t compStorage; diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 4f25bf31a2..b57e046ad8 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -71,7 +71,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid); int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo); int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid); int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); -void metaUpdateStbStats(SMeta* pMeta, int64_t uid, int64_t delta); +void metaUpdateStbStats(SMeta* pMeta, int64_t uid, int64_t deltaCtb, int32_t totalCols); int32_t metaUidFilterCacheGet(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, LRUHandle** pHandle); struct SMeta { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 88f0bf890a..2952c863cb 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -171,6 +171,7 @@ int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); int64_t metaGetTimeSeriesNum(SMeta* pMeta, int type); +void metaUpdTimeSeriesNum(SMeta* pMeta); SMCtbCursor* metaOpenCtbCursor(void* pVnode, tb_uid_t uid, int lock); int32_t metaResumeCtbCursor(SMCtbCursor* pCtbCur, int8_t first); void metaPauseCtbCursor(SMCtbCursor* pCtbCur); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index c7b7ce9686..1a9001ce2b 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -696,19 +696,31 @@ int64_t metaGetTbNum(SMeta *pMeta) { return pMeta->pVnode->config.vndStats.numOfCTables + pMeta->pVnode->config.vndStats.numOfNTables; } -// N.B. Called by statusReq per second -int64_t metaGetTimeSeriesNum(SMeta *pMeta, int type) { - // sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column) - if (type || pMeta->pVnode->config.vndStats.numOfTimeSeries <= 0 || - ++pMeta->pVnode->config.vndStats.itvTimeSeries % (60 * 5) == 0) { - int64_t num = 0; - vnodeGetTimeSeriesNum(pMeta->pVnode, &num); - pMeta->pVnode->config.vndStats.numOfTimeSeries = num; +void metaUpdTimeSeriesNum(SMeta *pMeta) { + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; + int64_t nCtbTimeSeries = 0; - pMeta->pVnode->config.vndStats.itvTimeSeries = (TD_VID(pMeta->pVnode) % 100) * 2; + vnodeGetTimeSeriesNum(pMeta->pVnode, &nCtbTimeSeries); + atomic_store_64(&pStats->numOfTimeSeries, nCtbTimeSeries); +} + +static int64_t metaGetTimeSeriesNumImpl(SMeta *pMeta, bool forceUpd) { + // sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column) + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; + if (forceUpd || pStats->numOfTimeSeries < 0) { + metaUpdTimeSeriesNum(pMeta); } - return pMeta->pVnode->config.vndStats.numOfTimeSeries + pMeta->pVnode->config.vndStats.numOfNTimeSeries; + return pStats->numOfTimeSeries + pStats->numOfNTimeSeries; +} + +// type: 1 report timeseries +int64_t metaGetTimeSeriesNum(SMeta *pMeta, int type) { + int64_t nTimeSeries = metaGetTimeSeriesNumImpl(pMeta, false); + if (type == 1) { + atomic_store_64(&pMeta->pVnode->config.vndStats.numOfReportedTimeSeries, nTimeSeries); + } + return nTimeSeries; } typedef struct { @@ -1506,9 +1518,10 @@ _exit: return code; } -int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables) { +int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols) { int32_t code = 0; - *numOfTables = 0; + + if (!numOfTables && !numOfCols) goto _exit; SVnode *pVnodeObj = pVnode; metaRLock(pVnodeObj->pMeta); @@ -1517,7 +1530,8 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables) { SMetaStbStats state = {0}; if (metaStatsCacheGet(pVnodeObj->pMeta, uid, &state) == TSDB_CODE_SUCCESS) { metaULock(pVnodeObj->pMeta); - *numOfTables = state.ctbNum; + if (numOfTables) *numOfTables = state.ctbNum; + if (numOfCols) *numOfCols = state.colNum; goto _exit; } @@ -1526,10 +1540,15 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables) { vnodeGetCtbNum(pVnode, uid, &ctbNum); metaULock(pVnodeObj->pMeta); - *numOfTables = ctbNum; + if (numOfTables) *numOfTables = ctbNum; + + int32_t colNum = 0; + vnodeGetStbColumnNum(pVnode, uid, &colNum); + if (numOfCols) *numOfCols = colNum; state.uid = uid; state.ctbNum = ctbNum; + state.colNum = colNum; // upsert the cache metaWLock(pVnodeObj->pMeta); @@ -1540,11 +1559,12 @@ _exit: return code; } -void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t delta) { +void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t deltaCtb, int32_t totalCols) { SMetaStbStats stats = {0}; if (metaStatsCacheGet(pMeta, uid, &stats) == TSDB_CODE_SUCCESS) { - stats.ctbNum += delta; + stats.ctbNum += deltaCtb; + if (totalCols > 0) stats.colNum = totalCols; metaStatsCacheUpsert(pMeta, &stats); } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 4102fa4312..c10811083f 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -15,7 +15,7 @@ #include "meta.h" -extern int8_t tsNeedUpdStatus; +extern tsem_t dmNotifySem; static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); @@ -194,6 +194,16 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche return 0; } +static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) { +#ifdef TD_ENTERPRISE + int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0); + int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries; + if (abs(deltaTS) > tsTimeSeriesThreshold) { + tsem_post(&dmNotifySem); + } +#endif +} + int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry me = {0}; int kLen = 0; @@ -393,6 +403,8 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // metaStatsCacheDrop(pMeta, nStbEntry.uid); + metaUpdateStbStats(pMeta, pReq->suid, 0, pReq->schemaRow.nCols); + metaULock(pMeta); if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); @@ -773,10 +785,12 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs ++pStats->numOfCTables; - pStats->numOfTimeSeries += 2; // 2 cols for test. + int32_t nCols = 0; + metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); + pStats->numOfTimeSeries = pStats->numOfCTables * (nCols - 1); metaWLock(pMeta); - metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1); + metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); metaUidCacheClear(pMeta, me.ctbEntry.suid); metaTbGroupCacheClear(pMeta, me.ctbEntry.suid); metaULock(pMeta); @@ -793,15 +807,8 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs } if (metaHandleEntry(pMeta, &me) < 0) goto _err; - // assert(pStats->numOfTimeSeries + pStats->numOfNTimeSeries < 200000); - // if (pStats->numOfTimeSeries + pStats->numOfNTimeSeries - pStats->numOfCmprTimeSeries > 100) { - // pStats->numOfCmprTimeSeries = pStats->numOfTimeSeries + pStats->numOfNTimeSeries; - // SDndNotifyInfo dNotifyInfo = {.vgId = pMeta->pVnode->config.vgId, - // .nTimeSeries = pStats->numOfTimeSeries + pStats->numOfNTimeSeries}; - // dmProcessNotifyReq(&dNotifyInfo); - // } - atomic_val_compare_exchange_8(&tsNeedUpdStatus, 0, 1); + metaTimeSeriesNotifyCheck(pMeta); if (pMetaRsp) { *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); @@ -1086,19 +1093,23 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { if (e.type != TSDB_SUPER_TABLE) metaDeleteTtl(pMeta, &e); + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; if (e.type == TSDB_CHILD_TABLE) { tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), pMeta->txn); - --pMeta->pVnode->config.vndStats.numOfCTables; + --pStats->numOfCTables; + int32_t nCols = 0; + metaGetStbStats(pMeta->pVnode, e.ctbEntry.suid, 0, &nCols); + pStats->numOfTimeSeries = pStats->numOfCTables * (nCols - 1); - metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1); + metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1, 0); metaUidCacheClear(pMeta, e.ctbEntry.suid); metaTbGroupCacheClear(pMeta, e.ctbEntry.suid); } else if (e.type == TSDB_NORMAL_TABLE) { // drop schema.db (todo) - --pMeta->pVnode->config.vndStats.numOfNTables; - pMeta->pVnode->config.vndStats.numOfNTimeSeries -= e.ntbEntry.schemaRow.nCols - 1; + --pStats->numOfNTables; + pStats->numOfNTimeSeries -= e.ntbEntry.schemaRow.nCols - 1; } else if (e.type == TSDB_SUPER_TABLE) { tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), pMeta->txn); // drop schema.db (todo) @@ -1106,13 +1117,14 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { metaStatsCacheDrop(pMeta, uid); metaUidCacheClear(pMeta, uid); metaTbGroupCacheClear(pMeta, uid); - --pMeta->pVnode->config.vndStats.numOfSTables; + metaUpdTimeSeriesNum(pMeta); + --pStats->numOfSTables; } - atomic_val_compare_exchange_8(&tsNeedUpdStatus, 0, 1); - metaCacheDrop(pMeta, uid); + metaTimeSeriesNotifyCheck(pMeta); + tDecoderClear(&dc); tdbFree(pData); @@ -1272,6 +1284,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl strcpy(pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].name, pAlterTbReq->colName); ++pMeta->pVnode->config.vndStats.numOfNTimeSeries; + metaTimeSeriesNotifyCheck(pMeta); break; case TSDB_ALTER_TABLE_DROP_COLUMN: if (pColumn == NULL) { @@ -1294,6 +1307,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl pSchema->nCols--; --pMeta->pVnode->config.vndStats.numOfNTimeSeries; + metaTimeSeriesNotifyCheck(pMeta); break; case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: if (pColumn == NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index be4941d444..c8b2a797fa 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -400,6 +400,15 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { return 0; } +int32_t vnodeGetLoadLite(SVnode *pVnode, SVnodeLoadLite *pLoad) { + SSyncState syncState = syncGetState(pVnode->sync); + if (syncState.state == TAOS_SYNC_STATE_LEADER) { + pLoad->vgId = TD_VID(pVnode); + pLoad->nTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta, 1); + return 0; + } + return -1; +} /** * @brief Reset the statistics value by monitor interval * @@ -544,7 +553,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { return TSDB_CODE_SUCCESS; } -static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { +int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 1); if (pSW) { *num = pSW->nCols; @@ -634,10 +643,8 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); int64_t ctbNum = 0; - metaGetStbStats(pVnode, suid, &ctbNum); - - int numOfCols = 0; - vnodeGetStbColumnNum(pVnode, suid, &numOfCols); + int32_t numOfCols = 0; + metaGetStbStats(pVnode, suid, &ctbNum, &numOfCols); *num += ctbNum * (numOfCols - 1); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 474128007a..b95a948937 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3928,7 +3928,7 @@ static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanO pAPI->metaFn.getTableUidByName(pInfo->readHandle.vnode, pSupp->stbNameFilter, &uid); int64_t numOfChildTables = 0; - pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, uid, &numOfChildTables); + pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, uid, &numOfChildTables, NULL); fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, numOfChildTables, pRes); } else { @@ -3979,7 +3979,7 @@ static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, S pRes->info.id.groupId = groupId; int64_t ctbNum = 0; - int32_t code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, stbUid, &ctbNum); + int32_t code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, stbUid, &ctbNum, NULL); fillTableCountScanDataBlock(pSupp, dbName, varDataVal(stbName), ctbNum, pRes); } From a4b92760f09d4eab9ac336abc7603b6a04995134 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 23 Sep 2023 18:10:08 +0800 Subject: [PATCH 10/16] enh: grant notify optimize --- source/common/src/tmsg.c | 7 ++++--- source/dnode/mnode/impl/src/mndDnode.c | 14 +++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index e1ea76f463..a9f83cc21a 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1076,9 +1076,10 @@ int32_t tDeserializeSNotifyReq(void *buf, int32_t bufLen, SNotifyReq *pReq) { goto _exit; } for (int32_t i = 0; i < nVgroup; ++i) { - SVnodeLoadLite *vload = TARRAY_GET_ELEM(pReq->pVloads, i); - if (tDecodeI32(&decoder, &(vload->vgId)) < 0) goto _exit; - if (tDecodeI64(&decoder, &(vload->nTimeSeries)) < 0) goto _exit; + SVnodeLoadLite vload; + if (tDecodeI32(&decoder, &(vload.vgId)) < 0) goto _exit; + if (tDecodeI64(&decoder, &(vload.nTimeSeries)) < 0) goto _exit; + taosArrayPush(pReq->pVloads, &vload); } } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 300bd6fbb9..9c7d8ee90f 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -697,13 +697,13 @@ static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { goto _OVER; } - // int64_t clusterid = mndGetClusterId(pMnode); - // if (notifyReq.clusterId != 0 && notifyReq.clusterId != clusterid) { - // code = TSDB_CODE_MND_DNODE_DIFF_CLUSTER; - // mWarn("dnode:%d, its clusterid:%" PRId64 " differ from current cluster:%" PRId64 " since %s", notifyReq.dnodeId, - // notifyReq.clusterId, clusterid, tstrerror(code)); - // goto _OVER; - // } + int64_t clusterid = mndGetClusterId(pMnode); + if (notifyReq.clusterId != 0 && notifyReq.clusterId != clusterid) { + code = TSDB_CODE_MND_DNODE_DIFF_CLUSTER; + mWarn("dnode:%d, its clusterid:%" PRId64 " differ from current cluster:%" PRId64 " since %s", notifyReq.dnodeId, + notifyReq.clusterId, clusterid, tstrerror(code)); + goto _OVER; + } int32_t nVgroup = taosArrayGetSize(notifyReq.pVloads); for (int32_t v = 0; v < nVgroup; ++v) { From 53a9459c206f4ad70ff205dc87646e1328de2796 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 24 Sep 2023 07:52:27 +0800 Subject: [PATCH 11/16] enh: update timeseries --- include/common/tgrant.h | 2 + include/dnode/mgmt/dnode.h | 1 - source/dnode/mnode/impl/inc/mndGrant.h | 2 - source/dnode/mnode/impl/src/mndDnode.c | 5 ++ source/dnode/mnode/impl/src/mndMain.c | 11 ++-- source/dnode/vnode/src/inc/meta.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 31 +++++---- source/dnode/vnode/src/meta/metaTable.c | 85 +++++++++++++++++-------- source/dnode/vnode/src/vnd/vnodeQuery.c | 2 +- 9 files changed, 88 insertions(+), 53 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 46e09a56b6..31d34add24 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -30,6 +30,8 @@ extern "C" { #define GRANTS_COL_MAX_LEN 196 #endif +#define GRANT_HEART_BEAT_MIN 2 + typedef enum { TSDB_GRANT_ALL, TSDB_GRANT_TIME, diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index c9f2339d11..82823e3f57 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -44,7 +44,6 @@ int32_t dmRun(); */ void dmStop(); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index a036bf6d7e..88f118cb8f 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -22,8 +22,6 @@ #include "mndInt.h" -#define GRANT_HB_INTERVAL 300 // 300 seconds - int32_t mndInitGrant(SMnode * pMnode); void mndCleanupGrant(); void grantParseParameter(); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 9c7d8ee90f..95a8f873cb 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -56,6 +56,7 @@ enum { DND_ADD, DND_DROP, }; + static int32_t mndCreateDefaultDnode(SMnode *pMnode); static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); @@ -550,6 +551,10 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, pDnode->accessTimes, needCheck, online, reboot, dnodeChanged, statusReq.statusSeq); + if (reboot) { + tsGrantHBInterval = GRANT_HEART_BEAT_MIN; + } + for (int32_t v = 0; v < taosArrayGetSize(statusReq.pVloads); ++v) { SVnodeLoad *pVload = taosArrayGet(statusReq.pVloads, v); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index e00af5375d..1c87cde78a 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -262,13 +262,6 @@ static void *mndThreadFp(void *param) { lastTime++; taosMsleep(100); if (mndGetStop(pMnode)) break; - - if (atomic_val_compare_exchange_32(&tsGrantHBInterval, -GRANT_HB_INTERVAL, GRANT_HB_INTERVAL) == - -GRANT_HB_INTERVAL || - (lastTime % (tsGrantHBInterval * 10) == 0)) { - mndPullupGrant(pMnode); - } - if (lastTime % 10 != 0) continue; int64_t sec = lastTime / 10; @@ -300,6 +293,10 @@ static void *mndThreadFp(void *param) { mndPullupTelem(pMnode); } + if (sec % tsGrantHBInterval == 0) { + mndPullupGrant(pMnode); + } + if (sec % tsUptimeInterval == 0) { mndIncreaseUpTime(pMnode); } diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index b57e046ad8..c74ccf6c11 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -71,7 +71,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid); int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo); int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid); int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); -void metaUpdateStbStats(SMeta* pMeta, int64_t uid, int64_t deltaCtb, int32_t totalCols); +void metaUpdateStbStats(SMeta* pMeta, int64_t uid, int64_t deltaCtb, int32_t deltaCol); int32_t metaUidFilterCacheGet(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, LRUHandle** pHandle); struct SMeta { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 1a9001ce2b..c9ce2a51b8 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -697,24 +697,23 @@ int64_t metaGetTbNum(SMeta *pMeta) { } void metaUpdTimeSeriesNum(SMeta *pMeta) { - SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - int64_t nCtbTimeSeries = 0; - - vnodeGetTimeSeriesNum(pMeta->pVnode, &nCtbTimeSeries); - atomic_store_64(&pStats->numOfTimeSeries, nCtbTimeSeries); + int64_t nCtbTimeSeries = 0; + if (vnodeGetTimeSeriesNum(pMeta->pVnode, &nCtbTimeSeries) == 0) { + atomic_store_64(&pMeta->pVnode->config.vndStats.numOfTimeSeries, nCtbTimeSeries); + } } -static int64_t metaGetTimeSeriesNumImpl(SMeta *pMeta, bool forceUpd) { +static FORCE_INLINE metaGetTimeSeriesNumImpl(SMeta *pMeta, bool forceUpd) { // sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column) SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; - if (forceUpd || pStats->numOfTimeSeries < 0) { + if (forceUpd || pStats->numOfTimeSeries <= 0) { metaUpdTimeSeriesNum(pMeta); } return pStats->numOfTimeSeries + pStats->numOfNTimeSeries; } -// type: 1 report timeseries +// type: 1 reported timeseries int64_t metaGetTimeSeriesNum(SMeta *pMeta, int type) { int64_t nTimeSeries = metaGetTimeSeriesNumImpl(pMeta, false); if (type == 1) { @@ -1532,18 +1531,19 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t metaULock(pVnodeObj->pMeta); if (numOfTables) *numOfTables = state.ctbNum; if (numOfCols) *numOfCols = state.colNum; + assert(state.colNum > 0); + assert(state.ctbNum >= 0); goto _exit; } // slow path: search TDB int64_t ctbNum = 0; - vnodeGetCtbNum(pVnode, uid, &ctbNum); - - metaULock(pVnodeObj->pMeta); - if (numOfTables) *numOfTables = ctbNum; - int32_t colNum = 0; + vnodeGetCtbNum(pVnode, uid, &ctbNum); vnodeGetStbColumnNum(pVnode, uid, &colNum); + metaULock(pVnodeObj->pMeta); + + if (numOfTables) *numOfTables = ctbNum; if (numOfCols) *numOfCols = colNum; state.uid = uid; @@ -1559,13 +1559,12 @@ _exit: return code; } -void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t deltaCtb, int32_t totalCols) { +void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t deltaCtb, int32_t deltaCol) { SMetaStbStats stats = {0}; if (metaStatsCacheGet(pMeta, uid, &stats) == TSDB_CODE_SUCCESS) { stats.ctbNum += deltaCtb; - if (totalCols > 0) stats.colNum = totalCols; - + stats.colNum += deltaCol; metaStatsCacheUpsert(pMeta, &stats); } } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index c10811083f..e95c11b0d9 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -28,7 +28,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type); +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid); static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); // opt ins_tables query static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -198,9 +198,7 @@ static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) { #ifdef TD_ENTERPRISE int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0); int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries; - if (abs(deltaTS) > tsTimeSeriesThreshold) { - tsem_post(&dmNotifySem); - } + if (deltaTS > tsTimeSeriesThreshold) tsem_post(&dmNotifySem); #endif } @@ -305,7 +303,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb for (int32_t iChild = 0; iChild < taosArrayGetSize(tbUidList); iChild++) { tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUidList, iChild); - metaDropTableByUid(pMeta, uid, NULL); + metaDropTableByUid(pMeta, uid, NULL, NULL); } // drop super table @@ -319,6 +317,8 @@ _drop_super_table: metaULock(pMeta); + metaUpdTimeSeriesNum(pMeta); + _exit: tdbFree(pKey); tdbFree(pData); @@ -403,7 +403,13 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // metaStatsCacheDrop(pMeta, nStbEntry.uid); - metaUpdateStbStats(pMeta, pReq->suid, 0, pReq->schemaRow.nCols); + int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols; + if (deltaCol != 0) { + metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol); + int64_t ctbNum; + metaGetStbStats(pMeta, pReq->suid, &ctbNum, NULL); + pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol); + } metaULock(pMeta); @@ -784,10 +790,9 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs #endif ++pStats->numOfCTables; - int32_t nCols = 0; metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); - pStats->numOfTimeSeries = pStats->numOfCTables * (nCols - 1); + pStats->numOfTimeSeries += nCols - 1; metaWLock(pMeta); metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); @@ -840,6 +845,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi int nData = 0; int rc = 0; tb_uid_t uid; + tb_uid_t suid; int type; rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); @@ -850,9 +856,19 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi uid = *(tb_uid_t *)pData; metaWLock(pMeta); - metaDropTableByUid(pMeta, uid, &type); + rc = metaDropTableByUid(pMeta, uid, &type, &suid); metaULock(pMeta); + if (rc < 0) goto _exit; + + if (type == TSDB_CHILD_TABLE) { + int32_t nCols = 0; + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; + if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) { + pStats->numOfTimeSeries -= nCols - 1; + } + } + if ((type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE) && tbUids) { taosArrayPush(tbUids, &uid); } @@ -861,20 +877,46 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi *tbUid = uid; } +_exit: tdbFree(pData); - return 0; + return rc; } void metaDropTables(SMeta *pMeta, SArray *tbUids) { if (taosArrayGetSize(tbUids) == 0) return; + int64_t ctbNum = 0; + SSHashObj *suidHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); + metaWLock(pMeta); for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i); - metaDropTableByUid(pMeta, uid, NULL); + tb_uid_t suid = 0; + metaDropTableByUid(pMeta, uid, NULL, &suid); + if (suid != 0 && suidHash) { + int64_t *cnt = tSimpleHashGet(suidHash, &suid, sizeof(tb_uid_t)); + if (cnt) { + ctbNum = *cnt + 1; + } else { + ctbNum = 1; + } + tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &ctbNum, sizeof(int64_t)); + } metaDebug("batch drop table:%" PRId64, uid); } metaULock(pMeta); + + // update timeseries + void *pCtbNum = NULL; + int32_t iter = 0; + while ((pCtbNum = tSimpleHashIterate(suidHash, pCtbNum, &iter))) { + tb_uid_t *pSuid = tSimpleHashGetKey(pCtbNum, NULL); + int32_t nCols = 0; + SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; + if (metaGetStbStats(pMeta->pVnode, *pSuid, NULL, &nCols) == 0) { + pStats->numOfTimeSeries -= *(int64_t *)pCtbNum * (nCols - 1); + } + } } static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { @@ -1009,7 +1051,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); } -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid) { void *pData = NULL; int nData = 0; int rc = 0; @@ -1034,9 +1076,11 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { if (type) *type = e.type; if (e.type == TSDB_CHILD_TABLE) { + if (pSuid) *pSuid = e.ctbEntry.suid; void *tData = NULL; int tLen = 0; + if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) { STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version}; if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) { @@ -1093,23 +1137,18 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { if (e.type != TSDB_SUPER_TABLE) metaDeleteTtl(pMeta, &e); - SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; if (e.type == TSDB_CHILD_TABLE) { tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), pMeta->txn); - --pStats->numOfCTables; - int32_t nCols = 0; - metaGetStbStats(pMeta->pVnode, e.ctbEntry.suid, 0, &nCols); - pStats->numOfTimeSeries = pStats->numOfCTables * (nCols - 1); - + --pMeta->pVnode->config.vndStats.numOfCTables; metaUpdateStbStats(pMeta, e.ctbEntry.suid, -1, 0); metaUidCacheClear(pMeta, e.ctbEntry.suid); metaTbGroupCacheClear(pMeta, e.ctbEntry.suid); } else if (e.type == TSDB_NORMAL_TABLE) { // drop schema.db (todo) - --pStats->numOfNTables; - pStats->numOfNTimeSeries -= e.ntbEntry.schemaRow.nCols - 1; + --pMeta->pVnode->config.vndStats.numOfNTables; + pMeta->pVnode->config.vndStats.numOfNTimeSeries -= e.ntbEntry.schemaRow.nCols - 1; } else if (e.type == TSDB_SUPER_TABLE) { tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), pMeta->txn); // drop schema.db (todo) @@ -1117,14 +1156,11 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { metaStatsCacheDrop(pMeta, uid); metaUidCacheClear(pMeta, uid); metaTbGroupCacheClear(pMeta, uid); - metaUpdTimeSeriesNum(pMeta); - --pStats->numOfSTables; + --pMeta->pVnode->config.vndStats.numOfSTables; } metaCacheDrop(pMeta, uid); - metaTimeSeriesNotifyCheck(pMeta); - tDecoderClear(&dc); tdbFree(pData); @@ -1307,7 +1343,6 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl pSchema->nCols--; --pMeta->pVnode->config.vndStats.numOfNTimeSeries; - metaTimeSeriesNotifyCheck(pMeta); break; case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: if (pColumn == NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index c8b2a797fa..01292f33e4 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -554,7 +554,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { } int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { - SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 1); + SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0); if (pSW) { *num = pSW->nCols; tDeleteSchemaWrapper(pSW); From ff3e36e6796b576ec806f2306c3a99c07ee0879e Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 24 Sep 2023 08:41:45 +0800 Subject: [PATCH 12/16] enh: timeseries calculation --- source/dnode/mnode/impl/src/mndStb.c | 4 ++++ source/dnode/vnode/src/meta/metaQuery.c | 2 +- source/dnode/vnode/src/meta/metaTable.c | 19 ++++++++++++++----- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index d0a242c8ea..c47c4994b7 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1678,6 +1678,10 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray return -1; } + if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) != 0) { + return -1; + } + pNew->numOfColumns = pNew->numOfColumns + ncols; if (mndAllocStbSchemas(pOld, pNew) != 0) { return -1; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index c9ce2a51b8..5401b42923 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -703,7 +703,7 @@ void metaUpdTimeSeriesNum(SMeta *pMeta) { } } -static FORCE_INLINE metaGetTimeSeriesNumImpl(SMeta *pMeta, bool forceUpd) { +static FORCE_INLINE int64_t metaGetTimeSeriesNumImpl(SMeta *pMeta, bool forceUpd) { // sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column) SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; if (forceUpd || pStats->numOfTimeSeries <= 0) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index e95c11b0d9..da84b6e139 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -314,6 +314,8 @@ _drop_super_table: tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, pMeta->txn); tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); + + metaStatsCacheDrop(pMeta, pReq->suid); metaULock(pMeta); @@ -389,6 +391,8 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { nStbEntry.stbEntry.schemaRow = pReq->schemaRow; nStbEntry.stbEntry.schemaTag = pReq->schemaTag; + int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols; + metaWLock(pMeta); // compare two entry if (oStbEntry.stbEntry.schemaRow.version != pReq->schemaRow.version) { @@ -403,21 +407,23 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // metaStatsCacheDrop(pMeta, nStbEntry.uid); - int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols; if (deltaCol != 0) { metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol); + } + metaULock(pMeta); + + if (deltaCol != 0) { int64_t ctbNum; - metaGetStbStats(pMeta, pReq->suid, &ctbNum, NULL); + metaGetStbStats(pMeta->pVnode, pReq->suid, &ctbNum, NULL); pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol); } - metaULock(pMeta); - +_exit: if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); tDecoderClear(&dc); tdbTbcClose(pTbDbc); tdbTbcClose(pUidIdxc); - return 0; + return ret; } int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; @@ -1308,6 +1314,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; goto _err; } + if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { + goto _err; + } pSchema->version++; pSchema->nCols++; pNewSchema = taosMemoryMalloc(sizeof(SSchema) * pSchema->nCols); From e50b8a3a0da7579d5e9b583ddb2cb6b0142b327b Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 24 Sep 2023 08:48:14 +0800 Subject: [PATCH 13/16] chore: make jenkins happy for community --- source/dnode/mnode/impl/src/mndDnode.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 95a8f873cb..bf6c90ead1 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -688,7 +688,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { _OVER: mndReleaseDnode(pMnode, pDnode); taosArrayDestroy(statusReq.pVloads); +#ifdef MAKE_JENKINS_HAPPY mndUpdClusterInfo(pReq); +#endif return code; } @@ -721,7 +723,9 @@ static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { } } _OVER: +#ifdef ENTERPIRSE_UPDATED mndUpdClusterInfo(pReq); +#endif tFreeSNotifyReq(¬ifyReq); return code; } From 986788d4fe8ce414e5187adb37d22d150bb20aee Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 24 Sep 2023 09:14:52 +0800 Subject: [PATCH 14/16] enh: timeseries calculation --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 16 +++++----- source/dnode/vnode/src/meta/metaQuery.c | 4 +-- source/dnode/vnode/src/meta/metaTable.c | 34 +++++++++++---------- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index d6a4b76964..c7af552da4 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -130,11 +130,11 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { taosThreadRwlockUnlock(&pMgmt->pData->lock); SMonVloadInfo vinfo = {0}; - (*pMgmt->getVnodeLoadsFp)(&vinfo); + (*pMgmt->getVnodeLoadsFp)(&vinfo); req.pVloads = vinfo.pVloads; SMonMloadInfo minfo = {0}; - (*pMgmt->getMnodeLoadsFp)(&minfo); + (*pMgmt->getMnodeLoadsFp)(&minfo); req.mload = minfo.load; (*pMgmt->getQnodeLoadsFp)(&req.qload); @@ -173,11 +173,11 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { void dmSendNotifyReq(SDnodeMgmt *pMgmt) { SNotifyReq req = {0}; - // taosThreadRwlockRdlock(&pMgmt->pData->lock); - // req.dnodeId = pMgmt->pData->dnodeId; - // taosThreadRwlockUnlock(&pMgmt->pData->lock); + taosThreadRwlockRdlock(&pMgmt->pData->lock); + req.dnodeId = pMgmt->pData->dnodeId; + taosThreadRwlockUnlock(&pMgmt->pData->lock); - // req.clusterId = pMgmt->pData->clusterId; + req.clusterId = pMgmt->pData->clusterId; SMonVloadInfo vinfo = {0}; (*pMgmt->getVnodeLoadsLiteFp)(&vinfo); @@ -228,7 +228,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SServerStatusRsp statusRsp = {0}; SMonMloadInfo minfo = {0}; - (*pMgmt->getMnodeLoadsFp)(&minfo); + (*pMgmt->getMnodeLoadsFp)(&minfo); if (minfo.isMnode && (minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; @@ -237,7 +237,7 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { } SMonVloadInfo vinfo = {0}; - (*pMgmt->getVnodeLoadsFp)(&vinfo); + (*pMgmt->getVnodeLoadsFp)(&vinfo); for (int32_t i = 0; i < taosArrayGetSize(vinfo.pVloads); ++i) { SVnodeLoad *pLoad = taosArrayGet(vinfo.pVloads, i); if (pLoad->syncState == TAOS_SYNC_STATE_ERROR || pLoad->syncState == TAOS_SYNC_STATE_OFFLINE) { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 5401b42923..2bf73198aa 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1531,8 +1531,8 @@ int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t metaULock(pVnodeObj->pMeta); if (numOfTables) *numOfTables = state.ctbNum; if (numOfCols) *numOfCols = state.colNum; - assert(state.colNum > 0); - assert(state.ctbNum >= 0); + ASSERTS(state.colNum > 0, "vgId:%d, suid:%" PRIi64 " nCols:%d <= 0 in metaCache", TD_VID(pVnodeObj), uid, + state.colNum); goto _exit; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index da84b6e139..0a25bdc9c7 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -314,7 +314,7 @@ _drop_super_table: tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, pMeta->txn); tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); - + metaStatsCacheDrop(pMeta, pReq->suid); metaULock(pMeta); @@ -423,7 +423,7 @@ _exit: tDecoderClear(&dc); tdbTbcClose(pTbDbc); tdbTbcClose(pUidIdxc); - return ret; + return 0; } int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry oStbEntry = {0}; @@ -850,8 +850,8 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi void *pData = NULL; int nData = 0; int rc = 0; - tb_uid_t uid; - tb_uid_t suid; + tb_uid_t uid = 0; + tb_uid_t suid = 0; int type; rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); @@ -891,38 +891,40 @@ _exit: void metaDropTables(SMeta *pMeta, SArray *tbUids) { if (taosArrayGetSize(tbUids) == 0) return; - int64_t ctbNum = 0; + int64_t nCtbDropped = 0; SSHashObj *suidHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); metaWLock(pMeta); for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i); tb_uid_t suid = 0; - metaDropTableByUid(pMeta, uid, NULL, &suid); - if (suid != 0 && suidHash) { - int64_t *cnt = tSimpleHashGet(suidHash, &suid, sizeof(tb_uid_t)); - if (cnt) { - ctbNum = *cnt + 1; + int type; + metaDropTableByUid(pMeta, uid, &type, &suid); + if (type == TSDB_CHILD_TABLE && suid != 0 && suidHash) { + int64_t *pVal = tSimpleHashGet(suidHash, &suid, sizeof(tb_uid_t)); + if (pVal) { + nCtbDropped = *pVal + 1; } else { - ctbNum = 1; + nCtbDropped = 1; } - tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &ctbNum, sizeof(int64_t)); + tSimpleHashPut(suidHash, &suid, sizeof(tb_uid_t), &nCtbDropped, sizeof(int64_t)); } metaDebug("batch drop table:%" PRId64, uid); } metaULock(pMeta); // update timeseries - void *pCtbNum = NULL; + void *pCtbDropped = NULL; int32_t iter = 0; - while ((pCtbNum = tSimpleHashIterate(suidHash, pCtbNum, &iter))) { - tb_uid_t *pSuid = tSimpleHashGetKey(pCtbNum, NULL); + while ((pCtbDropped = tSimpleHashIterate(suidHash, pCtbDropped, &iter))) { + tb_uid_t *pSuid = tSimpleHashGetKey(pCtbDropped, NULL); int32_t nCols = 0; SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; if (metaGetStbStats(pMeta->pVnode, *pSuid, NULL, &nCols) == 0) { - pStats->numOfTimeSeries -= *(int64_t *)pCtbNum * (nCols - 1); + pStats->numOfTimeSeries -= *(int64_t *)pCtbDropped * (nCols - 1); } } + tSimpleHashCleanup(suidHash); } static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) { From c69b9e0197046cbdf1c512159c7a4477e81b34ef Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 24 Sep 2023 09:45:19 +0800 Subject: [PATCH 15/16] chore: make jenkins happy --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 2 ++ source/dnode/mnode/impl/src/mndDnode.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 4cd7a9f951..5b2bd0783a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -294,7 +294,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { code = dmProcessGrantReq(&pMgmt->pData->clusterId, pMsg); break; case TDMT_MND_GRANT_NOTIFY: +#ifdef MAKE_JENKINS_HAPPY code = dmProcessGrantNotify(NULL, pMsg); +#endif break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index bf6c90ead1..a2c7f7b5cf 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -723,7 +723,7 @@ static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { } } _OVER: -#ifdef ENTERPIRSE_UPDATED +#ifdef MAKE_JENKINS_HAPPY mndUpdClusterInfo(pReq); #endif tFreeSNotifyReq(¬ifyReq); From 7b4ef6df9624ac895eadfbf19885a2c42dc48505 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 24 Sep 2023 13:50:20 +0800 Subject: [PATCH 16/16] fix: memory leak --- source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 3 ++- source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 93e2513f4a..9e43c2af47 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -58,8 +58,9 @@ int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg); // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); -int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); +int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt); +void dmStopNotifyThread(SDnodeMgmt *pMgmt); int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt); void dmStopMonitorThread(SDnodeMgmt *pMgmt); int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 1e8ad2f7c2..966891feb8 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -38,6 +38,7 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) { pMgmt->pData->stopped = true; dmStopMonitorThread(pMgmt); dmStopStatusThread(pMgmt); + dmStopNotifyThread(pMgmt); dmStopCrashReportThread(pMgmt); }