From b1a0912a614dd07a3ea343d3b21de67f2ca70e5d Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 28 Sep 2023 10:15:34 +0800 Subject: [PATCH 01/12] enh: exclude systables in log/audit db --- source/dnode/vnode/inc/vnode.h | 5 +++-- source/dnode/vnode/src/meta/metaCache.c | 24 ++++++++++++++++++++---- source/dnode/vnode/src/meta/metaTable.c | 15 +++++++++++---- source/dnode/vnode/src/vnd/vnodeQuery.c | 17 +++++++++++++---- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 85d840ccf5..d6491c8fc7 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -89,6 +89,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num); int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num); int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num); int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num); +bool vnodeSkipTimeSeries(SVnode *pVnode, const char *stbName); void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); @@ -132,8 +133,8 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen); -bool metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type); -int32_t metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type); +bool metaTbInFilterCache(void *pVnode, void* key, int8_t type); +int32_t metaPutTbToFilterCache(void *pVnode, void* key, int8_t type); int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 6918634b5d..0a8c1c410a 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -69,6 +69,7 @@ struct SMetaCache { struct STbFilterCache { SHashObj* pStb; + SHashObj* pStbName; } STbFilterCache; }; @@ -178,6 +179,12 @@ int32_t metaCacheOpen(SMeta* pMeta) { goto _err2; } + pCache->STbFilterCache.pStbName = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); + if (pCache->STbFilterCache.pStbName == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err2; + } + pMeta->pCache = pCache; return code; @@ -204,6 +211,7 @@ void metaCacheClose(SMeta* pMeta) { taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry); taosHashCleanup(pMeta->pCache->STbFilterCache.pStb); + taosHashCleanup(pMeta->pCache->STbFilterCache.pStbName); taosMemoryFree(pMeta->pCache); pMeta->pCache = NULL; @@ -893,21 +901,29 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { return TSDB_CODE_SUCCESS; } -bool metaTbInFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { +bool metaTbInFilterCache(void* pVnode, void* key, int8_t type) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; - if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid))) { + if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) { + return true; + } + + if (type == 1 && taosHashGet(pMeta->pCache->STbFilterCache.pStbName, key, strlen(key))) { return true; } return false; } -int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { +int32_t metaPutTbToFilterCache(void* pVnode, void* key, int8_t type) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; if (type == 0) { - return taosHashPut(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid), NULL, 0); + return taosHashPut(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t), NULL, 0); + } + + if (type == 1) { + return taosHashPut(pMeta->pCache->STbFilterCache.pStbName, key, strlen(key), NULL, 0); } return 0; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index f065fe3268..112b7bd5c2 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -392,6 +392,10 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { nStbEntry.stbEntry.schemaTag = pReq->schemaTag; int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols; + bool skipStatis = false; + if (deltaCol != 0) { + + } metaWLock(pMeta); // compare two entry @@ -796,9 +800,12 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs #endif ++pStats->numOfCTables; - int32_t nCols = 0; - metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); - pStats->numOfTimeSeries += nCols - 1; + + if (metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1)) { + int32_t nCols = 0; + metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); + pStats->numOfTimeSeries += nCols - 1; + } metaWLock(pMeta); metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0); @@ -1059,7 +1066,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); } -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid) { +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, char* stbName) { void *pData = NULL; int nData = 0; int rc = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 01292f33e4..c96428d835 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -566,7 +566,7 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { } #ifdef TD_ENTERPRISE -#define TK_LOG_STB_NUM 19 +#define TK_LOG_STB_NUM 20 static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "data_dir", "dnodes_info", @@ -585,7 +585,8 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "taosadapter_system_mem_percent", "temp_dir", "vgroups_info", - "vnodes_role"}; + "vnodes_role", + "operations"}; // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { @@ -598,7 +599,7 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]); if (suid != 0) { - metaPutTbToFilterCache(pVnode, suid, 0); + metaPutTbToFilterCache(pVnode, &suid, 0); } } tbSize = metaSizeOfTbFilterCache(pVnode, 0); @@ -606,12 +607,20 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { return tbSize; } + +bool vnodeSkipTimeSeries(SVnode *pVnode, const char *stbName) { + char *dbName = strchr(pVnode->config.dbname, '.'); + if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + return 0; + } + +} #endif static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { SVnode *pVnode = (SVnode *)arg1; - if (metaTbInFilterCache(pVnode, *(tb_uid_t *)(arg2), 0)) { + if (metaTbInFilterCache(pVnode, arg2, 0)) { return true; } return false; From 086213a6c201ae70c3df6f42826716a7a96ec0c0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 21:35:57 +0800 Subject: [PATCH 02/12] enh: dnode notify logic --- include/libs/monitor/monitor.h | 5 +++ source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 19 ++++++---- source/dnode/vnode/src/meta/metaTable.c | 10 ++++-- source/dnode/vnode/src/vnd/vnodeQuery.c | 39 ++++++++++++++------- 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 708953f45e..91b3a54ea1 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -206,6 +206,11 @@ typedef struct { bool comp; } SMonCfg; +typedef struct { + int8_t state; + tsem_t sem; +} SDmNotifyHandle; + int32_t monInit(const SMonCfg *pCfg); void monCleanup(); void monRecordLog(int64_t ts, ELogLevel level, const char *content); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index f567267ff8..598f449738 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -53,21 +53,28 @@ static void *dmStatusThreadFp(void *param) { return NULL; } -tsem_t dmNotifySem; -static void *dmNotifyThreadFp(void *param) { +SDmNotifyHandle dmNotifyHdl = {.state = 0}; +static void *dmNotifyThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-notify"); - if (tsem_init(&dmNotifySem, 0, 0) != 0) { + if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { return NULL; } while (1) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; - tsem_wait(&dmNotifySem); + _wait: + tsem_wait(&dmNotifyHdl.sem); + _send: + atomic_store_8(&dmNotifyHdl.state, 1); dmSendNotifyReq(pMgmt); + if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { + goto _wait; + } + goto _send; } return NULL; @@ -189,11 +196,11 @@ int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) { void dmStopNotifyThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->notifyThread)) { - tsem_post(&dmNotifySem); + tsem_post(&dmNotifyHdl.sem); taosThreadJoin(pMgmt->notifyThread, NULL); taosThreadClear(&pMgmt->notifyThread); } - tsem_destroy(&dmNotifySem); + tsem_destroy(&dmNotifyHdl.sem); } int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 112b7bd5c2..cbbee06e34 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -15,7 +15,7 @@ #include "meta.h" -extern tsem_t dmNotifySem; +extern SDmNotifyHandle dmNotifyHdl; static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema); @@ -198,7 +198,11 @@ static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) { #if defined(TD_ENTERPRISE) && !defined(_TD_DARWIN_64) int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0); int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries; - if (deltaTS > tsTimeSeriesThreshold) tsem_post(&dmNotifySem); + if (deltaTS > tsTimeSeriesThreshold) { + if (1 != atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) { + tsem_post(&dmNotifyHdl.sem); + } + } #endif } @@ -1066,7 +1070,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); } -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, char* stbName) { +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid) { //}, char* stbName) { void *pData = NULL; int nData = 0; int rc = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index c96428d835..2a4415ab8a 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -566,7 +566,8 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { } #ifdef TD_ENTERPRISE -#define TK_LOG_STB_NUM 20 +#define TK_LOG_STB_NUM 19 +#define TK_AUDIT_STB_NUM 1 static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "data_dir", "dnodes_info", @@ -585,24 +586,38 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "taosadapter_system_mem_percent", "temp_dir", "vgroups_info", - "vnodes_role", - "operations"}; + "vnodes_role"}; +static const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"}; // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { char *dbName = strchr(pVnode->config.dbname, '.'); - if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + if (!dbName) { return 0; } - int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0); - if (tbSize < TK_LOG_STB_NUM) { - for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { - tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]); - if (suid != 0) { - metaPutTbToFilterCache(pVnode, &suid, 0); - } - } + int32_t tbSize = 0; + if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { tbSize = metaSizeOfTbFilterCache(pVnode, 0); + if (tbSize < TK_LOG_STB_NUM) { + for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { + tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]); + if (suid != 0) { + metaPutTbToFilterCache(pVnode, &suid, 0); + } + } + tbSize = metaSizeOfTbFilterCache(pVnode, 0); + } + } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { + tbSize = metaSizeOfTbFilterCache(pVnode, 0); + if (tbSize < TK_AUDIT_STB_NUM) { + for (int32_t i = 0; i < TK_AUDIT_STB_NUM; ++i) { + tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkAuditStb[i]); + if (suid != 0) { + metaPutTbToFilterCache(pVnode, &suid, 0); + } + } + tbSize = metaSizeOfTbFilterCache(pVnode, 0); + } } return tbSize; From 88fdeca5cf0ddb129fb0a095307e3aa4a24b76a9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 22:31:25 +0800 Subject: [PATCH 03/12] enh: dnode notify logic --- source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 12 +++++------- source/dnode/vnode/src/meta/metaTable.c | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 598f449738..18da1d638c 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -56,25 +56,23 @@ static void *dmStatusThreadFp(void *param) { SDmNotifyHandle dmNotifyHdl = {.state = 0}; static void *dmNotifyThreadFp(void *param) { SDnodeMgmt *pMgmt = param; - int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-notify"); if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) { return NULL; } + bool wait = true; while (1) { if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; - - _wait: - tsem_wait(&dmNotifyHdl.sem); - _send: + if (wait) tsem_wait(&dmNotifyHdl.sem); atomic_store_8(&dmNotifyHdl.state, 1); dmSendNotifyReq(pMgmt); if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) { - goto _wait; + wait = true; + continue; } - goto _send; + wait = false; } return NULL; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index cbbee06e34..16bffce784 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -199,7 +199,7 @@ static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) { int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0); int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries; if (deltaTS > tsTimeSeriesThreshold) { - if (1 != atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) { + if (0 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) { tsem_post(&dmNotifyHdl.sem); } } From d0a046c7a354cc9985173cb6c7570386b886c996 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 23:30:30 +0800 Subject: [PATCH 04/12] enh: exclude sys tbl for time series --- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/meta/metaCache.c | 6 +++++- source/dnode/vnode/src/meta/metaTable.c | 19 ++++++++++-------- source/dnode/vnode/src/vnd/vnodeQuery.c | 26 ++++++++++++++++++++++++- 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index d6491c8fc7..1729ebb439 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -136,6 +136,7 @@ int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, in bool metaTbInFilterCache(void *pVnode, void* key, int8_t type); int32_t metaPutTbToFilterCache(void *pVnode, void* key, int8_t type); int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); +int32_t metaInitTbFilterCache(void *pVnode); int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 0a8c1c410a..a4a893c42d 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -179,11 +179,15 @@ int32_t metaCacheOpen(SMeta* pMeta) { goto _err2; } - pCache->STbFilterCache.pStbName = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); + pCache->STbFilterCache.pStbName = + taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); if (pCache->STbFilterCache.pStbName == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err2; } + if ((code = metaInitTbFilterCache(pMeta->pVnode)) != 0) { + goto _err2; + } pMeta->pCache = pCache; return code; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 16bffce784..e0727233a9 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -28,7 +28,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid); +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl); static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey); // opt ins_tables query static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); @@ -307,7 +307,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb for (int32_t iChild = 0; iChild < taosArrayGetSize(tbUidList); iChild++) { tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUidList, iChild); - metaDropTableByUid(pMeta, uid, NULL, NULL); + metaDropTableByUid(pMeta, uid, NULL, NULL, NULL); } // drop super table @@ -863,6 +863,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi int rc = 0; tb_uid_t uid = 0; tb_uid_t suid = 0; + int8_t sysTbl = 0; int type; rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); @@ -873,12 +874,12 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi uid = *(tb_uid_t *)pData; metaWLock(pMeta); - rc = metaDropTableByUid(pMeta, uid, &type, &suid); + rc = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); metaULock(pMeta); if (rc < 0) goto _exit; - if (type == TSDB_CHILD_TABLE) { + if (type == TSDB_CHILD_TABLE && !sysTbl) { int32_t nCols = 0; SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) { @@ -909,9 +910,10 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) { for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i); tb_uid_t suid = 0; + int8_t sysTbl = 0; int type; - metaDropTableByUid(pMeta, uid, &type, &suid); - if (type == TSDB_CHILD_TABLE && suid != 0 && suidHash) { + metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl); + if (!sysTbl && type == TSDB_CHILD_TABLE && suid != 0 && suidHash) { int64_t *pVal = tSimpleHashGet(suidHash, &suid, sizeof(tb_uid_t)); if (pVal) { nCtbDropped = *pVal + 1; @@ -1070,7 +1072,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); } -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid) { //}, char* stbName) { +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t* pSysTbl) { void *pData = NULL; int nData = 0; int rc = 0; @@ -1099,7 +1101,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p void *tData = NULL; int tLen = 0; - if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) { STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version}; if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) { @@ -1109,6 +1110,8 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p tDecoderInit(&tdc, tData, tLen); metaDecodeEntry(&tdc, &stbEntry); + if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta->pVnode, stbEntry.name, 1) ? 1 : 0; + SSchema *pTagColumn = NULL; SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 2a4415ab8a..e9307e6fb5 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -596,7 +596,8 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { return 0; } int32_t tbSize = 0; - if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + ++dbName; + if (0 == strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { tbSize = metaSizeOfTbFilterCache(pVnode, 0); if (tbSize < TK_LOG_STB_NUM) { for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { @@ -630,8 +631,31 @@ bool vnodeSkipTimeSeries(SVnode *pVnode, const char *stbName) { } } + +int32_t metaInitTbFilterCache(void *pVnode) { + char *dbName = strchr(pVnode->config.dbname, '.'); + if (!dbName) return 0; + ++dbName; + if (0 == strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { + for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { + if (metaPutTbToFilterCache(pVnode, &tkLogStb[i], strlen(tkLogStb[i])) != 0) { + return terrno ? terrno : -1; + } + } + } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { + for (int32_t i = 0; i < TK_AUDIT_STB_NUM; ++i) { + if (metaPutTbToFilterCache(pVnode, &tkLogStb[i], strlen(tkAuditStb[i])) != 0) { + return terrno ? terrno : -1; + } + } + } + + return 0; +} #endif +int32_t metaInitTbFilterCache(void *pVnode) { return 0; } + static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { SVnode *pVnode = (SVnode *)arg1; From 47e79ff5b7f30f1fb74e421c14e538c4ab06e81a Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 23:45:54 +0800 Subject: [PATCH 05/12] enh: exclude sys tbl for time series --- source/dnode/vnode/src/meta/metaTable.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index e0727233a9..0ceeaffa05 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -396,10 +396,6 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { nStbEntry.stbEntry.schemaTag = pReq->schemaTag; int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols; - bool skipStatis = false; - if (deltaCol != 0) { - - } metaWLock(pMeta); // compare two entry @@ -415,7 +411,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // metaStatsCacheDrop(pMeta, nStbEntry.uid); - if (deltaCol != 0) { + if (deltaCol != 0 && !metaTbInFilterCache(pMeta->pVnode, pReq->name, 1)) { metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol); } metaULock(pMeta); @@ -805,7 +801,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs ++pStats->numOfCTables; - if (metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1)) { + if (!metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1)) { int32_t nCols = 0; metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); pStats->numOfTimeSeries += nCols - 1; @@ -879,7 +875,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi if (rc < 0) goto _exit; - if (type == TSDB_CHILD_TABLE && !sysTbl) { + if (!sysTbl && type == TSDB_CHILD_TABLE) { int32_t nCols = 0; SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) { From 07f043f143fa3792a8885e5aaf51389d18245126 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 23:48:52 +0800 Subject: [PATCH 06/12] enh: exclude sys tbl for time series --- source/dnode/vnode/src/vnd/vnodeQuery.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index e9307e6fb5..e7436aa4dd 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -652,9 +652,9 @@ int32_t metaInitTbFilterCache(void *pVnode) { return 0; } -#endif - +#else int32_t metaInitTbFilterCache(void *pVnode) { return 0; } +#endif static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { SVnode *pVnode = (SVnode *)arg1; @@ -674,9 +674,9 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { } int32_t tbFilterSize = 0; - #ifdef TD_ENTERPRISE +#ifdef TD_ENTERPRISE tbFilterSize = vnodeGetTimeSeriesBlackList(pVnode); - #endif +#endif if ((!tbFilterSize && vnodeGetStbIdList(pVnode, 0, suidList) < 0) || (tbFilterSize && vnodeGetStbIdListByFilter(pVnode, 0, suidList, vnodeTimeSeriesFilter, pVnode) < 0)) { From e4a86aa70aafe9637f302544034cac4ae4948b36 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 5 Oct 2023 23:52:50 +0800 Subject: [PATCH 07/12] chore: remove obsolete codes --- source/dnode/vnode/inc/vnode.h | 1 - source/dnode/vnode/src/vnd/vnodeQuery.c | 8 -------- 2 files changed, 9 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 1729ebb439..ce57ef3bbd 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -89,7 +89,6 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num); int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num); int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num); int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num); -bool vnodeSkipTimeSeries(SVnode *pVnode, const char *stbName); void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index e7436aa4dd..85c4897914 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -624,14 +624,6 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { return tbSize; } -bool vnodeSkipTimeSeries(SVnode *pVnode, const char *stbName) { - char *dbName = strchr(pVnode->config.dbname, '.'); - if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { - return 0; - } - -} - int32_t metaInitTbFilterCache(void *pVnode) { char *dbName = strchr(pVnode->config.dbname, '.'); if (!dbName) return 0; From c59498d90a17a83a90a91707f3fd7d4b6b760bca Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 6 Oct 2023 07:37:18 +0800 Subject: [PATCH 08/12] chore: code refactor for time series statis --- source/dnode/vnode/src/vnd/vnodeQuery.c | 61 ++++++++++++------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 85c4897914..3a8ab824a1 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -591,28 +591,24 @@ static const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"}; // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { - char *dbName = strchr(pVnode->config.dbname, '.'); - if (!dbName) { - return 0; - } - int32_t tbSize = 0; - ++dbName; - if (0 == strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { - tbSize = metaSizeOfTbFilterCache(pVnode, 0); - if (tbSize < TK_LOG_STB_NUM) { - for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { - tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]); - if (suid != 0) { - metaPutTbToFilterCache(pVnode, &suid, 0); - } - } - tbSize = metaSizeOfTbFilterCache(pVnode, 0); - } + int32_t tbSize = 0; + int32_t tbNum = 0; + const char **pTbArr = NULL; + const char *dbName = NULL; + + if (!(dbName = strchr(((SVnode *)pVnode)->config.dbname, '.'))) return 0; + if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + tbNum = TK_LOG_STB_NUM; + pTbArr = (const char **)&tkLogStb; } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { + tbNum = TK_AUDIT_STB_NUM; + pTbArr = (const char **)&tkAuditStb; + } + if (tbNum && pTbArr) { tbSize = metaSizeOfTbFilterCache(pVnode, 0); - if (tbSize < TK_AUDIT_STB_NUM) { - for (int32_t i = 0; i < TK_AUDIT_STB_NUM; ++i) { - tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkAuditStb[i]); + if (tbSize < tbNum) { + for (int32_t i = 0; i < tbNum; ++i) { + tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, pTbArr[i]); if (suid != 0) { metaPutTbToFilterCache(pVnode, &suid, 0); } @@ -625,18 +621,21 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { } int32_t metaInitTbFilterCache(void *pVnode) { - char *dbName = strchr(pVnode->config.dbname, '.'); - if (!dbName) return 0; - ++dbName; - if (0 == strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { - for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { - if (metaPutTbToFilterCache(pVnode, &tkLogStb[i], strlen(tkLogStb[i])) != 0) { - return terrno ? terrno : -1; - } - } + int32_t tbNum = 0; + const char **pTbArr = NULL; + const char *dbName = NULL; + + if (!(dbName = strchr(((SVnode *)pVnode)->config.dbname, '.'))) return 0; + if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + tbNum = TK_LOG_STB_NUM; + pTbArr = (const char **)&tkLogStb; } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { - for (int32_t i = 0; i < TK_AUDIT_STB_NUM; ++i) { - if (metaPutTbToFilterCache(pVnode, &tkLogStb[i], strlen(tkAuditStb[i])) != 0) { + tbNum = TK_AUDIT_STB_NUM; + pTbArr = (const char **)&tkAuditStb; + } + if (tbNum && pTbArr) { + for (int32_t i = 0; i < tbNum; ++i) { + if (metaPutTbToFilterCache(pVnode, &pTbArr[i], strlen(pTbArr[i])) != 0) { return terrno ? terrno : -1; } } From c70a1b77e68c300a6969af647103481c18649475 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 7 Oct 2023 00:17:21 +0800 Subject: [PATCH 09/12] chore: code optimization for time series --- source/dnode/vnode/inc/vnode.h | 4 +-- source/dnode/vnode/src/inc/vnodeInt.h | 3 ++ source/dnode/vnode/src/meta/metaCache.c | 40 +++++++++++++++++++++---- source/dnode/vnode/src/meta/metaTable.c | 8 +++-- source/dnode/vnode/src/vnd/vnodeOpen.c | 4 +++ source/dnode/vnode/src/vnd/vnodeQuery.c | 32 ++------------------ 6 files changed, 50 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index ce57ef3bbd..6ee6985e0c 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -132,8 +132,8 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen); -bool metaTbInFilterCache(void *pVnode, void* key, int8_t type); -int32_t metaPutTbToFilterCache(void *pVnode, void* key, int8_t type); +bool metaTbInFilterCache(void *pVnode, const void* key, int8_t type); +int32_t metaPutTbToFilterCache(void *pVnode, const void* key, int8_t type); int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); int32_t metaInitTbFilterCache(void *pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 2fadccdf2d..9929b66932 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -98,6 +98,9 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" +#define TK_LOG_STB_NUM 19 +#define TK_AUDIT_STB_NUM 1 + // vnd.h typedef int32_t (*_query_reseek_func_t)(void* pQHandle); struct SQueryNode { diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index a4a893c42d..1411285b42 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -14,6 +14,11 @@ */ #include "meta.h" +#ifdef TD_ENTERPRISE +extern const char* tkLogStb[]; +extern const char* tkAuditStb[]; +#endif + #define TAG_FILTER_RES_KEY_LEN 32 #define META_CACHE_BASE_BUCKET 1024 #define META_CACHE_STATS_BUCKET 16 @@ -185,9 +190,6 @@ int32_t metaCacheOpen(SMeta* pMeta) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err2; } - if ((code = metaInitTbFilterCache(pMeta->pVnode)) != 0) { - goto _err2; - } pMeta->pCache = pCache; return code; @@ -905,7 +907,7 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { return TSDB_CODE_SUCCESS; } -bool metaTbInFilterCache(void* pVnode, void* key, int8_t type) { +bool metaTbInFilterCache(void* pVnode, const void* key, int8_t type) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) { @@ -919,7 +921,7 @@ bool metaTbInFilterCache(void* pVnode, void* key, int8_t type) { return false; } -int32_t metaPutTbToFilterCache(void* pVnode, void* key, int8_t type) { +int32_t metaPutTbToFilterCache(void* pVnode, const void* key, int8_t type) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; if (type == 0) { @@ -939,4 +941,30 @@ int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) { return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb); } return 0; -} \ No newline at end of file +} + +int32_t metaInitTbFilterCache(void* pVnode) { +#ifdef TD_ENTERPRISE + int32_t tbNum = 0; + const char** pTbArr = NULL; + const char* dbName = NULL; + + if (!(dbName = strchr(((SVnode*)pVnode)->config.dbname, '.'))) return 0; + if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { + tbNum = TK_LOG_STB_NUM; + pTbArr = (const char**)&tkLogStb; + } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { + tbNum = TK_AUDIT_STB_NUM; + pTbArr = (const char**)&tkAuditStb; + } + if (tbNum && pTbArr) { + for (int32_t i = 0; i < tbNum; ++i) { + if (metaPutTbToFilterCache(pVnode, pTbArr[i], 1) != 0) { + return terrno ? terrno : -1; + } + } + } +#else +#endif + return 0; +} diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0ceeaffa05..d795afde74 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -764,6 +764,8 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs } metaReaderClear(&mr); + bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1); + // build SMetaEntry SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; me.version = ver; @@ -800,8 +802,8 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs #endif ++pStats->numOfCTables; - - if (!metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1)) { + + if (!sysTbl) { int32_t nCols = 0; metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); pStats->numOfTimeSeries += nCols - 1; @@ -826,7 +828,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs if (metaHandleEntry(pMeta, &me) < 0) goto _err; - metaTimeSeriesNotifyCheck(pMeta); + if(!sysTbl) metaTimeSeriesNotifyCheck(pMeta); if (pMetaRsp) { *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index db94f32459..3797941e96 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -399,6 +399,10 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC goto _err; } + if (metaInitTbFilterCache(pVnode) != 0) { + goto _err; + } + if (metaUpgrade(pVnode, &pVnode->pMeta) < 0) { vError("vgId:%d, failed to upgrade meta since %s", TD_VID(pVnode), tstrerror(terrno)); } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 3a8ab824a1..d13d49fccc 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -566,9 +566,7 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { } #ifdef TD_ENTERPRISE -#define TK_LOG_STB_NUM 19 -#define TK_AUDIT_STB_NUM 1 -static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", +const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "data_dir", "dnodes_info", "d_info", @@ -587,7 +585,7 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "temp_dir", "vgroups_info", "vnodes_role"}; -static const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"}; +const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"}; // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { @@ -619,32 +617,6 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { return tbSize; } - -int32_t metaInitTbFilterCache(void *pVnode) { - int32_t tbNum = 0; - const char **pTbArr = NULL; - const char *dbName = NULL; - - if (!(dbName = strchr(((SVnode *)pVnode)->config.dbname, '.'))) return 0; - if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { - tbNum = TK_LOG_STB_NUM; - pTbArr = (const char **)&tkLogStb; - } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { - tbNum = TK_AUDIT_STB_NUM; - pTbArr = (const char **)&tkAuditStb; - } - if (tbNum && pTbArr) { - for (int32_t i = 0; i < tbNum; ++i) { - if (metaPutTbToFilterCache(pVnode, &pTbArr[i], strlen(pTbArr[i])) != 0) { - return terrno ? terrno : -1; - } - } - } - - return 0; -} -#else -int32_t metaInitTbFilterCache(void *pVnode) { return 0; } #endif static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { From 6efe3845d4d77288c7ff840c751852d2daa98ba7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 7 Oct 2023 07:02:16 +0800 Subject: [PATCH 10/12] chore: more logic --- source/dnode/vnode/src/meta/metaTable.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index d795afde74..9577c533cd 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -396,6 +396,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { nStbEntry.stbEntry.schemaTag = pReq->schemaTag; int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols; + bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta->pVnode, pReq->name, 1); metaWLock(pMeta); // compare two entry @@ -411,15 +412,16 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // metaStatsCacheDrop(pMeta, nStbEntry.uid); - if (deltaCol != 0 && !metaTbInFilterCache(pMeta->pVnode, pReq->name, 1)) { + if (updStat) { metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol); } metaULock(pMeta); - if (deltaCol != 0) { + if (updStat) { int64_t ctbNum; metaGetStbStats(pMeta->pVnode, pReq->suid, &ctbNum, NULL); pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol); + metaTimeSeriesNotifyCheck(pMeta); } _exit: @@ -828,7 +830,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs if (metaHandleEntry(pMeta, &me) < 0) goto _err; - if(!sysTbl) metaTimeSeriesNotifyCheck(pMeta); + metaTimeSeriesNotifyCheck(pMeta); if (pMetaRsp) { *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); From 85226d05a374c72a3ed09181cd5f89fdbbf73a98 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 7 Oct 2023 09:30:32 +0800 Subject: [PATCH 11/12] chore: code refactor for time series --- source/dnode/vnode/inc/vnode.h | 8 ++++---- source/dnode/vnode/src/meta/metaCache.c | 17 ++++++----------- source/dnode/vnode/src/meta/metaOpen.c | 4 ++++ source/dnode/vnode/src/meta/metaTable.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeOpen.c | 4 ---- source/dnode/vnode/src/vnd/vnodeQuery.c | 10 +++++----- 6 files changed, 22 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6ee6985e0c..76ed0f4ed0 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -132,10 +132,10 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen); -bool metaTbInFilterCache(void *pVnode, const void* key, int8_t type); -int32_t metaPutTbToFilterCache(void *pVnode, const void* key, int8_t type); -int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); -int32_t metaInitTbFilterCache(void *pVnode); +bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type); +int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type); +int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type); +int32_t metaInitTbFilterCache(SMeta *pMeta); int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 1411285b42..999ef4ee64 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -907,9 +907,7 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { return TSDB_CODE_SUCCESS; } -bool metaTbInFilterCache(void* pVnode, const void* key, int8_t type) { - SMeta* pMeta = ((SVnode*)pVnode)->pMeta; - +bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type) { if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) { return true; } @@ -921,9 +919,7 @@ bool metaTbInFilterCache(void* pVnode, const void* key, int8_t type) { return false; } -int32_t metaPutTbToFilterCache(void* pVnode, const void* key, int8_t type) { - SMeta* pMeta = ((SVnode*)pVnode)->pMeta; - +int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type) { if (type == 0) { return taosHashPut(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t), NULL, 0); } @@ -935,21 +931,20 @@ int32_t metaPutTbToFilterCache(void* pVnode, const void* key, int8_t type) { return 0; } -int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) { - SMeta* pMeta = ((SVnode*)pVnode)->pMeta; +int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type) { if (type == 0) { return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb); } return 0; } -int32_t metaInitTbFilterCache(void* pVnode) { +int32_t metaInitTbFilterCache(SMeta* pMeta) { #ifdef TD_ENTERPRISE int32_t tbNum = 0; const char** pTbArr = NULL; const char* dbName = NULL; - if (!(dbName = strchr(((SVnode*)pVnode)->config.dbname, '.'))) return 0; + if (!(dbName = strchr(pMeta->pVnode->config.dbname, '.'))) return 0; if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { tbNum = TK_LOG_STB_NUM; pTbArr = (const char**)&tkLogStb; @@ -959,7 +954,7 @@ int32_t metaInitTbFilterCache(void* pVnode) { } if (tbNum && pTbArr) { for (int32_t i = 0; i < tbNum; ++i) { - if (metaPutTbToFilterCache(pVnode, pTbArr[i], 1) != 0) { + if (metaPutTbToFilterCache(pMeta, pTbArr[i], 1) != 0) { return terrno ? terrno : -1; } } diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 3d445acd67..8cab17c417 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -176,6 +176,10 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) { goto _err; } + if (metaInitTbFilterCache(pMeta) != 0) { + goto _err; + } + metaDebug("vgId:%d, meta is opened", TD_VID(pVnode)); *ppMeta = pMeta; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 9577c533cd..863cd8006a 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -396,7 +396,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { nStbEntry.stbEntry.schemaTag = pReq->schemaTag; int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols; - bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta->pVnode, pReq->name, 1); + bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1); metaWLock(pMeta); // compare two entry @@ -766,7 +766,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs } metaReaderClear(&mr); - bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1); + bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1); // build SMetaEntry SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; @@ -1110,7 +1110,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p tDecoderInit(&tdc, tData, tLen); metaDecodeEntry(&tdc, &stbEntry); - if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta->pVnode, stbEntry.name, 1) ? 1 : 0; + if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta, stbEntry.name, 1) ? 1 : 0; SSchema *pTagColumn = NULL; SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 3797941e96..db94f32459 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -399,10 +399,6 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC goto _err; } - if (metaInitTbFilterCache(pVnode) != 0) { - goto _err; - } - if (metaUpgrade(pVnode, &pVnode->pMeta) < 0) { vError("vgId:%d, failed to upgrade meta since %s", TD_VID(pVnode), tstrerror(terrno)); } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index d13d49fccc..3ec85ac93a 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -594,7 +594,7 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { const char **pTbArr = NULL; const char *dbName = NULL; - if (!(dbName = strchr(((SVnode *)pVnode)->config.dbname, '.'))) return 0; + if (!(dbName = strchr(pVnode->config.dbname, '.'))) return 0; if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { tbNum = TK_LOG_STB_NUM; pTbArr = (const char **)&tkLogStb; @@ -603,15 +603,15 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { pTbArr = (const char **)&tkAuditStb; } if (tbNum && pTbArr) { - tbSize = metaSizeOfTbFilterCache(pVnode, 0); + tbSize = metaSizeOfTbFilterCache(pVnode->pMeta, 0); if (tbSize < tbNum) { for (int32_t i = 0; i < tbNum; ++i) { tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, pTbArr[i]); if (suid != 0) { - metaPutTbToFilterCache(pVnode, &suid, 0); + metaPutTbToFilterCache(pVnode->pMeta, &suid, 0); } } - tbSize = metaSizeOfTbFilterCache(pVnode, 0); + tbSize = metaSizeOfTbFilterCache(pVnode->pMeta, 0); } } @@ -622,7 +622,7 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { SVnode *pVnode = (SVnode *)arg1; - if (metaTbInFilterCache(pVnode, arg2, 0)) { + if (metaTbInFilterCache(pVnode->pMeta, arg2, 0)) { return true; } return false; From b9767707cb31a4ce29f22ed85586669db0fd8776 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 7 Oct 2023 15:57:26 +0800 Subject: [PATCH 12/12] chore: code optimization for array size --- source/dnode/vnode/src/inc/vnodeInt.h | 3 -- source/dnode/vnode/src/meta/metaCache.c | 6 ++-- source/dnode/vnode/src/vnd/vnodeQuery.c | 46 +++++++++++++------------ 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9929b66932..2fadccdf2d 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -98,9 +98,6 @@ typedef struct SQueryNode SQueryNode; #define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" -#define TK_LOG_STB_NUM 19 -#define TK_AUDIT_STB_NUM 1 - // vnd.h typedef int32_t (*_query_reseek_func_t)(void* pQHandle); struct SQueryNode { diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 999ef4ee64..ceb72aa14d 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -17,6 +17,8 @@ #ifdef TD_ENTERPRISE extern const char* tkLogStb[]; extern const char* tkAuditStb[]; +extern const int tkLogStbNum; +extern const int tkAuditStbNum; #endif #define TAG_FILTER_RES_KEY_LEN 32 @@ -946,10 +948,10 @@ int32_t metaInitTbFilterCache(SMeta* pMeta) { if (!(dbName = strchr(pMeta->pVnode->config.dbname, '.'))) return 0; if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { - tbNum = TK_LOG_STB_NUM; + tbNum = tkLogStbNum; pTbArr = (const char**)&tkLogStb; } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { - tbNum = TK_AUDIT_STB_NUM; + tbNum = tkAuditStbNum; pTbArr = (const char**)&tkAuditStb; } if (tbNum && pTbArr) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 3ec85ac93a..e9dbc5e659 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -566,26 +566,28 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { } #ifdef TD_ENTERPRISE -const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", - "data_dir", - "dnodes_info", - "d_info", - "grants_info", - "keeper_monitor", - "logs", - "log_dir", - "log_summary", - "m_info", - "taosadapter_restful_http_request_fail", - "taosadapter_restful_http_request_in_flight", - "taosadapter_restful_http_request_summary_milliseconds", - "taosadapter_restful_http_request_total", - "taosadapter_system_cpu_percent", - "taosadapter_system_mem_percent", - "temp_dir", - "vgroups_info", - "vnodes_role"}; -const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"}; +const char *tkLogStb[] = {"cluster_info", + "data_dir", + "dnodes_info", + "d_info", + "grants_info", + "keeper_monitor", + "logs", + "log_dir", + "log_summary", + "m_info", + "taosadapter_restful_http_request_fail", + "taosadapter_restful_http_request_in_flight", + "taosadapter_restful_http_request_summary_milliseconds", + "taosadapter_restful_http_request_total", + "taosadapter_system_cpu_percent", + "taosadapter_system_mem_percent", + "temp_dir", + "vgroups_info", + "vnodes_role"}; +const char *tkAuditStb[] = {"operations"}; +const int tkLogStbNum = ARRAY_SIZE(tkLogStb); +const int tkAuditStbNum = ARRAY_SIZE(tkAuditStb); // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { @@ -596,10 +598,10 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { if (!(dbName = strchr(pVnode->config.dbname, '.'))) return 0; if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { - tbNum = TK_LOG_STB_NUM; + tbNum = tkLogStbNum; pTbArr = (const char **)&tkLogStb; } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) { - tbNum = TK_AUDIT_STB_NUM; + tbNum = tkAuditStbNum; pTbArr = (const char **)&tkAuditStb; } if (tbNum && pTbArr) {