enh: exclude tk_log from timeseries check

This commit is contained in:
kailixu 2023-07-07 10:14:37 +08:00
parent 85782bbff9
commit 06c52af2c2
5 changed files with 130 additions and 30 deletions

View File

@ -126,6 +126,9 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t payloadLen); int32_t payloadLen);
bool metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
int32_t metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type);
int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables);

View File

@ -66,6 +66,10 @@ struct SMetaCache {
SHashObj* pTableEntry; SHashObj* pTableEntry;
SLRUCache* pResCache; SLRUCache* pResCache;
} STbGroupResCache; } STbGroupResCache;
struct STbFilterCache {
SHashObj* pTkLogStb;
} STbFilterCache;
}; };
static void entryCacheClose(SMeta* pMeta) { static void entryCacheClose(SMeta* pMeta) {
@ -168,6 +172,13 @@ int32_t metaCacheOpen(SMeta* pMeta) {
taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL); taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL);
pCache->STbFilterCache.pTkLogStb =
taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (pCache->STbFilterCache.pTkLogStb == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
}
pMeta->pCache = pCache; pMeta->pCache = pCache;
return code; return code;
@ -193,6 +204,8 @@ void metaCacheClose(SMeta* pMeta) {
taosThreadMutexDestroy(&pMeta->pCache->STbGroupResCache.lock); taosThreadMutexDestroy(&pMeta->pCache->STbGroupResCache.lock);
taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry); taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry);
taosHashCleanup(pMeta->pCache->STbFilterCache.pTkLogStb);
taosMemoryFree(pMeta->pCache); taosMemoryFree(pMeta->pCache);
pMeta->pCache = NULL; pMeta->pCache = NULL;
} }
@ -880,3 +893,31 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) {
metaDebug("vgId:%d suid:%" PRId64 " cached related tb group cleared", vgId, suid); metaDebug("vgId:%d suid:%" PRId64 " cached related tb group cleared", vgId, suid);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool metaTbInFilterCache(void* pVnode, tb_uid_t suid, int8_t type) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pTkLogStb, &suid, sizeof(suid))) {
return true;
}
return false;
}
int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
if (type == 0) {
return taosHashPut(pMeta->pCache->STbFilterCache.pTkLogStb, &suid, sizeof(suid), NULL, 0);
}
return 0;
}
int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
if (type == 0) {
return taosHashGetSize(pMeta->pCache->STbFilterCache.pTkLogStb);
}
return 0;
}

View File

@ -671,6 +671,7 @@ int64_t metaGetTbNum(SMeta *pMeta) {
// N.B. Called by statusReq per second // N.B. Called by statusReq per second
int64_t metaGetTimeSeriesNum(SMeta *pMeta) { int64_t metaGetTimeSeriesNum(SMeta *pMeta) {
fprintf(stderr, "@@@@@@@ %s:%d called @@@@@@@@@: vgId:%d, second:%d\n", __func__, __LINE__, TD_VID(pMeta->pVnode), taosGetTimestampSec());
// sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column) // sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column)
if (pMeta->pVnode->config.vndStats.numOfTimeSeries <= 0 || if (pMeta->pVnode->config.vndStats.numOfTimeSeries <= 0 ||
++pMeta->pVnode->config.vndStats.itvTimeSeries % (60 * 5) == 0) { ++pMeta->pVnode->config.vndStats.itvTimeSeries % (60 * 5) == 0) {

View File

@ -232,7 +232,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
++pMeta->pVnode->config.vndStats.numOfSTables; ++pMeta->pVnode->config.vndStats.numOfSTables;
metaDebug("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); metaError("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
return 0; return 0;
@ -798,7 +798,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
} }
} }
metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, metaError("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid,
pReq->type); pReq->type);
return 0; return 0;

View File

@ -531,6 +531,57 @@ static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// #ifndef TD_ENTERPRISE
#define TK_LOG_STB_NUM 19
static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info",
"data_dir",
"dnodes_info",
"d_info",
"grants_info",
"keeper_monitor",
"logs",
"log_dir",
"log_summary",
"m_info",
"taosadapter_restful_http_request_fail",
"taosadapter_restful_http_request_in_flight",
"taosadapter_restful_http_request_summary_milliseconds",
"taosadapter_restful_http_request_total",
"taosadapter_system_cpu_percent",
"taosadapter_system_mem_percent",
"temp_dir",
"vgroups_info",
"vnodes_role"};
// exclude stbs of taoskeeper log
static int32_t vnodeTimeSeriesFilter(SVnode *pVnode, SArray *suidList) {
char *dbName = strchr(pVnode->config.dbname, '.');
if (!dbName || 0 != strncmp(dbName, "log", TSDB_DB_NAME_LEN)) {
goto _exit;
}
int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0);
if (tbSize < TK_LOG_STB_NUM) {
for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) {
tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]);
if (suid != 0) {
metaPutTbToFilterCache(pVnode, suid, 0);
}
}
if (metaSizeOfTbFilterCache(pVnode, 0) <= 0) goto _exit;
}
for (int64_t i = 0; i < TARRAY_SIZE(suidList);) {
if (metaTbInFilterCache(pVnode, *(tb_uid_t *)TARRAY_GET_ELEM(suidList, i), sizeof(tb_uid_t))) {
taosArrayRemove(suidList, i);
continue;
}
}
_exit:
return 0;
}
// #endif
int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
SArray *suidList = NULL; SArray *suidList = NULL;
@ -545,6 +596,10 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
// #ifdef TD_ENTERPRISE
vnodeTimeSeriesFilter(pVnode, suidList);
// #endif
*num = 0; *num = 0;
int64_t arrSize = taosArrayGetSize(suidList); int64_t arrSize = taosArrayGetSize(suidList);
for (int64_t i = 0; i < arrSize; ++i) { for (int64_t i = 0; i < arrSize; ++i) {