xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh: exclude tk_log from timeseries check

This commit is contained in:
kailixu 2023-07-07 10:14:37 +08:00
parent 85782bbff9
commit 06c52af2c2
5 changed files with 130 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
source/dnode/vnode/inc/vnode.h
View File

@ -124,8 +124,11 @@ int32_t metaUidFilterCachePut(void *pVnode, uint64_t suid, const void *pKey, in
int32_t payloadLen, double selectivityRatio);
tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t payloadLen);
bool metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
int32_t metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type);
int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables);

41
source/dnode/vnode/src/meta/metaCache.c
View File

@ -66,6 +66,10 @@ struct SMetaCache {
SHashObj* pTableEntry;
SLRUCache* pResCache;
} STbGroupResCache;
struct STbFilterCache {
SHashObj* pTkLogStb;
} STbFilterCache;
};
static void entryCacheClose(SMeta* pMeta) {
@ -168,6 +172,13 @@ int32_t metaCacheOpen(SMeta* pMeta) {
taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp);
taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL);
pCache->STbFilterCache.pTkLogStb =
taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (pCache->STbFilterCache.pTkLogStb == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
}
pMeta->pCache = pCache;
return code;
@ -193,6 +204,8 @@ void metaCacheClose(SMeta* pMeta) {
taosThreadMutexDestroy(&pMeta->pCache->STbGroupResCache.lock);
taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry);
taosHashCleanup(pMeta->pCache->STbFilterCache.pTkLogStb);
taosMemoryFree(pMeta->pCache);
pMeta->pCache = NULL;
}
@ -880,3 +893,31 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) {
metaDebug("vgId:%d suid:%" PRId64 " cached related tb group cleared", vgId, suid);
return TSDB_CODE_SUCCESS;
}
bool metaTbInFilterCache(void* pVnode, tb_uid_t suid, int8_t type) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pTkLogStb, &suid, sizeof(suid))) {
return true;
}
return false;
}
int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
if (type == 0) {
return taosHashPut(pMeta->pCache->STbFilterCache.pTkLogStb, &suid, sizeof(suid), NULL, 0);
}
return 0;
}
int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
if (type == 0) {
return taosHashGetSize(pMeta->pCache->STbFilterCache.pTkLogStb);
}
return 0;
}

1
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -671,6 +671,7 @@ int64_t metaGetTbNum(SMeta *pMeta) {
// N.B. Called by statusReq per second
int64_t metaGetTimeSeriesNum(SMeta *pMeta) {
fprintf(stderr, "@@@@@@@ %s:%d called @@@@@@@@@: vgId:%d, second:%d\n", __func__, __LINE__, TD_VID(pMeta->pVnode), taosGetTimestampSec());
// sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column)
if (pMeta->pVnode->config.vndStats.numOfTimeSeries <= 0 ||
++pMeta->pVnode->config.vndStats.itvTimeSeries % (60 * 5) == 0) {

4
source/dnode/vnode/src/meta/metaTable.c
View File

@ -232,7 +232,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
++pMeta->pVnode->config.vndStats.numOfSTables;
metaDebug("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
metaError("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid);
return 0;
@ -798,7 +798,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
}
}
metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid,
metaError("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid,
pReq->type);
return 0;

109
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -410,9 +410,9 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
"nBatchInsertSuccess");
}
void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables) {
SVnode* pVnodeObj = pVnode;
SVnodeCfg* pConf = &pVnodeObj->config;
void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables) {
SVnode *pVnodeObj = pVnode;
SVnodeCfg *pConf = &pVnodeObj->config;
if (dbname) {
*dbname = pConf->dbname;
@ -431,7 +431,7 @@ void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* num
}
}
int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) {
int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList) {
if (type == TSDB_SUPER_TABLE) {
return vnodeGetStbIdList(pVnode, 0, pList);
} else {
@ -531,32 +531,87 @@ static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
return TSDB_CODE_SUCCESS;
}
// #ifndef TD_ENTERPRISE
#define TK_LOG_STB_NUM 19
static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info",
"data_dir",
"dnodes_info",
"d_info",
"grants_info",
"keeper_monitor",
"logs",
"log_dir",
"log_summary",
"m_info",
"taosadapter_restful_http_request_fail",
"taosadapter_restful_http_request_in_flight",
"taosadapter_restful_http_request_summary_milliseconds",
"taosadapter_restful_http_request_total",
"taosadapter_system_cpu_percent",
"taosadapter_system_mem_percent",
"temp_dir",
"vgroups_info",
"vnodes_role"};
// exclude stbs of taoskeeper log
static int32_t vnodeTimeSeriesFilter(SVnode *pVnode, SArray *suidList) {
char *dbName = strchr(pVnode->config.dbname, '.');
if (!dbName || 0 != strncmp(dbName, "log", TSDB_DB_NAME_LEN)) {
goto _exit;
}
int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0);
if (tbSize < TK_LOG_STB_NUM) {
for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) {
tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]);
if (suid != 0) {
metaPutTbToFilterCache(pVnode, suid, 0);
}
}
if (metaSizeOfTbFilterCache(pVnode, 0) <= 0) goto _exit;
}
for (int64_t i = 0; i < TARRAY_SIZE(suidList);) {
if (metaTbInFilterCache(pVnode, *(tb_uid_t *)TARRAY_GET_ELEM(suidList, i), sizeof(tb_uid_t))) {
taosArrayRemove(suidList, i);
continue;
}
}
_exit:
return 0;
}
// #endif
int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
SArray *suidList = NULL;
if (!(suidList = taosArrayInit(1, sizeof(tb_uid_t)))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
if (vnodeGetStbIdList(pVnode, 0, suidList) < 0) {
qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr());
taosArrayDestroy(suidList);
return TSDB_CODE_FAILED;
qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr());
taosArrayDestroy(suidList);
return TSDB_CODE_FAILED;
}
// #ifdef TD_ENTERPRISE
vnodeTimeSeriesFilter(pVnode, suidList);
// #endif
*num = 0;
int64_t arrSize = taosArrayGetSize(suidList);
for (int64_t i = 0; i < arrSize; ++i) {
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
int64_t ctbNum = 0;
metaGetStbStats(pVnode, suid, &ctbNum);
int64_t ctbNum = 0;
metaGetStbStats(pVnode, suid, &ctbNum);
int numOfCols = 0;
vnodeGetStbColumnNum(pVnode, suid, &numOfCols);
int numOfCols = 0;
vnodeGetStbColumnNum(pVnode, suid, &numOfCols);
*num += ctbNum * (numOfCols - 1);
*num += ctbNum * (numOfCols - 1);
}
taosArrayDestroy(suidList);
@ -566,20 +621,20 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) {
SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, 0);
if (!pCur) {
return TSDB_CODE_FAILED;
return TSDB_CODE_FAILED;
}
*num = 0;
while (1) {
tb_uid_t id = metaStbCursorNext(pCur);
if (id == 0) {
break;
}
tb_uid_t id = metaStbCursorNext(pCur);
if (id == 0) {
break;
}
int64_t ctbNum = 0;
vnodeGetCtbNum(pVnode, id, &ctbNum);
int64_t ctbNum = 0;
vnodeGetCtbNum(pVnode, id, &ctbNum);
*num += ctbNum;
*num += ctbNum;
}
metaCloseStbCursor(pCur);
@ -588,15 +643,15 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) {
void *vnodeGetIdx(void *pVnode) {
if (pVnode == NULL) {
return NULL;
return NULL;
}
return metaGetIdx(((SVnode*)pVnode)->pMeta);
return metaGetIdx(((SVnode *)pVnode)->pMeta);
}
void *vnodeGetIvtIdx(void *pVnode) {
if (pVnode == NULL) {
return NULL;
return NULL;
}
return metaGetIvtIdx(((SVnode*)pVnode)->pMeta);
return metaGetIvtIdx(((SVnode *)pVnode)->pMeta);
}