From 06c52af2c27d4e6a6cd640ef7eb2b7b34c6aa31d Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 7 Jul 2023 10:14:37 +0800 Subject: [PATCH 1/7] enh: exclude tk_log from timeseries check --- source/dnode/vnode/inc/vnode.h | 5 +- source/dnode/vnode/src/meta/metaCache.c | 41 +++++++++ source/dnode/vnode/src/meta/metaQuery.c | 1 + source/dnode/vnode/src/meta/metaTable.c | 4 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 109 ++++++++++++++++++------ 5 files changed, 130 insertions(+), 30 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 5fb30a0028..b2ae293132 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -124,8 +124,11 @@ int32_t metaUidFilterCachePut(void *pVnode, uint64_t suid, const void *pKey, in int32_t payloadLen, double selectivityRatio); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); -int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, +int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen); +bool metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type); +int32_t metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type); +int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables); diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 078e6ee6af..c1a4b5d75b 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -66,6 +66,10 @@ struct SMetaCache { SHashObj* pTableEntry; SLRUCache* pResCache; } STbGroupResCache; + + struct STbFilterCache { + SHashObj* pTkLogStb; + } STbFilterCache; }; static void entryCacheClose(SMeta* pMeta) { @@ -168,6 +172,13 @@ int32_t metaCacheOpen(SMeta* pMeta) { taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL); + pCache->STbFilterCache.pTkLogStb = + taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pCache->STbFilterCache.pTkLogStb == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err2; + } + pMeta->pCache = pCache; return code; @@ -193,6 +204,8 @@ void metaCacheClose(SMeta* pMeta) { taosThreadMutexDestroy(&pMeta->pCache->STbGroupResCache.lock); taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry); + taosHashCleanup(pMeta->pCache->STbFilterCache.pTkLogStb); + taosMemoryFree(pMeta->pCache); pMeta->pCache = NULL; } @@ -880,3 +893,31 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { metaDebug("vgId:%d suid:%" PRId64 " cached related tb group cleared", vgId, suid); return TSDB_CODE_SUCCESS; } + +bool metaTbInFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + + if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pTkLogStb, &suid, sizeof(suid))) { + return true; + } + + return false; +} + +int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + + if (type == 0) { + return taosHashPut(pMeta->pCache->STbFilterCache.pTkLogStb, &suid, sizeof(suid), NULL, 0); + } + + return 0; +} + +int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) { + SMeta* pMeta = ((SVnode*)pVnode)->pMeta; + if (type == 0) { + return taosHashGetSize(pMeta->pCache->STbFilterCache.pTkLogStb); + } + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index c26bb45c2b..34bc649927 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -671,6 +671,7 @@ int64_t metaGetTbNum(SMeta *pMeta) { // N.B. Called by statusReq per second int64_t metaGetTimeSeriesNum(SMeta *pMeta) { + fprintf(stderr, "@@@@@@@ %s:%d called @@@@@@@@@: vgId:%d, second:%d\n", __func__, __LINE__, TD_VID(pMeta->pVnode), taosGetTimestampSec()); // sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column) if (pMeta->pVnode->config.vndStats.numOfTimeSeries <= 0 || ++pMeta->pVnode->config.vndStats.itvTimeSeries % (60 * 5) == 0) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index cb4b3231f6..b0821be091 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -232,7 +232,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { ++pMeta->pVnode->config.vndStats.numOfSTables; - metaDebug("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + metaError("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; @@ -798,7 +798,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs } } - metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, + metaError("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, pReq->type); return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index c122a98a12..33a3bd5eb3 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -410,9 +410,9 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { "nBatchInsertSuccess"); } -void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables) { - SVnode* pVnodeObj = pVnode; - SVnodeCfg* pConf = &pVnodeObj->config; +void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables) { + SVnode *pVnodeObj = pVnode; + SVnodeCfg *pConf = &pVnodeObj->config; if (dbname) { *dbname = pConf->dbname; @@ -431,7 +431,7 @@ void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* num } } -int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) { +int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList) { if (type == TSDB_SUPER_TABLE) { return vnodeGetStbIdList(pVnode, 0, pList); } else { @@ -531,32 +531,87 @@ static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { return TSDB_CODE_SUCCESS; } +// #ifndef TD_ENTERPRISE +#define TK_LOG_STB_NUM 19 +static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", + "data_dir", + "dnodes_info", + "d_info", + "grants_info", + "keeper_monitor", + "logs", + "log_dir", + "log_summary", + "m_info", + "taosadapter_restful_http_request_fail", + "taosadapter_restful_http_request_in_flight", + "taosadapter_restful_http_request_summary_milliseconds", + "taosadapter_restful_http_request_total", + "taosadapter_system_cpu_percent", + "taosadapter_system_mem_percent", + "temp_dir", + "vgroups_info", + "vnodes_role"}; + +// exclude stbs of taoskeeper log +static int32_t vnodeTimeSeriesFilter(SVnode *pVnode, SArray *suidList) { + char *dbName = strchr(pVnode->config.dbname, '.'); + if (!dbName || 0 != strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { + goto _exit; + } + int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0); + if (tbSize < TK_LOG_STB_NUM) { + for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) { + tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]); + if (suid != 0) { + metaPutTbToFilterCache(pVnode, suid, 0); + } + } + if (metaSizeOfTbFilterCache(pVnode, 0) <= 0) goto _exit; + } + + for (int64_t i = 0; i < TARRAY_SIZE(suidList);) { + if (metaTbInFilterCache(pVnode, *(tb_uid_t *)TARRAY_GET_ELEM(suidList, i), sizeof(tb_uid_t))) { + taosArrayRemove(suidList, i); + continue; + } + } + +_exit: + return 0; +} +// #endif + int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { SArray *suidList = NULL; if (!(suidList = taosArrayInit(1, sizeof(tb_uid_t)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; } if (vnodeGetStbIdList(pVnode, 0, suidList) < 0) { - qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr()); - taosArrayDestroy(suidList); - return TSDB_CODE_FAILED; + qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr()); + taosArrayDestroy(suidList); + return TSDB_CODE_FAILED; } +// #ifdef TD_ENTERPRISE + vnodeTimeSeriesFilter(pVnode, suidList); +// #endif + *num = 0; int64_t arrSize = taosArrayGetSize(suidList); for (int64_t i = 0; i < arrSize; ++i) { - tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); + tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); - int64_t ctbNum = 0; - metaGetStbStats(pVnode, suid, &ctbNum); + int64_t ctbNum = 0; + metaGetStbStats(pVnode, suid, &ctbNum); - int numOfCols = 0; - vnodeGetStbColumnNum(pVnode, suid, &numOfCols); + int numOfCols = 0; + vnodeGetStbColumnNum(pVnode, suid, &numOfCols); - *num += ctbNum * (numOfCols - 1); + *num += ctbNum * (numOfCols - 1); } taosArrayDestroy(suidList); @@ -566,20 +621,20 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) { SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, 0); if (!pCur) { - return TSDB_CODE_FAILED; + return TSDB_CODE_FAILED; } *num = 0; while (1) { - tb_uid_t id = metaStbCursorNext(pCur); - if (id == 0) { - break; - } + tb_uid_t id = metaStbCursorNext(pCur); + if (id == 0) { + break; + } - int64_t ctbNum = 0; - vnodeGetCtbNum(pVnode, id, &ctbNum); + int64_t ctbNum = 0; + vnodeGetCtbNum(pVnode, id, &ctbNum); - *num += ctbNum; + *num += ctbNum; } metaCloseStbCursor(pCur); @@ -588,15 +643,15 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) { void *vnodeGetIdx(void *pVnode) { if (pVnode == NULL) { - return NULL; + return NULL; } - return metaGetIdx(((SVnode*)pVnode)->pMeta); + return metaGetIdx(((SVnode *)pVnode)->pMeta); } void *vnodeGetIvtIdx(void *pVnode) { if (pVnode == NULL) { - return NULL; + return NULL; } - return metaGetIvtIdx(((SVnode*)pVnode)->pMeta); + return metaGetIvtIdx(((SVnode *)pVnode)->pMeta); } From 802112c00a2e35b2381800636bd9468f0549a29b Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 9 Jul 2023 21:49:37 +0800 Subject: [PATCH 2/7] chore: more code --- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/vnd/vnodeQuery.c | 100 +++++++++++++++--------- 2 files changed, 64 insertions(+), 37 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index b2ae293132..e1b6c0b09a 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -78,6 +78,7 @@ ESyncRole vnodeGetRole(SVnode *pVnode); int32_t vnodeGetCtbIdList(void *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list); +int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void* arg1), void *arg); void *vnodeGetIdx(void *pVnode); void *vnodeGetIvtIdx(void *pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 33a3bd5eb3..ca2be5102e 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -496,6 +496,30 @@ int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { return TSDB_CODE_SUCCESS; } +int32_t vnodeGetStbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg, void *arg1), + void *arg) { + SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); + if (!pCur) { + return TSDB_CODE_FAILED; + } + + while (1) { + tb_uid_t id = metaStbCursorNext(pCur); + if (id == 0) { + break; + } + + if ((*filter) && (*filter)(arg, &id)) { + continue; + } + + taosArrayPush(list, &id); + } + + metaCloseStbCursor(pCur); + return TSDB_CODE_SUCCESS; +} + int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { SMCtbCursor *pCur = metaOpenCtbCursor(pVnode->pMeta, suid, 0); if (!pCur) { @@ -554,10 +578,10 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "vnodes_role"}; // exclude stbs of taoskeeper log -static int32_t vnodeTimeSeriesFilter(SVnode *pVnode, SArray *suidList) { +static int32_t vnodeGetTimeSeriBlackList(SVnode *pVnode) { char *dbName = strchr(pVnode->config.dbname, '.'); if (!dbName || 0 != strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { - goto _exit; + return 0; } int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0); if (tbSize < TK_LOG_STB_NUM) { @@ -567,51 +591,53 @@ static int32_t vnodeTimeSeriesFilter(SVnode *pVnode, SArray *suidList) { metaPutTbToFilterCache(pVnode, suid, 0); } } - if (metaSizeOfTbFilterCache(pVnode, 0) <= 0) goto _exit; } - for (int64_t i = 0; i < TARRAY_SIZE(suidList);) { - if (metaTbInFilterCache(pVnode, *(tb_uid_t *)TARRAY_GET_ELEM(suidList, i), sizeof(tb_uid_t))) { - taosArrayRemove(suidList, i); - continue; - } - } - -_exit: return 0; } // #endif +static bool filter(void *arg1, void *arg2) { + SVnode *pVnode = (SVnode *)arg1; + + if (metaTbInFilterCache(pVnode, *(tb_uid_t *)(arg2), 0)) { + return true; + } + return false; +} + int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { SArray *suidList = NULL; if (!(suidList = taosArrayInit(1, sizeof(tb_uid_t)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; } - if (vnodeGetStbIdList(pVnode, 0, suidList) < 0) { - qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr()); - taosArrayDestroy(suidList); - return TSDB_CODE_FAILED; - } + void *blackListArg = NULL; + // #ifdef TD_ENTERPRISE + vnodeTimeSeriesFilter(pVnode, blackListArg); + // #endif -// #ifdef TD_ENTERPRISE - vnodeTimeSeriesFilter(pVnode, suidList); -// #endif + if ((!blackListArg && vnodeGetStbIdList(pVnode, 0, suidList) < 0) || + (blackListArg && vnodeGetStbIdListByFilter(pVnode, 0, suidList, filter, pVnode) < 0)) { + qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr()); + taosArrayDestroy(suidList); + return TSDB_CODE_FAILED; + } *num = 0; int64_t arrSize = taosArrayGetSize(suidList); for (int64_t i = 0; i < arrSize; ++i) { - tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); + tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); - int64_t ctbNum = 0; - metaGetStbStats(pVnode, suid, &ctbNum); + int64_t ctbNum = 0; + metaGetStbStats(pVnode, suid, &ctbNum); - int numOfCols = 0; - vnodeGetStbColumnNum(pVnode, suid, &numOfCols); + int numOfCols = 0; + vnodeGetStbColumnNum(pVnode, suid, &numOfCols); - *num += ctbNum * (numOfCols - 1); + *num += ctbNum * (numOfCols - 1); } taosArrayDestroy(suidList); @@ -621,20 +647,20 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) { SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, 0); if (!pCur) { - return TSDB_CODE_FAILED; + return TSDB_CODE_FAILED; } *num = 0; while (1) { - tb_uid_t id = metaStbCursorNext(pCur); - if (id == 0) { - break; - } + tb_uid_t id = metaStbCursorNext(pCur); + if (id == 0) { + break; + } - int64_t ctbNum = 0; - vnodeGetCtbNum(pVnode, id, &ctbNum); + int64_t ctbNum = 0; + vnodeGetCtbNum(pVnode, id, &ctbNum); - *num += ctbNum; + *num += ctbNum; } metaCloseStbCursor(pCur); @@ -643,7 +669,7 @@ int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num) { void *vnodeGetIdx(void *pVnode) { if (pVnode == NULL) { - return NULL; + return NULL; } return metaGetIdx(((SVnode *)pVnode)->pMeta); @@ -651,7 +677,7 @@ void *vnodeGetIdx(void *pVnode) { void *vnodeGetIvtIdx(void *pVnode) { if (pVnode == NULL) { - return NULL; + return NULL; } return metaGetIvtIdx(((SVnode *)pVnode)->pMeta); } From f16fa6d000eef965fc45e9cbf5e4f23d733f027d Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 10 Jul 2023 19:48:50 +0800 Subject: [PATCH 3/7] enh: exclude tk log from time series check --- source/dnode/vnode/src/vnd/vnodeQuery.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index ca2be5102e..2551fd1112 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -555,7 +555,7 @@ static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { return TSDB_CODE_SUCCESS; } -// #ifndef TD_ENTERPRISE +#ifndef TD_ENTERPRISE #define TK_LOG_STB_NUM 19 static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "data_dir", @@ -578,7 +578,7 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "vnodes_role"}; // exclude stbs of taoskeeper log -static int32_t vnodeGetTimeSeriBlackList(SVnode *pVnode) { +static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { char *dbName = strchr(pVnode->config.dbname, '.'); if (!dbName || 0 != strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { return 0; @@ -591,13 +591,14 @@ static int32_t vnodeGetTimeSeriBlackList(SVnode *pVnode) { metaPutTbToFilterCache(pVnode, suid, 0); } } + tbSize = metaSizeOfTbFilterCache(pVnode, 0); } - return 0; + return tbSize; } -// #endif +#endif -static bool filter(void *arg1, void *arg2) { +static bool vnodeTimeSeriesStbFilter(void *arg1, void *arg2) { SVnode *pVnode = (SVnode *)arg1; if (metaTbInFilterCache(pVnode, *(tb_uid_t *)(arg2), 0)) { @@ -614,13 +615,13 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { return TSDB_CODE_FAILED; } - void *blackListArg = NULL; - // #ifdef TD_ENTERPRISE - vnodeTimeSeriesFilter(pVnode, blackListArg); - // #endif + int32_t tbFilterSize = 0; + #ifdef TD_ENTERPRISE + tbFilterSize = vnodeGetTimeSeriesBlackList(pVnode); + #endif - if ((!blackListArg && vnodeGetStbIdList(pVnode, 0, suidList) < 0) || - (blackListArg && vnodeGetStbIdListByFilter(pVnode, 0, suidList, filter, pVnode) < 0)) { + if ((!tbFilterSize && vnodeGetStbIdList(pVnode, 0, suidList) < 0) || + (tbFilterSize && vnodeGetStbIdListByFilter(pVnode, 0, suidList, vnodeTimeSeriesStbFilter, pVnode) < 0)) { qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr()); taosArrayDestroy(suidList); return TSDB_CODE_FAILED; From 57ceaed5356acc6173502b9f5d9f17b11d2963d0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 10 Jul 2023 19:55:01 +0800 Subject: [PATCH 4/7] chore: code revert --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 1 - source/dnode/vnode/src/meta/metaTable.c | 4 ++-- source/dnode/vnode/src/vnd/vnodeQuery.c | 14 +++++++------- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e1b6c0b09a..0b7820c030 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -125,7 +125,7 @@ int32_t metaUidFilterCachePut(void *pVnode, uint64_t suid, const void *pKey, in int32_t payloadLen, double selectivityRatio); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); -int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, +int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t payloadLen); bool metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type); int32_t metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 34bc649927..c26bb45c2b 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -671,7 +671,6 @@ int64_t metaGetTbNum(SMeta *pMeta) { // N.B. Called by statusReq per second int64_t metaGetTimeSeriesNum(SMeta *pMeta) { - fprintf(stderr, "@@@@@@@ %s:%d called @@@@@@@@@: vgId:%d, second:%d\n", __func__, __LINE__, TD_VID(pMeta->pVnode), taosGetTimestampSec()); // sum of (number of columns of stable - 1) * number of ctables (excluding timestamp column) if (pMeta->pVnode->config.vndStats.numOfTimeSeries <= 0 || ++pMeta->pVnode->config.vndStats.itvTimeSeries % (60 * 5) == 0) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index b0821be091..cb4b3231f6 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -232,7 +232,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { ++pMeta->pVnode->config.vndStats.numOfSTables; - metaError("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); + metaDebug("vgId:%d, stb:%s is created, suid:%" PRId64, TD_VID(pMeta->pVnode), pReq->name, pReq->suid); return 0; @@ -798,7 +798,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs } } - metaError("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, + metaDebug("vgId:%d, table:%s uid %" PRId64 " is created, type:%" PRId8, TD_VID(pMeta->pVnode), pReq->name, pReq->uid, pReq->type); return 0; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 2551fd1112..5170e25a1c 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -410,9 +410,9 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { "nBatchInsertSuccess"); } -void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables) { - SVnode *pVnodeObj = pVnode; - SVnodeCfg *pConf = &pVnodeObj->config; +void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t* numOfTables, int64_t* numOfNormalTables) { + SVnode* pVnodeObj = pVnode; + SVnodeCfg* pConf = &pVnodeObj->config; if (dbname) { *dbname = pConf->dbname; @@ -431,7 +431,7 @@ void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *num } } -int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList) { +int32_t vnodeGetTableList(void* pVnode, int8_t type, SArray* pList) { if (type == TSDB_SUPER_TABLE) { return vnodeGetStbIdList(pVnode, 0, pList); } else { @@ -555,7 +555,7 @@ static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { return TSDB_CODE_SUCCESS; } -#ifndef TD_ENTERPRISE +#ifdef TD_ENTERPRISE #define TK_LOG_STB_NUM 19 static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", "data_dir", @@ -673,12 +673,12 @@ void *vnodeGetIdx(void *pVnode) { return NULL; } - return metaGetIdx(((SVnode *)pVnode)->pMeta); + return metaGetIdx(((SVnode*)pVnode)->pMeta); } void *vnodeGetIvtIdx(void *pVnode) { if (pVnode == NULL) { return NULL; } - return metaGetIvtIdx(((SVnode *)pVnode)->pMeta); + return metaGetIvtIdx(((SVnode*)pVnode)->pMeta); } From f391462e34d58983f11f80c6e4bdcdde54ce8e0f Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 10 Jul 2023 20:01:50 +0800 Subject: [PATCH 5/7] chore: more code --- source/dnode/vnode/src/vnd/vnodeQuery.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 5170e25a1c..ed51301db8 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -598,7 +598,7 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { } #endif -static bool vnodeTimeSeriesStbFilter(void *arg1, void *arg2) { +static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { SVnode *pVnode = (SVnode *)arg1; if (metaTbInFilterCache(pVnode, *(tb_uid_t *)(arg2), 0)) { @@ -621,7 +621,7 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) { #endif if ((!tbFilterSize && vnodeGetStbIdList(pVnode, 0, suidList) < 0) || - (tbFilterSize && vnodeGetStbIdListByFilter(pVnode, 0, suidList, vnodeTimeSeriesStbFilter, pVnode) < 0)) { + (tbFilterSize && vnodeGetStbIdListByFilter(pVnode, 0, suidList, vnodeTimeSeriesFilter, pVnode) < 0)) { qError("vgId:%d, failed to get stb id list error: %s", TD_VID(pVnode), terrstr()); taosArrayDestroy(suidList); return TSDB_CODE_FAILED; From 14b438ec64c326232027c5f657f6ccc629beb5f2 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 11 Jul 2023 07:02:36 +0800 Subject: [PATCH 6/7] chore: bug fix --- source/dnode/vnode/src/vnd/vnodeQuery.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index ed51301db8..f8c50fb9f4 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -580,6 +580,7 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { char *dbName = strchr(pVnode->config.dbname, '.'); + ++dbName; if (!dbName || 0 != strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { return 0; } From d5435926c2ff6059e74c8b03aaaa15d913bd8591 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 11 Jul 2023 12:37:01 +0800 Subject: [PATCH 7/7] chore: more code --- source/dnode/vnode/src/meta/metaCache.c | 15 +++++++-------- source/dnode/vnode/src/vnd/vnodeQuery.c | 3 +-- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index c1a4b5d75b..6918634b5d 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -68,7 +68,7 @@ struct SMetaCache { } STbGroupResCache; struct STbFilterCache { - SHashObj* pTkLogStb; + SHashObj* pStb; } STbFilterCache; }; @@ -172,9 +172,8 @@ int32_t metaCacheOpen(SMeta* pMeta) { taosHashSetFreeFp(pCache->STbGroupResCache.pTableEntry, freeCacheEntryFp); taosThreadMutexInit(&pCache->STbGroupResCache.lock, NULL); - pCache->STbFilterCache.pTkLogStb = - taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - if (pCache->STbFilterCache.pTkLogStb == NULL) { + pCache->STbFilterCache.pStb = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pCache->STbFilterCache.pStb == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err2; } @@ -204,7 +203,7 @@ void metaCacheClose(SMeta* pMeta) { taosThreadMutexDestroy(&pMeta->pCache->STbGroupResCache.lock); taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry); - taosHashCleanup(pMeta->pCache->STbFilterCache.pTkLogStb); + taosHashCleanup(pMeta->pCache->STbFilterCache.pStb); taosMemoryFree(pMeta->pCache); pMeta->pCache = NULL; @@ -897,7 +896,7 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { bool metaTbInFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; - if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pTkLogStb, &suid, sizeof(suid))) { + if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid))) { return true; } @@ -908,7 +907,7 @@ int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; if (type == 0) { - return taosHashPut(pMeta->pCache->STbFilterCache.pTkLogStb, &suid, sizeof(suid), NULL, 0); + return taosHashPut(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid), NULL, 0); } return 0; @@ -917,7 +916,7 @@ int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) { int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) { SMeta* pMeta = ((SVnode*)pVnode)->pMeta; if (type == 0) { - return taosHashGetSize(pMeta->pCache->STbFilterCache.pTkLogStb); + return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb); } return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index f8c50fb9f4..51f4cee40c 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -580,8 +580,7 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info", // exclude stbs of taoskeeper log static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { char *dbName = strchr(pVnode->config.dbname, '.'); - ++dbName; - if (!dbName || 0 != strncmp(dbName, "log", TSDB_DB_NAME_LEN)) { + if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) { return 0; } int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0);