chore: code optimization for time series

This commit is contained in:
kailixu 2023-10-07 00:17:21 +08:00
parent c59498d90a
commit c70a1b77e6
6 changed files with 50 additions and 41 deletions

View File

@ -132,8 +132,8 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t payloadLen); int32_t payloadLen);
bool metaTbInFilterCache(void *pVnode, void* key, int8_t type); bool metaTbInFilterCache(void *pVnode, const void* key, int8_t type);
int32_t metaPutTbToFilterCache(void *pVnode, void* key, int8_t type); int32_t metaPutTbToFilterCache(void *pVnode, const void* key, int8_t type);
int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type); int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type);
int32_t metaInitTbFilterCache(void *pVnode); int32_t metaInitTbFilterCache(void *pVnode);

View File

@ -98,6 +98,9 @@ typedef struct SQueryNode SQueryNode;
#define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME "vnode.json"
#define VND_INFO_FNAME_TMP "vnode_tmp.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json"
#define TK_LOG_STB_NUM 19
#define TK_AUDIT_STB_NUM 1
// vnd.h // vnd.h
typedef int32_t (*_query_reseek_func_t)(void* pQHandle); typedef int32_t (*_query_reseek_func_t)(void* pQHandle);
struct SQueryNode { struct SQueryNode {

View File

@ -14,6 +14,11 @@
*/ */
#include "meta.h" #include "meta.h"
#ifdef TD_ENTERPRISE
extern const char* tkLogStb[];
extern const char* tkAuditStb[];
#endif
#define TAG_FILTER_RES_KEY_LEN 32 #define TAG_FILTER_RES_KEY_LEN 32
#define META_CACHE_BASE_BUCKET 1024 #define META_CACHE_BASE_BUCKET 1024
#define META_CACHE_STATS_BUCKET 16 #define META_CACHE_STATS_BUCKET 16
@ -185,9 +190,6 @@ int32_t metaCacheOpen(SMeta* pMeta) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2; goto _err2;
} }
if ((code = metaInitTbFilterCache(pMeta->pVnode)) != 0) {
goto _err2;
}
pMeta->pCache = pCache; pMeta->pCache = pCache;
return code; return code;
@ -905,7 +907,7 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool metaTbInFilterCache(void* pVnode, void* key, int8_t type) { bool metaTbInFilterCache(void* pVnode, const void* key, int8_t type) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta; SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) { if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) {
@ -919,7 +921,7 @@ bool metaTbInFilterCache(void* pVnode, void* key, int8_t type) {
return false; return false;
} }
int32_t metaPutTbToFilterCache(void* pVnode, void* key, int8_t type) { int32_t metaPutTbToFilterCache(void* pVnode, const void* key, int8_t type) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta; SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
if (type == 0) { if (type == 0) {
@ -939,4 +941,30 @@ int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) {
return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb); return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb);
} }
return 0; return 0;
} }
int32_t metaInitTbFilterCache(void* pVnode) {
#ifdef TD_ENTERPRISE
int32_t tbNum = 0;
const char** pTbArr = NULL;
const char* dbName = NULL;
if (!(dbName = strchr(((SVnode*)pVnode)->config.dbname, '.'))) return 0;
if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) {
tbNum = TK_LOG_STB_NUM;
pTbArr = (const char**)&tkLogStb;
} else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) {
tbNum = TK_AUDIT_STB_NUM;
pTbArr = (const char**)&tkAuditStb;
}
if (tbNum && pTbArr) {
for (int32_t i = 0; i < tbNum; ++i) {
if (metaPutTbToFilterCache(pVnode, pTbArr[i], 1) != 0) {
return terrno ? terrno : -1;
}
}
}
#else
#endif
return 0;
}

View File

@ -764,6 +764,8 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
} }
metaReaderClear(&mr); metaReaderClear(&mr);
bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1);
// build SMetaEntry // build SMetaEntry
SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; SVnodeStats *pStats = &pMeta->pVnode->config.vndStats;
me.version = ver; me.version = ver;
@ -800,8 +802,8 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
#endif #endif
++pStats->numOfCTables; ++pStats->numOfCTables;
if (!metaTbInFilterCache(pMeta->pVnode, pReq->ctb.stbName, 1)) { if (!sysTbl) {
int32_t nCols = 0; int32_t nCols = 0;
metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols); metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols);
pStats->numOfTimeSeries += nCols - 1; pStats->numOfTimeSeries += nCols - 1;
@ -826,7 +828,7 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
if (metaHandleEntry(pMeta, &me) < 0) goto _err; if (metaHandleEntry(pMeta, &me) < 0) goto _err;
metaTimeSeriesNotifyCheck(pMeta); if(!sysTbl) metaTimeSeriesNotifyCheck(pMeta);
if (pMetaRsp) { if (pMetaRsp) {
*pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp)); *pMetaRsp = taosMemoryCalloc(1, sizeof(STableMetaRsp));

View File

@ -399,6 +399,10 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
goto _err; goto _err;
} }
if (metaInitTbFilterCache(pVnode) != 0) {
goto _err;
}
if (metaUpgrade(pVnode, &pVnode->pMeta) < 0) { if (metaUpgrade(pVnode, &pVnode->pMeta) < 0) {
vError("vgId:%d, failed to upgrade meta since %s", TD_VID(pVnode), tstrerror(terrno)); vError("vgId:%d, failed to upgrade meta since %s", TD_VID(pVnode), tstrerror(terrno));
} }

View File

@ -566,9 +566,7 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
} }
#ifdef TD_ENTERPRISE #ifdef TD_ENTERPRISE
#define TK_LOG_STB_NUM 19 const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info",
#define TK_AUDIT_STB_NUM 1
static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info",
"data_dir", "data_dir",
"dnodes_info", "dnodes_info",
"d_info", "d_info",
@ -587,7 +585,7 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info",
"temp_dir", "temp_dir",
"vgroups_info", "vgroups_info",
"vnodes_role"}; "vnodes_role"};
static const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"}; const char *tkAuditStb[TK_AUDIT_STB_NUM] = {"operations"};
// exclude stbs of taoskeeper log // exclude stbs of taoskeeper log
static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) { static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) {
@ -619,32 +617,6 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) {
return tbSize; return tbSize;
} }
int32_t metaInitTbFilterCache(void *pVnode) {
int32_t tbNum = 0;
const char **pTbArr = NULL;
const char *dbName = NULL;
if (!(dbName = strchr(((SVnode *)pVnode)->config.dbname, '.'))) return 0;
if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) {
tbNum = TK_LOG_STB_NUM;
pTbArr = (const char **)&tkLogStb;
} else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) {
tbNum = TK_AUDIT_STB_NUM;
pTbArr = (const char **)&tkAuditStb;
}
if (tbNum && pTbArr) {
for (int32_t i = 0; i < tbNum; ++i) {
if (metaPutTbToFilterCache(pVnode, &pTbArr[i], strlen(pTbArr[i])) != 0) {
return terrno ? terrno : -1;
}
}
}
return 0;
}
#else
int32_t metaInitTbFilterCache(void *pVnode) { return 0; }
#endif #endif
static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) { static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) {