xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor(query): disable tag filter cache by default.

This commit is contained in:
Haojun Liao 2023-02-02 17:38:21 +08:00
parent edca89b731
commit e9490a3235
4 changed files with 61 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/os/osEnv.h
View File

@ -41,6 +41,7 @@ extern char tsSSE42Enable;
extern char tsAVXEnable; extern char tsAVXEnable;
extern char tsAVX2Enable; extern char tsAVX2Enable;
extern char tsFMAEnable; extern char tsFMAEnable;
extern char tsTagFilterCache;
extern char configDir[]; extern char configDir[];
extern char tsDataDir[]; extern char tsDataDir[];

3
source/common/src/tglobal.c
View File

@ -140,6 +140,7 @@ int32_t tsMaxMemUsedByInsert = 1024;
float tsSelectivityRatio = 1.0; float tsSelectivityRatio = 1.0;
int32_t tsTagFilterResCacheSize = 1024 * 10; int32_t tsTagFilterResCacheSize = 1024 * 10;
char tsTagFilterCache = 0;
// the maximum allowed query buffer size during query processing for each data node. // the maximum allowed query buffer size during query processing for each data node.
// -1 no limit (default) // -1 no limit (default)
@ -351,6 +352,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, 0) != 0) return -1; if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, 0) != 0) return -1;
if (cfgAddBool(pCfg, "FMA", tsFMAEnable, 0) != 0) return -1; if (cfgAddBool(pCfg, "FMA", tsFMAEnable, 0) != 0) return -1;
if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, 0) != 0) return -1; if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, 0) != 0) return -1;
if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, 0) != 0) return -1;
if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1;
@ -731,6 +733,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64;
tsSIMDBuiltins = (bool)cfgGetItem(pCfg, "SIMD-builtins")->bval; tsSIMDBuiltins = (bool)cfgGetItem(pCfg, "SIMD-builtins")->bval;
tsTagFilterCache = (bool)cfgGetItem(pCfg, "tagFilterCache")->bval;
tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;

84
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -2745,48 +2745,48 @@ static void resetScanBlockLastBlockDelIndex(SReaderStatus* pStatus, int32_t orde
} }
} }
static int32_t initOrderCheckInfo(SUidOrderedList* pOrderCheckInfo, STsdbReader* pReader) { //static int32_t initOrderCheckInfo(SUidOrderedList* pOrderCheckInfo, STsdbReader* pReader) {
SReaderStatus* pStatus = &pReader->status; // SReaderStatus* pStatus = &pReader->status;
//
int32_t total = taosHashGetSize(pStatus->pTableMap); // int32_t total = taosHashGetSize(pStatus->pTableMap);
if (total == 0) { // if (total == 0) {
return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
} // }
//
if (pOrderCheckInfo->tableUidList == NULL) { // if (pOrderCheckInfo->tableUidList == NULL) {
pOrderCheckInfo->currentIndex = 0; // pOrderCheckInfo->currentIndex = 0;
pOrderCheckInfo->tableUidList = taosMemoryMalloc(total * sizeof(uint64_t)); // pOrderCheckInfo->tableUidList = taosMemoryMalloc(total * sizeof(uint64_t));
if (pOrderCheckInfo->tableUidList == NULL) { // if (pOrderCheckInfo->tableUidList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; // return TSDB_CODE_OUT_OF_MEMORY;
} // }
//
extractOrderedTableUidList(pOrderCheckInfo, pStatus, pReader->order); // extractOrderedTableUidList(pOrderCheckInfo, pStatus, pReader->order);
uint64_t uid = pOrderCheckInfo->tableUidList[0]; // uint64_t uid = pOrderCheckInfo->tableUidList[0];
pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); // pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid));
} else { // } else {
if (pStatus->pTableIter == NULL) { // it is the last block of a new file // if (pStatus->pTableIter == NULL) { // it is the last block of a new file
pOrderCheckInfo->currentIndex = 0; // pOrderCheckInfo->currentIndex = 0;
uint64_t uid = pOrderCheckInfo->tableUidList[pOrderCheckInfo->currentIndex]; // uint64_t uid = pOrderCheckInfo->tableUidList[pOrderCheckInfo->currentIndex];
pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); // pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid));
//
// the tableMap has already updated, let's also update the order list // // the tableMap has already updated, let's also update the order list
if (pStatus->pTableIter == NULL) { // if (pStatus->pTableIter == NULL) {
void* p = taosMemoryRealloc(pOrderCheckInfo->tableUidList, total * sizeof(uint64_t)); // void* p = taosMemoryRealloc(pOrderCheckInfo->tableUidList, total * sizeof(uint64_t));
if (p == NULL) { // if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; // return TSDB_CODE_OUT_OF_MEMORY;
} // }
//
pOrderCheckInfo->tableUidList = p; // pOrderCheckInfo->tableUidList = p;
extractOrderedTableUidList(pOrderCheckInfo, pStatus, pReader->order); // extractOrderedTableUidList(pOrderCheckInfo, pStatus, pReader->order);
//
uid = pOrderCheckInfo->tableUidList[0]; // uid = pOrderCheckInfo->tableUidList[0];
pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); // pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid));
} // }
} // }
} // }
//
return TSDB_CODE_SUCCESS; // return TSDB_CODE_SUCCESS;
} //}
static void resetTableListIndex(SReaderStatus *pStatus) { static void resetTableListIndex(SReaderStatus *pStatus) {
SUidOrderedList* pList = &pStatus->uidCheckInfo; SUidOrderedList* pList = &pStatus->uidCheckInfo;

24
source/libs/executor/src/executil.c
View File

@ -1074,15 +1074,19 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
return code; return code;
} }
} else { } else {
// try to retrieve the result from meta cache
T_MD5_CTX context = {0};
genTagFilterDigest(pTagCond, &context);
bool acquired = false; T_MD5_CTX context = {0};
metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), res, &acquired);
if (acquired) { if (tsTagFilterCache) {
qDebug("retrieve table uid list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(res)); // try to retrieve the result from meta cache
goto _end; genTagFilterDigest(pTagCond, &context);
bool acquired = false;
metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), res, &acquired);
if (acquired) {
qDebug("retrieve table uid list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(res));
goto _end;
}
} }
if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table
@ -1118,7 +1122,9 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
memcpy(pPayload + sizeof(int32_t), taosArrayGet(res, 0), numOfTables * sizeof(uint64_t)); memcpy(pPayload + sizeof(int32_t), taosArrayGet(res, 0), numOfTables * sizeof(uint64_t));
} }
metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1); if (tsTagFilterCache) {
metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1);
}
} }
_end: _end: