diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 56d194252f..36aac2e5d4 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1261,7 +1261,7 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { goto END; } - bool first = true; + int count = 0; int32_t valid = 0; while (1) { void *entryKey = NULL, *entryVal = NULL; @@ -1269,24 +1269,26 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal); if (valid < 0) { - tdbFree(entryVal); break; } STagIdxKey *p = entryKey; if (p == NULL) break; - if (p->type != pCursor->type) { - if (first) { - valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); - if (valid < 0) break; - continue; - } else { + + if (count >= 2) { + if (p->type != pCursor->type || p->suid != pCursor->suid) { break; } + } else { + count++; + if (p->type != pCursor->type) { + valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); + if (valid < 0) { + break; + } else { + continue; + } + } } - if (p->suid != pKey->suid) { - break; - } - first = false; int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); if (cmp == 0) { // match @@ -1298,11 +1300,13 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { } taosArrayPush(pUids, &tuid); } else if (cmp == 1) { + if (count >= 2) break; // not match but should continue to iter } else { // not match and no more result - break; + if (count >= 2) break; } + count++; valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); if (valid < 0) { break; @@ -1335,7 +1339,7 @@ static int32_t metaGetTableTagByUid(SMeta *pMeta, int64_t suid, int64_t uid, voi return ret; } int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList, SHashObj *tags) { - const int32_t LIMIT = 128; + const int32_t LIMIT = 4096; int32_t isLock = false; int32_t sz = uidList ? taosArrayGetSize(uidList) : 0; @@ -1377,7 +1381,6 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj int64_t *uid = taosArrayGet(uidList, i); taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); } - return 0; } while (1) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 04b284a642..54c913df3b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -43,7 +43,14 @@ typedef struct tagFilterAssist { SArray* cInfoList; } tagFilterAssist; -static int32_t removeInvalidTable(SArray* uids, SHashObj* tags); +typedef enum { + FILTER_NO_LOGIC = 1, + FILTER_AND, + FILTER_OTHER, +} FilterCondType; + +static FilterCondType checkTagCond(SNode* cond); +static int32_t removeInvalidTable(SArray* uids, SHashObj* tags); static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond, SHashObj* tags); static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond); static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, @@ -392,7 +399,8 @@ static int32_t createResultData(SDataType* pType, int32_t numOfRows, SScalarPara return TSDB_CODE_SUCCESS; } -static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* uidList, SNode* pTagCond) { +static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* uidList, SNode* pTagCond, + SIdxFltStatus status) { int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; @@ -430,14 +438,22 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* // int64_t stt = taosGetTimestampUs(); tags = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + FilterCondType condType = checkTagCond(pTagCond); + int32_t filter = optimizeTbnameInCond(metaHandle, suid, uidList, pTagCond, tags); if (filter == -1) { - code = metaGetTableTags(metaHandle, suid, uidList, tags); + if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) { + code = metaGetTableTagsByUids(metaHandle, suid, uidList, tags); + } else { + code = metaGetTableTags(metaHandle, suid, uidList, tags); + } if (code != TSDB_CODE_SUCCESS) { qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid); terrno = code; goto end; } + } else { + qDebug("succ to get table tags from meta by tbname in cond, suid:%" PRIu64, suid); } if (suid != 0) { removeInvalidTable(uidList, tags); @@ -842,6 +858,15 @@ static int tableUidCompare(const void* a, const void* b) { return u1 < u2 ? -1 : 1; } +static FilterCondType checkTagCond(SNode* cond) { + if (nodeType(cond) == QUERY_NODE_OPERATOR) { + return FILTER_NO_LOGIC; + } + if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { + return FILTER_AND; + } + return FILTER_OTHER; +} static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond, SHashObj* tags) { int32_t ret = -1; if (nodeType(cond) == QUERY_NODE_OPERATOR) { @@ -983,13 +1008,14 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) { taosMemoryFree(payload); } -static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* res, SNode* pTagCond, void* metaHandle) { +static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* res, SNode* pTagCond, void* metaHandle, + SIdxFltStatus status) { if (pTagCond == NULL) { return TSDB_CODE_SUCCESS; } terrno = TDB_CODE_SUCCESS; - SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond); + SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond, status); if (terrno != TDB_CODE_SUCCESS) { colDataDestroy(pColInfoData); taosMemoryFreeClear(pColInfoData); @@ -1034,12 +1060,13 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, pListInfo->suid = pScanNode->suid; SArray* res = taosArrayInit(8, sizeof(uint64_t)); + SIdxFltStatus status = SFLT_NOT_INDEX; if (pScanNode->tableType != TSDB_SUPER_TABLE) { if (metaIsTableExist(metaHandle, tableUid)) { taosArrayPush(res, &tableUid); } - code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle); + code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle, status); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1064,7 +1091,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SIndexMetaArg metaArg = { .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid}; - SIdxFltStatus status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, res, &status); if (code != 0 || status == SFLT_NOT_INDEX) { qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); @@ -1073,7 +1099,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } } - code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle); + code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle, status); if (code != TSDB_CODE_SUCCESS) { return code; }