diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 93e4f18ae9..9260ec4b1e 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -213,6 +213,7 @@ typedef struct SIndexMetaArg { void* idx; void* ivtIdx; uint64_t suid; + int (*metaFilterFunc)(void* metaEx, void* param, SArray* result); } SIndexMetaArg; typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus; diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index f20ba287de..7c5ca628cd 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -55,6 +55,16 @@ extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, SNode **pOtherCond); +typedef struct SMetaFltParam { + tb_uid_t suid; + int16_t cid; + int16_t type; + void *val; + bool reverse; + bool equal; + int (*filterFunc)(void *a, void *b, int16_t type); +} SMetaFltParam; + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 8afeb05716..ddaf0d9bbc 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -26,6 +26,7 @@ #include "tfs.h" #include "wal.h" +#include "filter.h" #include "tcommon.h" #include "tfs.h" #include "tgrant.h" @@ -126,10 +127,10 @@ int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, in int32_t payloadLen, double selectivityRatio); int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); -int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid); -int32_t metaGetCachedTbGroup(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray** pList); -int32_t metaPutTbGroupToCache(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, - int32_t payloadLen); +int32_t metaTbGroupCacheClear(SMeta *pMeta, uint64_t suid); +int32_t metaGetCachedTbGroup(SMeta *pMeta, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); +int32_t metaPutTbGroupToCache(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, + int32_t payloadLen); int64_t metaGetTbNum(SMeta *pMeta); int64_t metaGetNtbNum(SMeta *pMeta); @@ -184,7 +185,8 @@ typedef struct STsdbReader STsdbReader; #define CACHESCAN_RETRIEVE_LAST 0x8 int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables); + SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, + SHashObj **pIgnoreTables); int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); void tsdbReaderClose(STsdbReader *pReader); @@ -199,9 +201,9 @@ void *tsdbGetIdx(SMeta *pMeta); void *tsdbGetIvtIdx(SMeta *pMeta); uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader); -int64_t tsdbGetLastTimestamp(SVnode* pVnode, void* pTableList, int32_t numOfTables, const char* pIdStr); +int64_t tsdbGetLastTimestamp(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); -int32_t tsdbReuseCacherowsReader(void* pReader, void* pTableIdList, int32_t numOfTables); +int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, @@ -266,10 +268,10 @@ int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); -bool tqNextBlockInWal(STqReader* pReader, const char* idstr); -bool tqNextBlockImpl(STqReader *pReader, const char* idstr); +bool tqNextBlockInWal(STqReader *pReader, const char *idstr); +bool tqNextBlockImpl(STqReader *pReader, const char *idstr); -int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, const char* id); +int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, const char *id); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock** pRes, const char* idstr); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 4cc4176fda..48bfc5d86f 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1083,8 +1083,10 @@ typedef struct { int32_t vLen; } SIdxCursor; -int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; +int32_t metaFilterCreateTime(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + int32_t ret = 0; SIdxCursor *pCursor = NULL; pCursor = (SIdxCursor *)taosMemoryCalloc(1, sizeof(SIdxCursor)); @@ -1141,9 +1143,11 @@ END: return ret; } -int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; - char *buf = NULL; +int32_t metaFilterTableName(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + int32_t ret = 0; + char *buf = NULL; STagIdxKey *pKey = NULL; int32_t nKey = 0; @@ -1206,9 +1210,11 @@ END: return ret; } -int32_t metaFilterTtl(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { - int32_t ret = 0; - char *buf = NULL; +int32_t metaFilterTtl(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + int32_t ret = 0; + char *buf = NULL; STtlIdxKey *pKey = NULL; int32_t nKey = 0; @@ -1235,7 +1241,10 @@ END: // impl later return 0; } -int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { +int32_t metaFilterTableIds(void *meta, void *arg, SArray *pUids) { + SMeta *pMeta = meta; + SMetaFltParam *param = arg; + SMetaEntry oStbEntry = {0}; int32_t ret = -1; char *buf = NULL; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c742794498..ec23f7b724 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -436,7 +436,6 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) { taosMemoryFree(payload); } - static void genTbGroupDigest(const SNode* pGroup, uint8_t* filterDigest, T_MD5_CTX* pContext) { char* payload = NULL; int32_t len = 0; @@ -493,18 +492,19 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf if (tsTagFilterCache) { SNodeListNode* listNode = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); listNode->pNodeList = group; - genTbGroupDigest((SNode *)listNode, digest, &context); + genTbGroupDigest((SNode*)listNode, digest, &context); nodesFree(listNode); pAPI->metaFn.getCachedTableList(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), &tableList); if (tableList) { taosArrayDestroy(pTableListInfo->pTableList); pTableListInfo->pTableList = tableList; - qDebug("retrieve tb group list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(pTableListInfo->pTableList)); + qDebug("retrieve tb group list from cache, numOfTables:%d", + (int32_t)taosArrayGetSize(pTableListInfo->pTableList)); goto end; } } - + pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo)); for (int32_t i = 0; i < rows; ++i) { STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i); @@ -634,7 +634,7 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf tableList = taosArrayDup(pTableListInfo->pTableList, NULL); pAPI->metaFn.putTableListIntoCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); } - + // int64_t st2 = taosGetTimestampUs(); // qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); @@ -839,7 +839,6 @@ static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, S return -1; } - static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode, SStorageAPI* pStorageAPI) { SSDataBlock* pResBlock = createDataBlock(); @@ -1518,7 +1517,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu return TSDB_CODE_OUT_OF_MEMORY; } - SHashObj *pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + SHashObj* pSelectFuncs = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); for (int32_t i = 0; i < numOfOutput; ++i) { const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; if ((strcmp(pName, "_select_value") == 0) || (strcmp(pName, "_group_key") == 0)) { @@ -1845,9 +1844,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) { return taosArrayGetSize(pTableList->pTableList); } -uint64_t tableListGetSuid(const STableListInfo* pTableList) { - return pTableList->idInfo.suid; -} +uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->idInfo.suid; } STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) { if (taosArrayGetSize(pTableList->pTableList) == 0) { diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 8ef134444f..6f3f48610c 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -12,7 +12,6 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common -# PUBLIC vnode PUBLIC nodes PUBLIC scalar PUBLIC function diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 7b3013362d..2dce931ad4 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "filter.h" #include "index.h" #include "indexComm.h" #include "indexInt.h"