From c1794e02527fb8ddb1b4e7655026db66178ff1c3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 7 Nov 2022 09:38:44 +0800 Subject: [PATCH 1/9] refactor: do some internal refactor. --- source/libs/executor/src/sortoperator.c | 29 +++++++++++++++---------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 7abf05e7d6..db9916da76 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -479,24 +479,31 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort SExecTaskInfo* pTaskInfo) { SGroupSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) { + if (pInfo == NULL || pOperator == NULL) { goto _error; } + SExprSupp* pSup = &pOperator->exprSupp; SDataBlockDescNode* pDescNode = pSortPhyNode->node.pOutputDataBlockDesc; int32_t numOfCols = 0; - SSDataBlock* pResBlock = createResDataBlock(pDescNode); SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); + pSup->pExprInfo = pExprInfo; + pSup->numOfExprs = numOfCols; + + initResultSizeInfo(&pOperator->resultInfo, 1024); + pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); + + pInfo->binfo.pRes = createResDataBlock(pDescNode); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + int32_t numOfOutputCols = 0; int32_t code = extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo); - - pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); - pInfo->binfo.pRes = pResBlock; - - initResultSizeInfo(&pOperator->resultInfo, 1024); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); @@ -505,8 +512,6 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->exprSupp.numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, @@ -520,8 +525,10 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort return pOperator; _error: - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pInfo); + pTaskInfo->code = code; + if (pInfo != NULL) { + destroyGroupSortOperatorInfo(pInfo); + } taosMemoryFree(pOperator); return NULL; } From be5b72b25df51ab7d7fbabcd8c1ee2adc1e92b76 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 7 Nov 2022 10:17:42 +0800 Subject: [PATCH 2/9] enh(query): add cache for table meta entry in table scan. --- source/dnode/vnode/src/meta/metaQuery.c | 6 +-- source/libs/executor/inc/executorimpl.h | 5 ++- source/libs/executor/src/scanoperator.c | 59 ++++++++++++++++--------- 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 43e80b99d4..4aabd39800 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -153,7 +153,7 @@ bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { SMeta *pMeta = pReader->pMeta; - int64_t version; + int64_t version1; // query uid.idx if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pReader->pBuf, &pReader->szBuf) < 0) { @@ -161,8 +161,8 @@ int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { return -1; } - version = ((SUidIdxVal *)pReader->pBuf)[0].version; - return metaGetTableEntryByVersion(pReader, version, uid); + version1 = ((SUidIdxVal *)pReader->pBuf)[0].version; + return metaGetTableEntryByVersion(pReader, version1, uid); } int metaGetTableEntryByName(SMetaReader *pReader, const char *name) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index e20c8ee955..320dc22655 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -318,6 +318,7 @@ typedef struct STableScanInfo { int8_t scanMode; SAggOptrPushDownInfo pdInfo; int8_t assignBlockUid; + SLRUCache* pTableMetaEntryCache; // 100 by default } STableScanInfo; typedef struct STableMergeScanInfo { @@ -896,8 +897,8 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SColMatchInfo* pColMatchInfo, SFilterInfo* pFilterInfo); -int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, - SSDataBlock* pBlock, int32_t rows, const char* idStr); +int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, + SSDataBlock* pBlock, int32_t rows, const char* idStr, SLRUCache* pCache); void cleanupAggSup(SAggSupporter* pAggSup); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 937069b5cc..269f1bff5b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -495,51 +495,68 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction SET_REVERSE_SCAN_FLAG(pTableScanInfo); switchCtxOrder(pCtx, numOfOutput); - // setupQueryRangeForReverseScan(pTableScanInfo); - pTableScanInfo->cond.order = TSDB_ORDER_DESC; STimeWindow* pTWindow = &pTableScanInfo->cond.twindows; TSWAP(pTWindow->skey, pTWindow->ekey); } -int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, - SSDataBlock* pBlock, int32_t rows, const char* idStr) { +typedef struct STableCachedVal { + const char* pName; + STag* pTags; +} STableCachedVal; + +int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, + SSDataBlock* pBlock, int32_t rows, const char* idStr, SLRUCache* pCache) { // currently only the tbname pseudo column - if (numOfPseudoExpr <= 0) { + if (numOfExpr <= 0) { return TSDB_CODE_SUCCESS; } + int32_t code = 0; + // backup the rows int32_t backupRows = pBlock->info.rows; pBlock->info.rows = rows; - SMetaReader mr = {0}; - metaReaderInit(&mr, pHandle->meta, 0); - int32_t code = metaGetTableEntryByUid(&mr, pBlock->info.uid); - metaReaderReleaseLock(&mr); + STableCachedVal val = {0}; + bool needFreeReader = true; - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); - metaReaderClear(&mr); - return terrno; + // 1. check if it is existed in meta cache + LRUHandle *h = taosLRUCacheLookup(pCache, &pBlock->info.uid, sizeof(pBlock->info.uid)); + if (h == NULL) { + SMetaReader mr = {0}; + metaReaderInit(&mr, pHandle->meta, 0); + code = metaGetTableEntryByUid(&mr, pBlock->info.uid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); + metaReaderClear(&mr); + return terrno; + } + + metaReaderReleaseLock(&mr); + val.pName = mr.me.name; + val.pTags = (STag*)mr.me.ctbEntry.pTags; + } else { + STableCachedVal* pVal = taosLRUCacheValue(pCache, h); + val = *pVal; } - for (int32_t j = 0; j < numOfPseudoExpr; ++j) { - SExprInfo* pExpr = &pPseudoExpr[j]; - int32_t dstSlotId = pExpr->base.resSchema.slotId; + for (int32_t j = 0; j < numOfExpr; ++j) { + const SExprInfo* pExpr1 = &pExpr[j]; + int32_t dstSlotId = pExpr1->base.resSchema.slotId; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); colInfoDataCleanup(pColInfoData, pBlock->info.rows); - int32_t functionId = pExpr->pExpr->_function.functionId; + int32_t functionId = pExpr1->pExpr->_function.functionId; // this is to handle the tbname if (fmIsScanPseudoColumnFunc(functionId)) { - setTbNameColData(pBlock, pColInfoData, functionId, mr.me.name); + setTbNameColData(pBlock, pColInfoData, functionId, val.pName); } else { // these are tags STagVal tagVal = {0}; - tagVal.cid = pExpr->base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal(mr.me.ctbEntry.pTags, pColInfoData->info.type, &tagVal); + tagVal.cid = pExpr1->base.pParam[0].pCol->colId; + const char* p = metaGetTableTagVal(val.pTags, pColInfoData->info.type, &tagVal); char* data = NULL; if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { @@ -881,6 +898,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pOperator->exprSupp.numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; + pInfo->pTableMetaEntryCache = taosLRUCacheInit(100, -1, .5); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, getTableScannerExecInfo); From cd29972a4da3c20519e3b391d7fe66600861374c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 7 Nov 2022 11:53:23 +0800 Subject: [PATCH 3/9] enh(query): add cache for table meta. --- source/libs/executor/inc/executorimpl.h | 11 ++- source/libs/executor/src/cachescanoperator.c | 4 +- source/libs/executor/src/scanoperator.c | 82 ++++++++++++++++---- 3 files changed, 79 insertions(+), 18 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 320dc22655..c3fabea442 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -298,6 +298,12 @@ typedef struct { SExprSupp* pExprSup; // expr supporter of aggregate operator } SAggOptrPushDownInfo; +typedef struct STableMetaCacheInfo { + SLRUCache* pTableMetaEntryCache; // 100 by default + uint64_t metaFetch; + uint64_t cacheHit; +} STableMetaCacheInfo; + typedef struct STableScanInfo { STsdbReader* dataReader; SReadHandle readHandle; @@ -318,7 +324,7 @@ typedef struct STableScanInfo { int8_t scanMode; SAggOptrPushDownInfo pdInfo; int8_t assignBlockUid; - SLRUCache* pTableMetaEntryCache; // 100 by default + STableMetaCacheInfo metaCache; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -327,7 +333,6 @@ typedef struct STableMergeScanInfo { int32_t tableEndIndex; bool hasGroupId; uint64_t groupId; - SArray* dataReaders; // array of tsdbReaderT* SArray* queryConds; // array of queryTableDataCond STsdbReader* pReader; SReadHandle readHandle; @@ -898,7 +903,7 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SColMatchInfo* pColMatchInfo, SFilterInfo* pFilterInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, - SSDataBlock* pBlock, int32_t rows, const char* idStr, SLRUCache* pCache); + SSDataBlock* pBlock, int32_t rows, const char* idStr, STableMetaCacheInfo * pCache); void cleanupAggSup(SAggSupporter* pAggSup); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 95d3c5cf23..76866eedb7 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -172,7 +172,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { SExprSupp* pSup = &pInfo->pseudoExprSup; int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes, - pRes->info.rows, GET_TASKID(pTaskInfo)); + pRes->info.rows, GET_TASKID(pTaskInfo), NULL); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -221,7 +221,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0); code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows, - GET_TASKID(pTaskInfo)); + GET_TASKID(pTaskInfo), NULL); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 269f1bff5b..c049aac5b8 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "executorimpl.h" #include "filter.h" #include "function.h" @@ -339,7 +340,7 @@ static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlo SExprSupp* pSup = &pTableScanInfo->pseudoSup; int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, rows, - GET_TASKID(pTaskInfo)); + GET_TASKID(pTaskInfo), &pTableScanInfo->metaCache); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -505,8 +506,25 @@ typedef struct STableCachedVal { STag* pTags; } STableCachedVal; +static void freeTableCachedVal(void* param) { + if (param == NULL) { + return; + } + + STableCachedVal* pVal = param; + taosMemoryFree((void*)pVal->pName); + taosMemoryFree(pVal->pTags); + taosMemoryFree(pVal); +} + +//const void *key, size_t keyLen, void *value +static void freeCachedMetaItem(const void *key, size_t keyLen, void *value) { + taosMemoryFree((void*)key); + freeTableCachedVal(value); +} + int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, - SSDataBlock* pBlock, int32_t rows, const char* idStr, SLRUCache* pCache) { + SSDataBlock* pBlock, int32_t rows, const char* idStr, STableMetaCacheInfo* pCache) { // currently only the tbname pseudo column if (numOfExpr <= 0) { return TSDB_CODE_SUCCESS; @@ -518,13 +536,13 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int int32_t backupRows = pBlock->info.rows; pBlock->info.rows = rows; + bool freeReader = false; STableCachedVal val = {0}; - bool needFreeReader = true; + + SMetaReader mr = {0}; // 1. check if it is existed in meta cache - LRUHandle *h = taosLRUCacheLookup(pCache, &pBlock->info.uid, sizeof(pBlock->info.uid)); - if (h == NULL) { - SMetaReader mr = {0}; + if (pCache == NULL) { metaReaderInit(&mr, pHandle->meta, 0); code = metaGetTableEntryByUid(&mr, pBlock->info.uid); if (code != TSDB_CODE_SUCCESS) { @@ -534,11 +552,46 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int } metaReaderReleaseLock(&mr); + val.pName = mr.me.name; val.pTags = (STag*)mr.me.ctbEntry.pTags; + + freeReader = true; } else { - STableCachedVal* pVal = taosLRUCacheValue(pCache, h); - val = *pVal; + pCache->metaFetch += 1; + + LRUHandle* h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(pBlock->info.uid)); + if (h == NULL) { + metaReaderInit(&mr, pHandle->meta, 0); + code = metaGetTableEntryByUid(&mr, pBlock->info.uid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); + metaReaderClear(&mr); + return terrno; + } + + metaReaderReleaseLock(&mr); + + STableCachedVal* pVal = taosMemoryMalloc(sizeof(STableCachedVal)); + pVal->pName = strdup(mr.me.name); + STag* pTag = (STag*)mr.me.ctbEntry.pTags; + + pVal->pTags = taosMemoryMalloc(pTag->len); + memcpy(pVal->pTags, mr.me.ctbEntry.pTags, pTag->len); + + freeReader = true; + int32_t ret = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(uint64_t), pVal, sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW); + if (ret != TAOS_LRU_STATUS_OK) { + qError("failed to put meta into lru cache, code:%d, %s", ret, idStr); + freeTableCachedVal(pVal); + } else { + val = *pVal; + } + } else { + pCache->cacheHit += 1; + STableCachedVal* pVal = taosLRUCacheValue(pCache->pTableMetaEntryCache, h); + val = *pVal; + } } for (int32_t j = 0; j < numOfExpr; ++j) { @@ -581,10 +634,12 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int } } - metaReaderClear(&mr); - // restore the rows pBlock->info.rows = backupRows; + if (freeReader) { + metaReaderClear(&mr); + } + return TSDB_CODE_SUCCESS; } @@ -832,6 +887,7 @@ static void destroyTableScanOperatorInfo(void* param) { taosArrayDestroy(pTableScanInfo->matchInfo.pList); } + taosLRUCacheCleanup(pTableScanInfo->metaCache.pTableMetaEntryCache); cleanupExprSupp(&pTableScanInfo->pseudoSup); taosMemoryFreeClear(param); } @@ -898,7 +954,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pOperator->exprSupp.numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pInfo->pTableMetaEntryCache = taosLRUCacheInit(100, -1, .5); + pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(100, -1, .5); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, getTableScannerExecInfo); @@ -1650,7 +1706,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, - pInfo->pRes->info.rows, GET_TASKID(pTaskInfo)); + pInfo->pRes->info.rows, GET_TASKID(pTaskInfo), NULL); if (code != TSDB_CODE_SUCCESS) { blockDataFreeRes((SSDataBlock*)pBlock); T_LONG_JMP(pTaskInfo->env, code); @@ -4380,7 +4436,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc SExprSupp* pSup = &pTableScanInfo->pseudoSup; int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, - pBlock->info.rows, GET_TASKID(pTaskInfo)); + pBlock->info.rows, GET_TASKID(pTaskInfo), NULL); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } From 8ec132be6e248e6ec39bd6de25d8d1a057cc98f4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 7 Nov 2022 13:25:09 +0800 Subject: [PATCH 4/9] other: add log. --- source/libs/executor/src/scanoperator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c049aac5b8..a8a0867fc5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -592,6 +592,9 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int STableCachedVal* pVal = taosLRUCacheValue(pCache->pTableMetaEntryCache, h); val = *pVal; } + + qDebug("retrieve table meta:%"PRIu64", hit:%"PRIu64 " miss:%"PRIu64", %s", pCache->metaFetch, pCache->cacheHit, + (pCache->metaFetch - pCache->cacheHit), idStr); } for (int32_t j = 0; j < numOfExpr; ++j) { From 558cf28ae79fa9284d3e8ff1ffb6a60294fafd7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 7 Nov 2022 13:26:10 +0800 Subject: [PATCH 5/9] other: add log. --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a8a0867fc5..632ab41eae 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -593,7 +593,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int val = *pVal; } - qDebug("retrieve table meta:%"PRIu64", hit:%"PRIu64 " miss:%"PRIu64", %s", pCache->metaFetch, pCache->cacheHit, + qDebug("retrieve table meta from cache:%"PRIu64", hit:%"PRIu64 " miss:%"PRIu64", %s", pCache->metaFetch, pCache->cacheHit, (pCache->metaFetch - pCache->cacheHit), idStr); } From 2a49fed4ad8badafb55c21e1159c015629795830 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 7 Nov 2022 14:02:23 +0800 Subject: [PATCH 6/9] fix(query): remove invalid free. --- source/libs/executor/src/scanoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 632ab41eae..f82b636ebe 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -519,7 +519,6 @@ static void freeTableCachedVal(void* param) { //const void *key, size_t keyLen, void *value static void freeCachedMetaItem(const void *key, size_t keyLen, void *value) { - taosMemoryFree((void*)key); freeTableCachedVal(value); } From c86427c77170bd0b90dce92c8333f9084bd61025 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 7 Nov 2022 15:27:09 +0800 Subject: [PATCH 7/9] fix(query): enable update buf. --- source/libs/executor/src/scanoperator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 476510fcad..37372c47b2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -950,6 +950,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pOperator->pTaskInfo = pTaskInfo; pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(100, -1, .5); + taosLRUCacheSetStrictCapacity(pInfo->metaCache.pTableMetaEntryCache, false); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, getTableScannerExecInfo); From 088a61bee54d90a286dfc33882a812949d06f7a9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 7 Nov 2022 16:02:53 +0800 Subject: [PATCH 8/9] fix(query): release the handle --- source/libs/executor/src/scanoperator.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 37372c47b2..2a61fe9d2b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -535,6 +535,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int STableCachedVal val = {0}; SMetaReader mr = {0}; + LRUHandle* h = NULL; // 1. check if it is existed in meta cache if (pCache == NULL) { @@ -555,7 +556,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int } else { pCache->metaFetch += 1; - LRUHandle* h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(pBlock->info.uid)); + h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(pBlock->info.uid)); if (h == NULL) { metaReaderInit(&mr, pHandle->meta, 0); code = metaGetTableEntryByUid(&mr, pBlock->info.uid); @@ -569,23 +570,28 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int STableCachedVal* pVal = taosMemoryMalloc(sizeof(STableCachedVal)); pVal->pName = strdup(mr.me.name); - STag* pTag = (STag*)mr.me.ctbEntry.pTags; + pVal->pTags = NULL; - pVal->pTags = taosMemoryMalloc(pTag->len); - memcpy(pVal->pTags, mr.me.ctbEntry.pTags, pTag->len); + // only child table has tag value + if (mr.me.type == TSDB_CHILD_TABLE) { + STag* pTag = (STag*)mr.me.ctbEntry.pTags; + pVal->pTags = taosMemoryMalloc(pTag->len); + memcpy(pVal->pTags, mr.me.ctbEntry.pTags, pTag->len); + } + val = *pVal; freeReader = true; + int32_t ret = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(uint64_t), pVal, sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW); if (ret != TAOS_LRU_STATUS_OK) { qError("failed to put meta into lru cache, code:%d, %s", ret, idStr); freeTableCachedVal(pVal); - } else { - val = *pVal; } } else { pCache->cacheHit += 1; STableCachedVal* pVal = taosLRUCacheValue(pCache->pTableMetaEntryCache, h); val = *pVal; + taosLRUCacheRelease(pCache->pTableMetaEntryCache, h, false); } qDebug("retrieve table meta from cache:%"PRIu64", hit:%"PRIu64 " miss:%"PRIu64", %s", pCache->metaFetch, pCache->cacheHit, From 0e11d344ccae3adac66444faa5180fde57cd56d2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 7 Nov 2022 16:04:13 +0800 Subject: [PATCH 9/9] fix(query): increase the buffer size to 128Kb. --- source/libs/executor/src/scanoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2a61fe9d2b..ee935837d0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -955,7 +955,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pOperator->exprSupp.numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(100, -1, .5); + pInfo->metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024*128, -1, .5); taosLRUCacheSetStrictCapacity(pInfo->metaCache.pTableMetaEntryCache, false); pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo,