From c079b774cc18d0578b9b1c35c4dcade10fdc197f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 16 Dec 2020 16:07:49 +0800 Subject: [PATCH] [TD-2176]: improve the first/last query performance against super table. --- src/client/src/tscFunctionImpl.c | 35 +++++++++++++++----------------- src/query/src/qExecutor.c | 25 ++++++++++------------- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index eee2ff7f75..f313d355f1 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -708,15 +708,14 @@ static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY en return BLK_DATA_ALL_NEEDED; } - return BLK_DATA_ALL_NEEDED; - // TODO pCtx->aOutputBuf is the previous windowRes output buffer, not current unloaded block. so the following filter - // is invalid -// SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); -// if (pInfo->hasResult != DATA_SET_FLAG) { -// return BLK_DATA_ALL_NEEDED; -// } else { // data in current block is not earlier than current result -// return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; -// } + // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->aOutputBuf is + // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); + if (pInfo->hasResult != DATA_SET_FLAG) { + return BLK_DATA_ALL_NEEDED; + } else { // data in current block is not earlier than current result + return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + } } static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { @@ -729,16 +728,14 @@ static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end return BLK_DATA_ALL_NEEDED; } - return BLK_DATA_ALL_NEEDED; - // TODO pCtx->aOutputBuf is the previous windowRes output buffer, not current unloaded block. so the following filter - // is invalid - -// SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); -// if (pInfo->hasResult != DATA_SET_FLAG) { -// return BLK_DATA_ALL_NEEDED; -// } else { -// return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; -// } + // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->aOutputBuf is + // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); + if (pInfo->hasResult != DATA_SET_FLAG) { + return BLK_DATA_ALL_NEEDED; + } else { + return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + } } ////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 68e7ff77ab..0d3b068a4a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -624,9 +624,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRow SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, uid); if (pResultRow == NULL) { *newWind = false; - - // no master scan, no result generated means error occurs - return masterscan? -1:0; + return masterscan? -1:0; // no master scan, no result generated means error occurs } *newWind = true; @@ -2617,10 +2615,11 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) { } int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pWindowResInfo, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) { - SQuery *pQuery = pRuntimeEnv->pQuery; - *status = BLK_DATA_NO_NEEDED; + SQuery *pQuery = pRuntimeEnv->pQuery; + SQueryCostInfo* pCost = &pRuntimeEnv->summary; + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf > 0) { *status = BLK_DATA_ALL_NEEDED; } else { // check if this data block is required to load @@ -2641,7 +2640,6 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey; - STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery); if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow, &pResult) != TSDB_CODE_SUCCESS) { @@ -2665,35 +2663,34 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW if ((*status) == BLK_DATA_NO_NEEDED) { qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pRuntimeEnv->summary.discardBlocks += 1; + pCost->discardBlocks += 1; } else if ((*status) == BLK_DATA_STATIS_NEEDED) { // this function never returns error? tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); - - pRuntimeEnv->summary.loadBlockStatis += 1; + pCost->loadBlockStatis += 1; if (*pStatis == NULL) { // data block statistics does not exist, load data block *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); - pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; + pCost->totalCheckedRows += pBlockInfo->rows; } } else { assert((*status) == BLK_DATA_ALL_NEEDED); // load the data block statistics to perform further filter - pRuntimeEnv->summary.loadBlockStatis += 1; + pCost->loadBlockStatis += 1; tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { // current block has been discard due to filter applied - pRuntimeEnv->summary.discardBlocks += 1; + pCost->discardBlocks += 1; qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); (*status) = BLK_DATA_DISCARD; } - pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; - pRuntimeEnv->summary.loadBlocks += 1; + pCost->totalCheckedRows += pBlockInfo->rows; + pCost->loadBlocks += 1; *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); if (*pDataBlock == NULL) { return terrno;