From bf70cdb474dd32075058da3f787eb5be480cf712 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Jan 2023 19:27:16 +0800 Subject: [PATCH 1/7] enh(query): opt filter perf. --- source/libs/executor/src/executorimpl.c | 102 +++++++++++++++++++----- 1 file changed, 84 insertions(+), 18 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 149efef884..cdb924290f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -854,32 +854,100 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD } else if (status == FILTER_RESULT_NONE_QUALIFIED) { pBlock->info.rows = 0; } else { - SSDataBlock* px = createOneDataBlock(pBlock, true); + int32_t bmLen = BitmapLen(totalRows); + char* pBitmap = NULL; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); // it is a reserved column for scalar function, and no data in this column yet. - if (pDst->pData == NULL || pSrc->pData == NULL) { + if (pDst->pData == NULL) { continue; } - colInfoDataCleanup(pDst, pBlock->info.rows); - int32_t numOfRows = 0; - for (int32_t j = 0; j < totalRows; ++j) { - if (((int8_t*)p->pData)[j] == 0) { - continue; - } - if (colDataIsNull_s(pSrc, j)) { - colDataAppendNULL(pDst, numOfRows); - } else { - colDataAppend(pDst, numOfRows, colDataGetData(pSrc, j), false); - } - numOfRows += 1; - } + switch (pDst->info.type) { + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_NCHAR: + break; + default: + if (pBitmap == NULL) { + pBitmap = taosMemoryCalloc(1, bmLen); + } + memcpy(pBitmap, pDst->nullbitmap, bmLen); + memset(pDst->nullbitmap, 0, bmLen); + + int32_t j = 0; + + switch (pDst->info.type) { + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + while (j < totalRows) { + if (((int8_t*)p->pData)[j] == 0) { + j += 1; + continue; + } + + if (colDataIsNull_f(pBitmap, j)) { + colDataAppendNULL(pDst, numOfRows); + } else { + ((int64_t*)pDst->pData)[numOfRows] = ((int64_t*)pDst->pData)[j]; + } + numOfRows += 1; + } + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + while (j < totalRows) { + if (((int8_t*)p->pData)[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataAppendNULL(pDst, numOfRows); + } else { + ((int32_t*)pDst->pData)[numOfRows++] = ((int32_t*)pDst->pData)[j]; + } + numOfRows += 1; + } + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + while (j < totalRows) { + if (((int8_t*)p->pData)[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataAppendNULL(pDst, numOfRows); + } else { + ((int16_t*)pDst->pData)[numOfRows++] = ((int16_t*)pDst->pData)[j]; + } + numOfRows += 1; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + while (j < totalRows) { + if (((int8_t*)p->pData)[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataAppendNULL(pDst, numOfRows); + } else { + ((int8_t*)pDst->pData)[numOfRows] = ((int8_t*)pDst->pData)[j]; + } + numOfRows += 1; + } + break; + } + }; // todo this value can be assigned directly if (pBlock->info.rows == totalRows) { @@ -888,8 +956,6 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD ASSERT(pBlock->info.rows == numOfRows); } } - - blockDataDestroy(px); // fix memory leak } } From 05a2eeed44f7cf368162d1ffb4bdaa9c874ed123 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Jan 2023 19:36:53 +0800 Subject: [PATCH 2/7] fix(query): fix memory leak. --- source/libs/executor/src/executorimpl.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index cdb924290f..ff8f34b0c6 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -956,6 +956,10 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD ASSERT(pBlock->info.rows == numOfRows); } } + + if (pBitmap != NULL) { + taosMemoryFree(pBitmap); + } } } From 5c0fd801955c285f1e4e844754b09ad1fcdc34ce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Jan 2023 19:45:41 +0800 Subject: [PATCH 3/7] fix(query): fix memory leak. --- source/libs/executor/src/executorimpl.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ff8f34b0c6..33661e301f 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -847,6 +847,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD return; } + int8_t* pIndicator = p->pData; int32_t totalRows = pBlock->info.rows; if (status == FILTER_RESULT_ALL_QUALIFIED) { @@ -886,7 +887,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: while (j < totalRows) { - if (((int8_t*)p->pData)[j] == 0) { + if (pIndicator[j] == 0) { j += 1; continue; } @@ -897,13 +898,14 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD ((int64_t*)pDst->pData)[numOfRows] = ((int64_t*)pDst->pData)[j]; } numOfRows += 1; + j += 1; } break; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: while (j < totalRows) { - if (((int8_t*)p->pData)[j] == 0) { + if (pIndicator[j] == 0) { j += 1; continue; } @@ -913,12 +915,13 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD ((int32_t*)pDst->pData)[numOfRows++] = ((int32_t*)pDst->pData)[j]; } numOfRows += 1; + j += 1; } break; case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: while (j < totalRows) { - if (((int8_t*)p->pData)[j] == 0) { + if (pIndicator[j] == 0) { j += 1; continue; } @@ -928,13 +931,14 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD ((int16_t*)pDst->pData)[numOfRows++] = ((int16_t*)pDst->pData)[j]; } numOfRows += 1; + j += 1; } break; case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: while (j < totalRows) { - if (((int8_t*)p->pData)[j] == 0) { + if (pIndicator[j] == 0) { j += 1; continue; } @@ -944,6 +948,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD ((int8_t*)pDst->pData)[numOfRows] = ((int8_t*)pDst->pData)[j]; } numOfRows += 1; + j += 1; } break; } From 00c2d382b131608ac218bfc25804c8d5a3043302 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Jan 2023 22:45:02 +0800 Subject: [PATCH 4/7] fix(query): opt filter perf. --- source/common/src/tdatablock.c | 2 +- source/libs/executor/src/executorimpl.c | 192 +++++++++++++----------- 2 files changed, 105 insertions(+), 89 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 2b43229b83..9c8ffa11fe 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -111,7 +111,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con uint32_t len = pColumnInfoData->varmeta.length; pColumnInfoData->varmeta.offset[currentRow] = len; - memcpy(pColumnInfoData->pData + len, pData, dataLen); + memmove(pColumnInfoData->pData + len, pData, dataLen); pColumnInfoData->varmeta.length += dataLen; } else { memcpy(pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow, pData, pColumnInfoData->info.bytes); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 33661e301f..a8c02b3700 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -847,7 +847,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD return; } - int8_t* pIndicator = p->pData; + int8_t* pIndicator = (int8_t*)p->pData; int32_t totalRows = pBlock->info.rows; if (status == FILTER_RESULT_ALL_QUALIFIED) { @@ -856,7 +856,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD pBlock->info.rows = 0; } else { int32_t bmLen = BitmapLen(totalRows); - char* pBitmap = NULL; + char* pBitmap = NULL; + int32_t maxRows = 0; size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { @@ -868,100 +869,115 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD int32_t numOfRows = 0; - switch (pDst->info.type) { - case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_NCHAR: - break; - default: - if (pBitmap == NULL) { - pBitmap = taosMemoryCalloc(1, bmLen); + if (IS_VAR_DATA_TYPE(pDst->info.type)) { + int32_t j = 0; + while(j < totalRows) { + if (pIndicator[j] == 0) { + continue; } - memcpy(pBitmap, pDst->nullbitmap, bmLen); - memset(pDst->nullbitmap, 0, bmLen); - int32_t j = 0; - - switch (pDst->info.type) { - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - - if (colDataIsNull_f(pBitmap, j)) { - colDataAppendNULL(pDst, numOfRows); - } else { - ((int64_t*)pDst->pData)[numOfRows] = ((int64_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataAppendNULL(pDst, numOfRows); - } else { - ((int32_t*)pDst->pData)[numOfRows++] = ((int32_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataAppendNULL(pDst, numOfRows); - } else { - ((int16_t*)pDst->pData)[numOfRows++] = ((int16_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - while (j < totalRows) { - if (pIndicator[j] == 0) { - j += 1; - continue; - } - if (colDataIsNull_f(pBitmap, j)) { - colDataAppendNULL(pDst, numOfRows); - } else { - ((int8_t*)pDst->pData)[numOfRows] = ((int8_t*)pDst->pData)[j]; - } - numOfRows += 1; - j += 1; - } - break; + if (colDataIsNull_var(pDst, j)) { + colDataSetNull_var(pDst, numOfRows); + } else { + char* p1 = colDataGetVarData(pDst, j); + colDataAppend(pDst, numOfRows, p1, false); } - }; + numOfRows += 1; + j += 1; + } - // todo this value can be assigned directly - if (pBlock->info.rows == totalRows) { - pBlock->info.rows = numOfRows; + if (maxRows < numOfRows) { + maxRows = numOfRows; + } } else { - ASSERT(pBlock->info.rows == numOfRows); + if (pBitmap == NULL) { + pBitmap = taosMemoryCalloc(1, bmLen); + } + + memcpy(pBitmap, pDst->nullbitmap, bmLen); + memset(pDst->nullbitmap, 0, bmLen); + + int32_t j = 0; + + switch (pDst->info.type) { + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } + + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int64_t*)pDst->pData)[numOfRows] = ((int64_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int32_t*)pDst->pData)[numOfRows++] = ((int32_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int16_t*)pDst->pData)[numOfRows++] = ((int16_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + while (j < totalRows) { + if (pIndicator[j] == 0) { + j += 1; + continue; + } + if (colDataIsNull_f(pBitmap, j)) { + colDataSetNull_f(pDst->nullbitmap, numOfRows); + } else { + ((int8_t*)pDst->pData)[numOfRows] = ((int8_t*)pDst->pData)[j]; + } + numOfRows += 1; + j += 1; + } + break; + } + } + + if (maxRows < numOfRows) { + maxRows = numOfRows; } } + pBlock->info.rows = maxRows; if (pBitmap != NULL) { taosMemoryFree(pBitmap); } From 1a97a659bccd787eb7513b46685ebc5a732efa7d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Jan 2023 00:19:04 +0800 Subject: [PATCH 5/7] fix(query): fix error in filter. --- source/libs/executor/src/executorimpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a8c02b3700..3803bd8b01 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -930,7 +930,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD if (colDataIsNull_f(pBitmap, j)) { colDataSetNull_f(pDst->nullbitmap, numOfRows); } else { - ((int32_t*)pDst->pData)[numOfRows++] = ((int32_t*)pDst->pData)[j]; + ((int32_t*)pDst->pData)[numOfRows] = ((int32_t*)pDst->pData)[j]; } numOfRows += 1; j += 1; @@ -946,7 +946,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD if (colDataIsNull_f(pBitmap, j)) { colDataSetNull_f(pDst->nullbitmap, numOfRows); } else { - ((int16_t*)pDst->pData)[numOfRows++] = ((int16_t*)pDst->pData)[j]; + ((int16_t*)pDst->pData)[numOfRows] = ((int16_t*)pDst->pData)[j]; } numOfRows += 1; j += 1; From e07c30be1cd855d65fd2fa0a69b9b19809c2bf2d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Jan 2023 00:49:38 +0800 Subject: [PATCH 6/7] fix(query): clear the length before append var data. --- source/libs/executor/src/executorimpl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3803bd8b01..2e8611c7dd 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -871,6 +871,8 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD if (IS_VAR_DATA_TYPE(pDst->info.type)) { int32_t j = 0; + pDst->varmeta.length = 0; + while(j < totalRows) { if (pIndicator[j] == 0) { continue; From 5a343bd01c0565a651f7345aaf69da35de571e03 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Jan 2023 11:48:54 +0800 Subject: [PATCH 7/7] fix(query): increase the index when handling the var data type. --- source/libs/executor/src/executorimpl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 2e8611c7dd..ecd6f382f4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -875,6 +875,7 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD while(j < totalRows) { if (pIndicator[j] == 0) { + j += 1; continue; }