From 3341d02060e221f7b1537a7527daf4f09ac98cee Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 22 Feb 2024 17:03:17 +0800 Subject: [PATCH 1/7] fix: don't copy all data when use limit --- source/common/src/tdatablock.c | 1 + source/libs/executor/src/executor.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 9439c172c4..690eda17e1 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1688,6 +1688,7 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + pColInfoData->varmeta.length = pColInfoData->varmeta.offset[n]; // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 2534c5e9f0..cbf0d38bd9 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -648,7 +648,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo ASSERT(p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT); taosArrayPush(pResList, &p); - if (current >= rowsThreshold) { + if (current >= rowsThreshold || current >= pRes->info.rows) { break; } From 5647d990b40caf1c5d70d3f3154b1bf89626c18c Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Sun, 25 Feb 2024 21:37:33 +0800 Subject: [PATCH 2/7] fix: colDataAssign --- source/common/src/tdatablock.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 690eda17e1..1e566ccfbe 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -452,20 +452,31 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + int32_t newLen = pSource->varmeta.offset[numOfRows - 1]; + if (newLen != -1) { + if (pSource->info.type == TSDB_DATA_TYPE_JSON) { + newLen += getJsonValueLen(pSource->pData + newLen); + } else { + newLen += varDataTLen(pSource->pData + newLen); + } + } else { + newLen = pSource->varmeta.length; + } + memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); - if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { - char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); + if (pColumnInfoData->varmeta.allocLen < newLen) { + char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newLen); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } pColumnInfoData->pData = tmp; - pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; + pColumnInfoData->varmeta.allocLen = newLen; } - pColumnInfoData->varmeta.length = pSource->varmeta.length; + pColumnInfoData->varmeta.length = newLen; if (pColumnInfoData->pData != NULL && pSource->pData != NULL) { - memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); + memcpy(pColumnInfoData->pData, pSource->pData, newLen); } } else { memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); @@ -1688,7 +1699,7 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - pColInfoData->varmeta.length = pColInfoData->varmeta.offset[n]; + // pColInfoData->varmeta.length = pColInfoData->varmeta.offset[n]; // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); } From 8230b238fb8dbb5f375bd6855d2d58cba235afb4 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Sun, 25 Feb 2024 22:56:32 +0800 Subject: [PATCH 3/7] fix: keepFirstNRows --- source/common/src/tdatablock.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 1e566ccfbe..6b6c3642de 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -452,17 +452,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - int32_t newLen = pSource->varmeta.offset[numOfRows - 1]; - if (newLen != -1) { - if (pSource->info.type == TSDB_DATA_TYPE_JSON) { - newLen += getJsonValueLen(pSource->pData + newLen); - } else { - newLen += varDataTLen(pSource->pData + newLen); - } - } else { - newLen = pSource->varmeta.length; - } - + int32_t newLen = pSource->varmeta.length; memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); if (pColumnInfoData->varmeta.allocLen < newLen) { char* tmp = taosMemoryRealloc(pColumnInfoData->pData, newLen); @@ -1699,7 +1689,17 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - // pColInfoData->varmeta.length = pColInfoData->varmeta.offset[n]; + int32_t newLen = pColInfoData->varmeta.offset[n - 1]; + if (newLen != -1) { + if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { + newLen += getJsonValueLen(pColInfoData->pData + newLen); + } else { + newLen += varDataTLen(pColInfoData->pData + newLen); + } + } else { + newLen = pColInfoData->varmeta.length; + } + pColInfoData->varmeta.length = newLen; // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); } From fc4caa23b5fb1221a83c63e7d7ebe07b911e8af5 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 26 Feb 2024 13:52:10 +0800 Subject: [PATCH 4/7] fix: cache --- source/libs/executor/src/executor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index cbf0d38bd9..2534c5e9f0 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -648,7 +648,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo ASSERT(p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT); taosArrayPush(pResList, &p); - if (current >= rowsThreshold || current >= pRes->info.rows) { + if (current >= rowsThreshold) { break; } From 551a65e16108deef13dd5149459bfbb080f0f5d9 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 29 Feb 2024 10:44:25 +0800 Subject: [PATCH 5/7] fix: varmeta length reset --- source/common/src/tdatablock.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 6b6c3642de..a20ecd344d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1688,17 +1688,19 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { } static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { + if (n >= total || n == 0) return; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - int32_t newLen = pColInfoData->varmeta.offset[n - 1]; - if (newLen != -1) { - if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { - newLen += getJsonValueLen(pColInfoData->pData + newLen); - } else { - newLen += varDataTLen(pColInfoData->pData + newLen); + int32_t newLen = -1; + for (int i = n; i >= 0; --i) { + if (pColInfoData->varmeta.offset[i] != -1) { + newLen = pColInfoData->varmeta.offset[i]; + break; } - } else { - newLen = pColInfoData->varmeta.length; } + if(-1 == newLen) { + return; + } + pColInfoData->varmeta.length = newLen; // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); From ae660f95ed01007a4c1614a393493038f0c4afb1 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 29 Feb 2024 15:02:55 +0800 Subject: [PATCH 6/7] fix: get firstNRows Length --- source/common/src/tdatablock.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a20ecd344d..f2cd9313e1 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1690,17 +1690,20 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (n >= total || n == 0) return; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - int32_t newLen = -1; - for (int i = n; i >= 0; --i) { - if (pColInfoData->varmeta.offset[i] != -1) { + int32_t newLen = pColInfoData->varmeta.offset[n]; + if (-1 == newLen) { + for (int i = n - 1; i >= 0; --i) { newLen = pColInfoData->varmeta.offset[i]; - break; + if (newLen != -1) { + if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { + newLen += getJsonValueLen(pColInfoData->pData + newLen); + } else { + newLen += varDataTLen(pColInfoData->pData + newLen); + } + break; + } } } - if(-1 == newLen) { - return; - } - pColInfoData->varmeta.length = newLen; // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); From 1fe35031e089dac1ac1a500029b21b8d6bc2794a Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 29 Feb 2024 16:32:58 +0800 Subject: [PATCH 7/7] fix: get firstNRows Length --- source/common/src/tdatablock.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f2cd9313e1..c996bc2e4a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1690,21 +1690,27 @@ int32_t blockDataTrimFirstRows(SSDataBlock* pBlock, size_t n) { static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (n >= total || n == 0) return; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - int32_t newLen = pColInfoData->varmeta.offset[n]; - if (-1 == newLen) { - for (int i = n - 1; i >= 0; --i) { - newLen = pColInfoData->varmeta.offset[i]; - if (newLen != -1) { - if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { - newLen += getJsonValueLen(pColInfoData->pData + newLen); - } else { - newLen += varDataTLen(pColInfoData->pData + newLen); + if (pColInfoData->varmeta.length != 0) { + int32_t newLen = pColInfoData->varmeta.offset[n]; + if (-1 == newLen) { + for (int i = n - 1; i >= 0; --i) { + newLen = pColInfoData->varmeta.offset[i]; + if (newLen != -1) { + if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { + newLen += getJsonValueLen(pColInfoData->pData + newLen); + } else { + newLen += varDataTLen(pColInfoData->pData + newLen); + } + break; } - break; } } + if (newLen <= -1) { + uFatal("colDataKeepFirstNRows: newLen:%d old:%d", newLen, pColInfoData->varmeta.length); + } else { + pColInfoData->varmeta.length = newLen; + } } - pColInfoData->varmeta.length = newLen; // pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); }