From 9a06b6eb58944da6c35c7a9d26e1065fc9a11140 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Fri, 6 Sep 2024 18:11:32 +0800 Subject: [PATCH 01/11] docs: add browser version requirements --- docs/en/14-reference/01-components/07-explorer.md | 2 +- docs/zh/14-reference/01-components/07-explorer.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/01-components/07-explorer.md b/docs/en/14-reference/01-components/07-explorer.md index 2a824c6d8f..c270dadb3b 100644 --- a/docs/en/14-reference/01-components/07-explorer.md +++ b/docs/en/14-reference/01-components/07-explorer.md @@ -4,7 +4,7 @@ sidebar_label: Taos-Explorer description: User guide about taosExplorer --- -taos-explorer is a web service which provides GUI based interactive database management tool. +taos-explorer is a web service which provides GUI based interactive database management tool. To ensure the best experience when accessing taosExplorer, please use Chrome version 79 or higher, Edge version 79 or higher. ## Install diff --git a/docs/zh/14-reference/01-components/07-explorer.md b/docs/zh/14-reference/01-components/07-explorer.md index 5a84490b81..6a8972deea 100644 --- a/docs/zh/14-reference/01-components/07-explorer.md +++ b/docs/zh/14-reference/01-components/07-explorer.md @@ -4,7 +4,7 @@ sidebar_label: taosExplorer toc_max_heading_level: 4 --- -taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工具的 web 服务。本节主要讲述其安装和部署。它的各项功能都是基于简单易上手的图形界面，可以直接尝试，如果有需要也可以考高级功能和运维指南中的相关内容。 +taosExplorer 是一个为用户提供 TDengine 实例的可视化管理交互工具的 web 服务。本节主要讲述其安装和部署。它的各项功能都是基于简单易上手的图形界面，可以直接尝试，如果有需要也可以考高级功能和运维指南中的相关内容。为了确保访问 taosExplorer 的最佳体验，请使用 Chrome 79 及以上版本，或 Edge 79 及以上版本。 ## 安装 From b57cd278278b11bf46cbe485844c8dce9356fab1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 9 Sep 2024 18:20:00 +0800 Subject: [PATCH 02/11] enh: add block check --- include/common/tdatablock.h | 1 + source/common/src/tdatablock.c | 70 +++++++++++++++++++++++++++++ source/libs/executor/src/operator.c | 2 + 3 files changed, 73 insertions(+) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 22a43012c5..1d280b5d6b 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -233,6 +233,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); * @brief find how many rows already in order start from first row */ int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo); +void blockDataCheck(const SSDataBlock* pDataBlock); int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index ea3e88919b..22bca726d7 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2936,6 +2936,8 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha // return length of encoded data, return -1 if failed int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { + blockDataCheck(pBlock); + int32_t dataLen = 0; // todo extract method @@ -3177,6 +3179,9 @@ int32_t blockDecode(SSDataBlock* pBlock, const char* pData, const char** pEndPos } *pEndPos = pStart; + + blockDataCheck(pBlock); + return code; } @@ -3386,3 +3391,68 @@ int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo) { return nextRowIdx; } + +void blockDataCheck(const SSDataBlock* pDataBlock) { + if (NULL == pDataBlock || pDataBlock->info.rows == 0) { + return; + } + + ASSERT(pDataBlock->info.rows > 0); + + if (!pDataBlock->info.dataLoad) { + return; + } + + bool isVarType = false; + int32_t colLen = 0; + int32_t nextPos = 0; + int64_t checkRows = 0; + int64_t typeValue = 0; + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < colNum; ++i) { + SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pDataBlock->pDataBlock, i); + isVarType = IS_VAR_DATA_TYPE(pCol->info.type); + checkRows = pDataBlock->info.rows; + + if (isVarType) { + ASSERT(pCol->varmeta.length); + } else { + ASSERT(pCol->nullbitmap); + } + + for (int64_t r = 0; r < checkRows; ++r) { + if (!colDataIsNull_s(pCol, r)) { + ASSERT(pCol->pData); + ASSERT(pCol->varmeta.length <= pCol->varmeta.allocLen); + + if (isVarType) { + ASSERT(pCol->varmeta.allocLen > 0); + ASSERT(pCol->varmeta.offset[r] < pCol->varmeta.length); + if (pCol->reassigned) { + ASSERT(pCol->varmeta.offset[r] >= 0); + } else { + ASSERT(pCol->varmeta.offset[r] == nextPos); + } + + colLen = varDataTLen(pCol->pData + pCol->varmeta.offset[r]); + ASSERT(colLen >= VARSTR_HEADER_SIZE); + ASSERT(colLen <= pCol->info.bytes); + + if (pCol->reassigned) { + ASSERT((pCol->varmeta.offset[r] + colLen) <= pCol->varmeta.length); + } else { + nextPos += colLen; + ASSERT(nextPos <= pCol->varmeta.length); + } + + typeValue = *(char*)(pCol->pData + pCol->varmeta.offset[r] + colLen - 1); + } else { + GET_TYPED_DATA(typeValue, int64_t, pCol->info.type, colDataGetNumData(pCol, r)); + } + } + } + } + + return; +} + diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 6fe270161f..de478a146f 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -868,12 +868,14 @@ int32_t setOperatorParams(struct SOperatorInfo* pOperator, SOperatorParam* pInpu SSDataBlock* getNextBlockFromDownstream(struct SOperatorInfo* pOperator, int32_t idx) { SSDataBlock* p = NULL; int32_t code = getNextBlockFromDownstreamImpl(pOperator, idx, true, &p); + blockDataCheck(p); return (code == 0)? p:NULL; } SSDataBlock* getNextBlockFromDownstreamRemain(struct SOperatorInfo* pOperator, int32_t idx) { SSDataBlock* p = NULL; int32_t code = getNextBlockFromDownstreamImpl(pOperator, idx, false, &p); + blockDataCheck(p); return (code == 0)? p:NULL; } From d199f852f5b383114587a98567b2822acd9484a3 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 9 Sep 2024 19:28:08 +0800 Subject: [PATCH 03/11] fix: check issue --- source/common/src/tdatablock.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 22bca726d7..840434a351 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3415,11 +3415,12 @@ void blockDataCheck(const SSDataBlock* pDataBlock) { checkRows = pDataBlock->info.rows; if (isVarType) { - ASSERT(pCol->varmeta.length); + ASSERT(pCol->varmeta.offset); } else { ASSERT(pCol->nullbitmap); } - + + nextPos = 0; for (int64_t r = 0; r < checkRows; ++r) { if (!colDataIsNull_s(pCol, r)) { ASSERT(pCol->pData); From 4bc8db22fa747ca2f3376d3828e9d054c957c6a7 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 9 Sep 2024 19:54:56 +0800 Subject: [PATCH 04/11] enh: grant column length adaption --- include/common/tgrant.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index f6b98cf844..1660cc6a1b 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -74,26 +74,26 @@ int32_t tGetMachineId(char **result); #ifdef TD_ENTERPRISE #define GRANTS_SCHEMA \ static const SSysDbTableSchema grantsSchema[] = { \ - {.name = "version", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "version", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ - {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ - {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ - {.name = "cpu_cores", .bytes = 13 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "timeseries", .bytes = 43 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "dnodes", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "cpu_cores", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ } #else #define GRANTS_SCHEMA \ static const SSysDbTableSchema grantsSchema[] = { \ - {.name = "version", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "version", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ - {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ - {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ - {.name = "cpu_cores", .bytes = 13 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "timeseries", .bytes = 43 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "dnodes", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "cpu_cores", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ } #endif // #define GRANT_CFG_ADD From 529af8d20b4a738bc70189eac52fcfcf5bb6774c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 10 Sep 2024 16:48:16 +0800 Subject: [PATCH 05/11] fix(tsdb/cache): comment mem's early ts block --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 2b76e8841d..d19a3b034d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -2691,12 +2691,14 @@ static int32_t getNextRowFromMem(void *iter, TSDBROW **ppRow, bool *pIgnoreEarli switch (state->state) { case SMEMNEXTROW_ENTER: { if (state->pMem != NULL) { + /* if (state->pMem->maxKey <= state->lastTs) { *ppRow = NULL; *pIgnoreEarlierTs = true; TAOS_RETURN(code); } + */ tsdbTbDataIterOpen(state->pMem, NULL, 1, &state->iter); TSDBROW *pMemRow = tsdbTbDataIterGet(&state->iter); From dc983b9a39a3b317d40caf12870b37fa59b6020c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 10 Sep 2024 16:56:36 +0800 Subject: [PATCH 06/11] fix: add block data check --- include/common/tdatablock.h | 2 +- source/common/src/tdatablock.c | 13 +++++++++---- source/libs/executor/src/dynqueryctrloperator.c | 1 + source/libs/executor/src/executor.c | 5 +++++ source/libs/executor/src/executorInt.c | 2 ++ source/libs/executor/src/groupcacheoperator.c | 2 ++ source/libs/executor/src/mergejoinoperator.c | 8 ++++++-- source/libs/executor/src/mergeoperator.c | 2 ++ source/libs/executor/src/operator.c | 4 ++-- source/libs/executor/src/sortoperator.c | 5 ++++- 10 files changed, 34 insertions(+), 10 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 1d280b5d6b..cbfd1efb84 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -233,7 +233,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); * @brief find how many rows already in order start from first row */ int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo); -void blockDataCheck(const SSDataBlock* pDataBlock); +void blockDataCheck(const SSDataBlock* pDataBlock, bool forceChk); int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 840434a351..6e4162268d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -731,6 +731,10 @@ int32_t blockDataMergeNRows(SSDataBlock* pDest, const SSDataBlock* pSrc, int32_t } void blockDataShrinkNRows(SSDataBlock* pBlock, int32_t numOfRows) { + if (numOfRows == 0) { + return; + } + if (numOfRows >= pBlock->info.rows) { blockDataCleanup(pBlock); return; @@ -2936,7 +2940,7 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha // return length of encoded data, return -1 if failed int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { - blockDataCheck(pBlock); + blockDataCheck(pBlock, false); int32_t dataLen = 0; @@ -3180,7 +3184,7 @@ int32_t blockDecode(SSDataBlock* pBlock, const char* pData, const char** pEndPos *pEndPos = pStart; - blockDataCheck(pBlock); + blockDataCheck(pBlock, false); return code; } @@ -3392,14 +3396,14 @@ int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo) { return nextRowIdx; } -void blockDataCheck(const SSDataBlock* pDataBlock) { +void blockDataCheck(const SSDataBlock* pDataBlock, bool forceChk) { if (NULL == pDataBlock || pDataBlock->info.rows == 0) { return; } ASSERT(pDataBlock->info.rows > 0); - if (!pDataBlock->info.dataLoad) { + if (!pDataBlock->info.dataLoad && !forceChk) { return; } @@ -3457,3 +3461,4 @@ void blockDataCheck(const SSDataBlock* pDataBlock) { return; } + diff --git a/source/libs/executor/src/dynqueryctrloperator.c b/source/libs/executor/src/dynqueryctrloperator.c index 44e8a3cb8a..aa0cc215df 100644 --- a/source/libs/executor/src/dynqueryctrloperator.c +++ b/source/libs/executor/src/dynqueryctrloperator.c @@ -528,6 +528,7 @@ static void seqJoinLaunchNewRetrieveImpl(SOperatorInfo* pOperator, SSDataBlock** qDebug("%s dynamic post task begin", GET_TASKID(pOperator->pTaskInfo)); code = pOperator->pDownstream[1]->fpSet.getNextExtFn(pOperator->pDownstream[1], pParam, ppRes); if (*ppRes && (code == 0)) { + blockDataCheck(*ppRes, false); pPost->isStarted = true; pStbJoin->execInfo.postBlkNum++; pStbJoin->execInfo.postBlkRows += (*ppRes)->info.rows; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 9e33a3d890..cca1895a85 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -692,8 +692,10 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo if (pTaskInfo->pOpParam && !pTaskInfo->paramSet) { pTaskInfo->paramSet = true; code = pTaskInfo->pRoot->fpSet.getNextExtFn(pTaskInfo->pRoot, pTaskInfo->pOpParam, &pRes); + blockDataCheck(pRes, false); } else { code = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot, &pRes); + blockDataCheck(pRes, false); } QUERY_CHECK_CODE(code, lino, _end); @@ -740,6 +742,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo } code = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot, &pRes); + blockDataCheck(pRes, false); QUERY_CHECK_CODE(code, lino, _end); } @@ -839,6 +842,8 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { qError("%s failed at line %d, code:%s %s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); } + blockDataCheck(*pRes, false); + uint64_t el = (taosGetTimestampUs() - st); pTaskInfo->cost.elapsedTime += el; diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 00138d6871..c8ec1e8fcc 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -617,6 +617,8 @@ int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* p code = TSDB_CODE_SUCCESS; _err: + blockDataCheck(pBlock, true); + colDataDestroy(p); taosMemoryFree(p); return code; diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index 1796aa7b64..990c9322b7 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -733,6 +733,8 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p } } + blockDataCheck(pBlock, false); + *ppRes = pBlock; return code; } diff --git a/source/libs/executor/src/mergejoinoperator.c b/source/libs/executor/src/mergejoinoperator.c index 90c2248c12..c29c49e280 100644 --- a/source/libs/executor/src/mergejoinoperator.c +++ b/source/libs/executor/src/mergejoinoperator.c @@ -474,8 +474,11 @@ int32_t mJoinCopyMergeMidBlk(SMJoinMergeCtx* pCtx, SSDataBlock** ppMid, SSDataBl pCtx->midRemains = false; } else { int32_t copyRows = pMore->info.capacity - pMore->info.rows; - MJ_ERR_RET(blockDataMergeNRows(pMore, pLess, pLess->info.rows - copyRows, copyRows)); - blockDataShrinkNRows(pLess, copyRows); + if (copyRows > 0) { + MJ_ERR_RET(blockDataMergeNRows(pMore, pLess, pLess->info.rows - copyRows, copyRows)); + blockDataShrinkNRows(pLess, copyRows); + } + pCtx->midRemains = true; } @@ -1742,6 +1745,7 @@ int32_t mJoinMainProcess(struct SOperatorInfo* pOperator, SSDataBlock** pResBloc if (pBlock && pBlock->info.rows > 0) { *pResBlock = pBlock; } + return code; } diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index 7357329ca6..c94a330dbc 100644 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -66,6 +66,7 @@ static int32_t sortMergeloadNextDataBlock(void* param, SSDataBlock** ppBlock); int32_t sortMergeloadNextDataBlock(void* param, SSDataBlock** ppBlock) { SOperatorInfo* pOperator = (SOperatorInfo*)param; int32_t code = pOperator->fpSet.getNextFn(pOperator, ppBlock); + blockDataCheck(*ppBlock, false); return code; } @@ -524,6 +525,7 @@ int32_t doMultiwayMerge(SOperatorInfo* pOperator, SSDataBlock** pResBlock) { if ((*pResBlock) != NULL) { pOperator->resultInfo.totalRows += (*pResBlock)->info.rows; + blockDataCheck(*pResBlock, false); } else { setOperatorCompleted(pOperator); } diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index de478a146f..cb1149d8a3 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -868,14 +868,14 @@ int32_t setOperatorParams(struct SOperatorInfo* pOperator, SOperatorParam* pInpu SSDataBlock* getNextBlockFromDownstream(struct SOperatorInfo* pOperator, int32_t idx) { SSDataBlock* p = NULL; int32_t code = getNextBlockFromDownstreamImpl(pOperator, idx, true, &p); - blockDataCheck(p); + blockDataCheck(p, false); return (code == 0)? p:NULL; } SSDataBlock* getNextBlockFromDownstreamRemain(struct SOperatorInfo* pOperator, int32_t idx) { SSDataBlock* p = NULL; int32_t code = getNextBlockFromDownstreamImpl(pOperator, idx, false, &p); - blockDataCheck(p); + blockDataCheck(p, false); return (code == 0)? p:NULL; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 9b659ac761..27ae5e7281 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -328,7 +328,9 @@ static int32_t getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t loadNextDataBlock(void* param, SSDataBlock** ppBlock) { SOperatorInfo* pOperator = (SOperatorInfo*)param; - return pOperator->fpSet.getNextFn(pOperator, ppBlock); + int32_t code = pOperator->fpSet.getNextFn(pOperator, ppBlock); + blockDataCheck(*ppBlock, false); + return code; } // todo refactor: merged with fetch fp @@ -611,6 +613,7 @@ int32_t fetchNextGroupSortDataBlock(void* param, SSDataBlock** ppBlock) { QUERY_CHECK_CODE(code, lino, _end); if (block != NULL) { + blockDataCheck(block, false); if (block->info.id.groupId == grpSortOpInfo->currGroupId) { grpSortOpInfo->childOpStatus = CHILD_OP_SAME_GROUP; *ppBlock = block; From d403a6129adbd22a4f58fd6352cfcda2d61d58d7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 10 Sep 2024 19:14:27 +0800 Subject: [PATCH 07/11] fix mem leak --- source/libs/transport/src/trans.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 1c783f44fe..fa0be7bcd8 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -103,7 +103,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->timeToGetConn = 10 * 1000; } pRpc->notWaitAvaliableConn = pInit->notWaitAvaliableConn; - + pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); @@ -155,10 +155,13 @@ void* rpcReallocCont(void* ptr, int64_t contLen) { char* st = (char*)ptr - TRANS_MSG_OVERHEAD; int64_t sz = contLen + TRANS_MSG_OVERHEAD; - st = taosMemoryRealloc(st, sz); - if (st == NULL) { + char* nst = taosMemoryRealloc(st, sz); + if (nst == NULL) { + taosMemoryFree(st); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; + } else { + st = nst; } return st + TRANS_MSG_OVERHEAD; @@ -168,7 +171,7 @@ int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64 return transSendRequest(shandle, pEpSet, pMsg, NULL); } int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { - if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0|| pRid == NULL) { + if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { return transSendRequest(shandle, pEpSet, pMsg, pCtx); } else { return transSendRequestWithId(shandle, pEpSet, pMsg, pRid); From 1519be0282078696ec10fc3a2f55570d9950e7ac Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 10 Sep 2024 19:19:57 +0800 Subject: [PATCH 08/11] fix mem leak --- source/libs/transport/src/trans.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index fa0be7bcd8..0881f91d69 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -140,6 +140,7 @@ void* rpcMallocCont(int64_t contLen) { char* start = taosMemoryCalloc(1, size); if (start == NULL) { tError("failed to malloc msg, size:%" PRId64, size); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } else { tTrace("malloc mem:%p size:%" PRId64, start, size); From 7d5ef92bb22ac2061b53fb8638f8982fd9a3c7e6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 11 Sep 2024 09:42:59 +0800 Subject: [PATCH 09/11] fix: block data shrink issue --- source/common/src/tdatablock.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 6e4162268d..dee9ec2c0c 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3397,11 +3397,15 @@ int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo) { } void blockDataCheck(const SSDataBlock* pDataBlock, bool forceChk) { + return; + if (NULL == pDataBlock || pDataBlock->info.rows == 0) { return; } - ASSERT(pDataBlock->info.rows > 0); +#define BLOCK_DATA_CHECK_TRESSA(o) ; + + BLOCK_DATA_CHECK_TRESSA(pDataBlock->info.rows > 0); if (!pDataBlock->info.dataLoad && !forceChk) { return; @@ -3419,35 +3423,35 @@ void blockDataCheck(const SSDataBlock* pDataBlock, bool forceChk) { checkRows = pDataBlock->info.rows; if (isVarType) { - ASSERT(pCol->varmeta.offset); + BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset); } else { - ASSERT(pCol->nullbitmap); + BLOCK_DATA_CHECK_TRESSA(pCol->nullbitmap); } nextPos = 0; for (int64_t r = 0; r < checkRows; ++r) { if (!colDataIsNull_s(pCol, r)) { - ASSERT(pCol->pData); - ASSERT(pCol->varmeta.length <= pCol->varmeta.allocLen); + BLOCK_DATA_CHECK_TRESSA(pCol->pData); + BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.length <= pCol->varmeta.allocLen); if (isVarType) { - ASSERT(pCol->varmeta.allocLen > 0); - ASSERT(pCol->varmeta.offset[r] < pCol->varmeta.length); + BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.allocLen > 0); + BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] < pCol->varmeta.length); if (pCol->reassigned) { - ASSERT(pCol->varmeta.offset[r] >= 0); + BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] >= 0); } else { - ASSERT(pCol->varmeta.offset[r] == nextPos); + BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] == nextPos); } colLen = varDataTLen(pCol->pData + pCol->varmeta.offset[r]); - ASSERT(colLen >= VARSTR_HEADER_SIZE); - ASSERT(colLen <= pCol->info.bytes); + BLOCK_DATA_CHECK_TRESSA(colLen >= VARSTR_HEADER_SIZE); + BLOCK_DATA_CHECK_TRESSA(colLen <= pCol->info.bytes); if (pCol->reassigned) { - ASSERT((pCol->varmeta.offset[r] + colLen) <= pCol->varmeta.length); + BLOCK_DATA_CHECK_TRESSA((pCol->varmeta.offset[r] + colLen) <= pCol->varmeta.length); } else { nextPos += colLen; - ASSERT(nextPos <= pCol->varmeta.length); + BLOCK_DATA_CHECK_TRESSA(nextPos <= pCol->varmeta.length); } typeValue = *(char*)(pCol->pData + pCol->varmeta.offset[r] + colLen - 1); From 82adb5347878ec8a5d38727634116a5fbb27bbc9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 11 Sep 2024 13:31:28 +0800 Subject: [PATCH 10/11] fix: correct block check --- source/common/src/tdatablock.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index dee9ec2c0c..bd169914df 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3397,13 +3397,14 @@ int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo) { } void blockDataCheck(const SSDataBlock* pDataBlock, bool forceChk) { - return; + //return; if (NULL == pDataBlock || pDataBlock->info.rows == 0) { return; } -#define BLOCK_DATA_CHECK_TRESSA(o) ; +//#define BLOCK_DATA_CHECK_TRESSA(o) ; +#define BLOCK_DATA_CHECK_TRESSA(o) ASSERT(o) BLOCK_DATA_CHECK_TRESSA(pDataBlock->info.rows > 0); @@ -3439,6 +3440,8 @@ void blockDataCheck(const SSDataBlock* pDataBlock, bool forceChk) { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] < pCol->varmeta.length); if (pCol->reassigned) { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] >= 0); + } else if (0 == r) { + nextPos = pCol->varmeta.offset[r]; } else { BLOCK_DATA_CHECK_TRESSA(pCol->varmeta.offset[r] == nextPos); } From aecdabe9a4ad0cb9938292b27598f727543fdaeb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 11 Sep 2024 13:38:24 +0800 Subject: [PATCH 11/11] fix: remove block check --- source/common/src/tdatablock.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index bd169914df..08f6842c4b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -3397,14 +3397,14 @@ int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo) { } void blockDataCheck(const SSDataBlock* pDataBlock, bool forceChk) { - //return; + return; if (NULL == pDataBlock || pDataBlock->info.rows == 0) { return; } -//#define BLOCK_DATA_CHECK_TRESSA(o) ; -#define BLOCK_DATA_CHECK_TRESSA(o) ASSERT(o) +#define BLOCK_DATA_CHECK_TRESSA(o) ; +//#define BLOCK_DATA_CHECK_TRESSA(o) A S S E R T(o) BLOCK_DATA_CHECK_TRESSA(pDataBlock->info.rows > 0);