From bff8226c0f6ef56860ce10c305ad5b5b2c69efa9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 11 Apr 2024 08:27:17 +0800 Subject: [PATCH 1/5] enh: none column for primary key --- include/util/taoserror.h | 1 + source/common/src/tdataformat.c | 2 +- source/util/src/terror.c | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 2389079fd2..a3cae6a7db 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -767,6 +767,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_COL_PK_TYPE TAOS_DEF_ERROR_CODE(0, 0x2673) #define TSDB_CODE_PAR_INVALID_PK_OP TAOS_DEF_ERROR_CODE(0, 0x2674) #define TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x2675) +#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE TAOS_DEF_ERROR_CODE(0, 0x2676) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) //planner diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f8d2da0bd5..9686059052 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -102,7 +102,7 @@ typedef struct { } SRowBuildScanInfo; static FORCE_INLINE int32_t tRowBuildScanAddNone(SRowBuildScanInfo *sinfo, const STColumn *pTColumn) { - if ((pTColumn->flags & COL_IS_KEY)) return TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL; + if ((pTColumn->flags & COL_IS_KEY)) return TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE; sinfo->numOfNone++; return 0; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 0812181c5c..8d80e3883d 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -629,6 +629,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SECOND_COL_PK, "primary key must be TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COL_PK_TYPE, "primary key column must be of type int, uint, bigint, ubigint, and varchar") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PK_OP, "primary key column can not be added, modified, and dropped") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL, "Primary key column should not be null") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE, "Primary key column should not be none") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error") //planner From 79be7eea8c1e8b3c02f1a51b71f734b94d94e674 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Apr 2024 10:06:07 +0800 Subject: [PATCH 2/5] fix(tsdb): fix invalid read, and do some internal refactor. --- source/common/src/tdatablock.c | 5 ++ source/dnode/vnode/src/tsdb/tsdbRead2.c | 85 +++++++++------------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 67 +++++++++++------ source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 7 +- 4 files changed, 90 insertions(+), 74 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 69b2a2e6a3..8d9ef6831d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1331,6 +1331,11 @@ void* blockDataDestroy(SSDataBlock* pBlock) { return NULL; } + if (IS_VAR_DATA_TYPE(pBlock->info.pks[0].type)) { + taosMemoryFreeClear(pBlock->info.pks[0].pData); + taosMemoryFreeClear(pBlock->info.pks[1].pData); + } + blockDataFreeRes(pBlock); taosMemoryFreeClear(pBlock); return NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index c08face243..be01afb960 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -67,8 +67,7 @@ static int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRo STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SRowKey* pKey, STsdbReader* pReader); -static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pBlockScanInfo, - STsdbReader* pReader); +static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost); static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr, @@ -392,7 +391,7 @@ _err: return code; } -static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order) { +void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order) { pIter->order = order; pIter->index = -1; pIter->numOfBlocks = 0; @@ -403,8 +402,6 @@ static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order) { } } -static void cleanupDataBlockIterator(SDataBlockIter* pIter) { taosArrayDestroy(pIter->blockList); } - static void initReaderStatus(SReaderStatus* pStatus) { pStatus->pTableIter = NULL; pStatus->loadFromFile = true; @@ -657,21 +654,19 @@ _end: return code; } -static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, - SArray* pTableScanInfoList) { - size_t sizeInDisk = 0; - int64_t st = taosGetTimestampUs(); +static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, + SArray* pTableScanInfoList) { + int32_t k = 0; + size_t sizeInDisk = 0; + int64_t st = taosGetTimestampUs(); + bool asc = ASCENDING_TRAVERSE(pReader->info.order); + STimeWindow w = pReader->info.window; + SBrinRecord* pRecord = NULL; + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); + SBrinRecordIter iter = {0}; // clear info for the new file cleanupInfoForNextFileset(pReader->status.pTableMap); - - int32_t k = 0; - int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); - bool asc = ASCENDING_TRAVERSE(pReader->info.order); - STimeWindow w = pReader->info.window; - SBrinRecord* pRecord = NULL; - - SBrinRecordIter iter = {0}; initBrinRecordIter(&iter, pReader->pFileReader, pIndexList); while (((pRecord = getNextBrinRecord(&iter)) != NULL)) { @@ -743,14 +738,27 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN } if (pScanInfo->pBlockList == NULL) { - pScanInfo->pBlockList = taosArrayInit(4, sizeof(SBrinRecord)); + pScanInfo->pBlockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); + if (pScanInfo->pBlockList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - void* p1 = taosArrayPush(pScanInfo->pBlockList, pRecord); + if (pScanInfo->pBlockIdxList == NULL) { + pScanInfo->pBlockIdxList = taosArrayInit(4, sizeof(STableDataBlockIdx)); + if (pScanInfo->pBlockIdxList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + SFileDataBlockInfo blockInfo = {.tbBlockIdx = TARRAY_SIZE(pScanInfo->pBlockList)}; + recordToBlockInfo(&blockInfo, pRecord); + void* p1 = taosArrayPush(pScanInfo->pBlockList, &blockInfo); if (p1 == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + // todo: refactor to record the fileset skey/ekey if (pScanInfo->filesetWindow.skey > pRecord->firstKey.key.ts) { pScanInfo->filesetWindow.skey = pRecord->firstKey.key.ts; } @@ -1323,10 +1331,12 @@ static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* p } static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo, - STableBlockScanInfo* pTableBlockScanInfo, int32_t* nextIndex, int32_t order, + STableBlockScanInfo* pScanInfo, int32_t* nextIndex, int32_t order, SBrinRecord* pRecord) { - bool asc = ASCENDING_TRAVERSE(order); - if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockIdxList) - 1) { + bool asc = ASCENDING_TRAVERSE(order); + int32_t step = asc ? 1 : -1; + + if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pScanInfo->pBlockIdxList) - 1) { return false; } @@ -1334,9 +1344,7 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo return false; } - int32_t step = asc ? 1 : -1; - STableDataBlockIdx* pTableDataBlockIdx = - taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); + STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); blockInfoToRecord(pRecord, p); @@ -1344,22 +1352,6 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo return true; } -static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pFBlockInfo) { - int32_t step = ASCENDING_TRAVERSE(pBlockIter->order) ? 1 : -1; - int32_t index = pBlockIter->index; - - while (index < pBlockIter->numOfBlocks && index >= 0) { - SFileDataBlockInfo* pFBlock = taosArrayGet(pBlockIter->blockList, index); - if (pFBlock->uid == pFBlockInfo->uid && pFBlock->tbBlockIdx == pFBlockInfo->tbBlockIdx) { - return index; - } - - index += step; - } - - return -1; -} - static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t index, int32_t step) { if (index < 0 || index >= pBlockIter->numOfBlocks) { @@ -2706,7 +2698,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr } if (taosArrayGetSize(pIndexList) > 0 || pReader->status.pCurrentFileset->lvlArr->size > 0) { - code = doLoadFileBlock(pReader, pIndexList, pBlockNum, pTableList); + code = loadFileBlockBrinInfo(pReader, pIndexList, pBlockNum, pTableList); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; @@ -3154,23 +3146,14 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; if (pBlockInfo) { - // todo handle -// STableBlockScanInfo* pScanInfo = tSimpleHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); -// if (pScanInfo) { -// tsdbRowKeyAssign(&pDumpInfo->lastKey, &pScanInfo->lastProcKey); -// lastKey = pScanInfo->lastProcKey; -// } - pDumpInfo->totalRows = pBlockInfo->numRow; pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1; } else { pDumpInfo->totalRows = 0; pDumpInfo->rowIndex = 0; -// pDumpInfo->lastKey.key.ts = lastKey; } pDumpInfo->allDumped = false; -// pDumpInfo->lastKey = lastKey; } static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 2a7b0140df..7e81f1df36 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -167,6 +167,13 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in return TSDB_CODE_SUCCESS; } +void clearRowKey(SRowKey* pKey) { + if (pKey == NULL || pKey->numOfPKs == 0 || (!IS_VAR_DATA_TYPE(pKey->pks[0].type))) { + return; + } + taosMemoryFree(pKey->pks[0].pData); +} + static void initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader) { int32_t numOfPks = pReader->suppInfo.numOfPks; bool asc = ASCENDING_TRAVERSE(pReader->info.order); @@ -293,6 +300,11 @@ void clearBlockScanInfo(STableBlockScanInfo* p) { p->pBlockIdxList = taosArrayDestroy(p->pBlockIdxList); p->pMemDelData = taosArrayDestroy(p->pMemDelData); p->pFileDelData = taosArrayDestroy(p->pFileDelData); + + clearRowKey(&p->lastProcKey); + clearRowKey(&p->sttRange.skey); + clearRowKey(&p->sttRange.ekey); + clearRowKey(&p->sttKeyInfo.nextProcKey); } void destroyAllBlockScanInfo(SSHashObj* pTableMap) { @@ -415,7 +427,7 @@ static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, v return pLeftBlock->offset > pRightBlock->offset ? 1 : -1; } -static void recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record, STsdbReader* pReader) { +void recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record) { pBlockInfo->uid = record->uid; pBlockInfo->firstKey = record->firstKey.key.ts; pBlockInfo->lastKey = record->lastKey.key.ts; @@ -449,12 +461,36 @@ static void recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* recor } } +static void freeItem(void* pItem) { + SFileDataBlockInfo* p = pItem; + if (p->firstPKLen > 0) { + taosMemoryFreeClear(p->firstPk.pData); + } + + if (p->lastPKLen > 0) { + taosMemoryFreeClear(p->lastPk.pData); + } +} + +void clearDataBlockIterator(SDataBlockIter* pIter) { + pIter->index = -1; + pIter->numOfBlocks = 0; + taosArrayClearEx(pIter->blockList, freeItem); +} + +void cleanupDataBlockIterator(SDataBlockIter* pIter) { + pIter->index = -1; + pIter->numOfBlocks = 0; + taosArrayDestroyEx(pIter->blockList, freeItem); +} + int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { bool asc = ASCENDING_TRAVERSE(pReader->info.order); SBlockOrderSupporter sup = {0}; + clearDataBlockIterator(pBlockIter); + pBlockIter->numOfBlocks = numOfBlocks; - taosArrayClear(pBlockIter->blockList); // access data blocks according to the offset of each block in asc/desc order. int32_t numOfTables = taosArrayGetSize(pTableList); @@ -482,9 +518,9 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; for (int32_t k = 0; k < num; ++k) { - SBrinRecord* pRecord = taosArrayGet(pTableScanInfo->pBlockList, k); + SFileDataBlockInfo* pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, k); sup.pDataBlockInfo[sup.numOfTables][k] = - (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pRecord->blockOffset, .pInfo = pTableScanInfo}; + (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pBlockInfo->blockOffset, .pInfo = pTableScanInfo}; cnt++; } @@ -499,20 +535,12 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 // since there is only one table qualified, blocks are not sorted if (sup.numOfTables == 1) { STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, 0); - if (pTableScanInfo->pBlockIdxList == NULL) { - pTableScanInfo->pBlockIdxList = taosArrayInit(numOfBlocks, sizeof(STableDataBlockIdx)); - } - for (int32_t i = 0; i < numOfBlocks; ++i) { - SFileDataBlockInfo blockInfo = {.tbBlockIdx = i}; - SBrinRecord* record = (SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i); - recordToBlockInfo(&blockInfo, record, pReader); - - taosArrayPush(pBlockIter->blockList, &blockInfo); STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i}; taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx); } + taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList); pTableScanInfo->pBlockList = taosArrayDestroy(pTableScanInfo->pBlockList); int64_t et = taosGetTimestampUs(); @@ -540,18 +568,13 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3 int32_t pos = tMergeTreeGetChosenIndex(pTree); int32_t index = sup.indexPerTable[pos]++; - SFileDataBlockInfo blockInfo = {.tbBlockIdx = index}; - SBrinRecord* record = (SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); - recordToBlockInfo(&blockInfo, record, pReader); + SFileDataBlockInfo* pBlockInfo = taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index); + taosArrayPush(pBlockIter->blockList, pBlockInfo); - taosArrayPush(pBlockIter->blockList, &blockInfo); STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo; - if (pTableScanInfo->pBlockIdxList == NULL) { - size_t szTableDataBlocks = taosArrayGetSize(pTableScanInfo->pBlockList); - pTableScanInfo->pBlockIdxList = taosArrayInit(szTableDataBlocks, sizeof(STableDataBlockIdx)); - } - STableDataBlockIdx tableDataBlockIdx = {.globalIndex = numOfTotal}; + STableDataBlockIdx tableDataBlockIdx = {.globalIndex = numOfTotal}; taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx); + // set data block index overflow, in order to disable the offset comparator if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) { sup.indexPerTable[pos] = sup.numOfBlocksPerTable[pos] + 1; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index bece22adad..0e7895c272 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -237,7 +237,6 @@ typedef struct SDataBlockIter { typedef struct SFileBlockDumpInfo { int32_t totalRows; int32_t rowIndex; -// int64_t lastKey; // STsdbRowKey lastKey; // this key should be removed bool allDumped; } SFileBlockDumpInfo; @@ -338,6 +337,7 @@ int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileRead int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, TStatisBlkArray* pStatisBlkArray, uint64_t suid, const uint64_t* pUidList, int32_t numOfTables); +void recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record); void destroyLDataIter(SLDataIter* pIter); int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet); @@ -347,6 +347,11 @@ bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STab bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order); int32_t pkCompEx(__compar_fn_t comparFn, SRowKey* p1, SRowKey* p2); int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc); +void clearRowKey(SRowKey* pKey); + +void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order); +void clearDataBlockIterator(SDataBlockIter* pIter); +void cleanupDataBlockIterator(SDataBlockIter* pIter); typedef struct { SArray* pTombData; From 9f95360eccac219a8ad9a1f7a1ed05a470627d9b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Apr 2024 10:14:15 +0800 Subject: [PATCH 3/5] fix(tsdb): fix error in read rowkey with null. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index be01afb960..8f4c719a35 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -173,11 +173,16 @@ static void tRowGetKeyDeepCopy(SRow* pRow, SRowKey* pKey) { for (int32_t i = 0; i < pRow->numOfPKs; i++) { pKey->pks[i].type = indices[i].type; + uint8_t *tdata = data + indices[i].offset; + if (pRow->flag >> 4) { + tdata += tGetI16v(tdata, NULL); + } + if (IS_VAR_DATA_TYPE(indices[i].type)) { tGetU32v(pKey->pks[i].pData, &pKey->pks[i].nData); - pKey->pks[i].pData = memcpy(pKey->pks[i].pData, data + indices[i].offset, pKey->pks[i].nData); + pKey->pks[i].pData = memcpy(pKey->pks[i].pData, tdata, pKey->pks[i].nData); pKey->pks[i].pData += pKey->pks[i].nData; - } else { + } else { // todo pKey->pks[i].val = *(int64_t*) (data + indices[i].offset); } } @@ -4353,9 +4358,11 @@ void tsdbReaderClose2(STsdbReader* pReader) { SReadCostSummary* pCost = &pReader->cost; SFilesetIter* pFilesetIter = &pReader->status.fileIter; if (pFilesetIter->pSttBlockReader != NULL) { - SSttBlockReader* pLReader = pFilesetIter->pSttBlockReader; - tMergeTreeClose(&pLReader->mergeTree); - taosMemoryFree(pLReader); + SSttBlockReader* pSttBlockReader = pFilesetIter->pSttBlockReader; + tMergeTreeClose(&pSttBlockReader->mergeTree); + + clearRowKey(&pSttBlockReader->currentKey); + taosMemoryFree(pSttBlockReader); } destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost); From 1f85a47cabe964d046f401612d80da502ea9ef67 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Apr 2024 10:47:05 +0800 Subject: [PATCH 4/5] fix(tsdb): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 18 ++++++------ source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 34 ++++++++++++---------- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 6 ++-- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 8f4c719a35..3409559867 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -182,8 +182,8 @@ static void tRowGetKeyDeepCopy(SRow* pRow, SRowKey* pKey) { tGetU32v(pKey->pks[i].pData, &pKey->pks[i].nData); pKey->pks[i].pData = memcpy(pKey->pks[i].pData, tdata, pKey->pks[i].nData); pKey->pks[i].pData += pKey->pks[i].nData; - } else { // todo - pKey->pks[i].val = *(int64_t*) (data + indices[i].offset); + } else { + memcpy(&pKey->pks[i].val, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes); } } } @@ -396,14 +396,14 @@ _err: return code; } -void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order) { +void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk) { pIter->order = order; pIter->index = -1; pIter->numOfBlocks = 0; if (pIter->blockList == NULL) { pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo)); } else { - taosArrayClear(pIter->blockList); + clearDataBlockIterator(pIter, hasPk); } } @@ -3183,7 +3183,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks, pTableList); } else { // no block data, only last block exists tBlockDataReset(&pReader->status.fileBlockData); - resetDataBlockIterator(pBlockIter, pReader->info.order); + resetDataBlockIterator(pBlockIter, pReader->info.order, pReader->suppInfo.numOfPks > 0); resetTableListIndex(&pReader->status); } @@ -3293,7 +3293,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } tBlockDataReset(pBlockData); - resetDataBlockIterator(pBlockIter, pReader->info.order); + resetDataBlockIterator(pBlockIter, pReader->info.order, pReader->suppInfo.numOfPks > 0); resetTableListIndex(&pReader->status); ERetrieveType type = doReadDataFromSttFiles(pReader); @@ -4138,7 +4138,7 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { } initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); - resetDataBlockIterator(&pStatus->blockIter, pReader->info.order); + resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, pReader->suppInfo.numOfPks > 0); int32_t code = TSDB_CODE_SUCCESS; if (pStatus->fileIter.numOfFiles == 0) { @@ -4342,7 +4342,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { taosMemoryFree(pSupInfo->colId); tBlockDataDestroy(&pReader->status.fileBlockData); - cleanupDataBlockIterator(&pReader->status.blockIter); + cleanupDataBlockIterator(&pReader->status.blockIter, pReader->suppInfo.numOfPks > 0); size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (pReader->status.pTableMap != NULL) { @@ -5018,7 +5018,7 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) { int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader); - resetDataBlockIterator(pBlockIter, pReader->info.order); + resetDataBlockIterator(pBlockIter, pReader->info.order, pReader->suppInfo.numOfPks > 0); resetTableListIndex(&pReader->status); bool asc = ASCENDING_TRAVERSE(pReader->info.order); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 7e81f1df36..d93c8c8f79 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -461,34 +461,38 @@ void recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record) { } } -static void freeItem(void* pItem) { +static void freePkItem(void* pItem) { SFileDataBlockInfo* p = pItem; - if (p->firstPKLen > 0) { - taosMemoryFreeClear(p->firstPk.pData); - } + taosMemoryFreeClear(p->firstPk.pData); + taosMemoryFreeClear(p->lastPk.pData); +} - if (p->lastPKLen > 0) { - taosMemoryFreeClear(p->lastPk.pData); +void clearDataBlockIterator(SDataBlockIter* pIter, bool hasPk) { + pIter->index = -1; + pIter->numOfBlocks = 0; + + if (hasPk) { + taosArrayClearEx(pIter->blockList, freePkItem); + } else { + taosArrayClear(pIter->blockList); } } -void clearDataBlockIterator(SDataBlockIter* pIter) { +void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk) { pIter->index = -1; pIter->numOfBlocks = 0; - taosArrayClearEx(pIter->blockList, freeItem); -} - -void cleanupDataBlockIterator(SDataBlockIter* pIter) { - pIter->index = -1; - pIter->numOfBlocks = 0; - taosArrayDestroyEx(pIter->blockList, freeItem); + if (hasPk) { + taosArrayDestroyEx(pIter->blockList, freePkItem); + } else { + taosArrayClear(pIter->blockList); + } } int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { bool asc = ASCENDING_TRAVERSE(pReader->info.order); SBlockOrderSupporter sup = {0}; - clearDataBlockIterator(pBlockIter); + clearDataBlockIterator(pBlockIter, pReader->suppInfo.numOfPks > 0); pBlockIter->numOfBlocks = numOfBlocks; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 0e7895c272..94909aabf4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -349,9 +349,9 @@ int32_t pkCompEx(__compar_fn_t comparFn, SRowKey* p1, SRowKey* p2); int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc); void clearRowKey(SRowKey* pKey); -void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order); -void clearDataBlockIterator(SDataBlockIter* pIter); -void cleanupDataBlockIterator(SDataBlockIter* pIter); +void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk); +void clearDataBlockIterator(SDataBlockIter* pIter, bool hasPk); +void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk); typedef struct { SArray* pTombData; From 563efeb560a9317c927a9798fc0c8d4c9d220127 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 11 Apr 2024 11:09:57 +0800 Subject: [PATCH 5/5] fix(tsdb): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index d93c8c8f79..1fba39227c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -484,7 +484,7 @@ void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk) { if (hasPk) { taosArrayDestroyEx(pIter->blockList, freePkItem); } else { - taosArrayClear(pIter->blockList); + taosArrayDestroy(pIter->blockList); } }