opti:[TD-28142]memory size if block num is too large
This commit is contained in:
wangmm0220
parent
51a472f281
commit
6444af84e4
|
|
@ -882,6 +882,21 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo
|
|||
}
|
||||
}
|
||||
|
||||
static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInfo){
|
||||
record->uid = pBlockInfo->uid;
|
||||
record->firstKey = pBlockInfo->firstKey;
|
||||
record->lastKey = pBlockInfo->lastKey;
|
||||
record->minVer = pBlockInfo->minVer;
|
||||
record->maxVer = pBlockInfo->maxVer;
|
||||
record->blockOffset = pBlockInfo->blockOffset;
|
||||
record->smaOffset = pBlockInfo->smaOffset;
|
||||
record->blockSize = pBlockInfo->blockSize;
|
||||
record->blockKeySize = pBlockInfo->blockKeySize;
|
||||
record->smaSize = pBlockInfo->smaSize;
|
||||
record->numRow = pBlockInfo->numRow;
|
||||
record->count = pBlockInfo->count;
|
||||
}
|
||||
|
||||
static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) {
|
||||
SReaderStatus* pStatus = &pReader->status;
|
||||
SDataBlockIter* pBlockIter = &pStatus->blockIter;
|
||||
|
|
@ -899,7 +914,9 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) {
|
|||
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
|
||||
int32_t step = asc ? 1 : -1;
|
||||
|
||||
SBrinRecord* pRecord = &pBlockInfo->record;
|
||||
SBrinRecord tmp;
|
||||
blockInfoToRecord(&tmp, pBlockInfo);
|
||||
SBrinRecord* pRecord = &tmp;
|
||||
|
||||
// no data exists, return directly.
|
||||
if (pBlockData->nRow == 0 || pBlockData->aTSKEY == 0) {
|
||||
|
|
@ -1092,14 +1109,16 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
|
|||
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter);
|
||||
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
|
||||
|
||||
SBrinRecord* pRecord = &pBlockInfo->record;
|
||||
SBrinRecord tmp;
|
||||
blockInfoToRecord(&tmp, pBlockInfo);
|
||||
SBrinRecord* pRecord = &tmp;
|
||||
code = tsdbDataFileReadBlockDataByColumn(pReader->pFileReader, pRecord, pBlockData, pSchema, &pSup->colId[1],
|
||||
pSup->numOfCols - 1);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
|
||||
", rows:%d, code:%s %s",
|
||||
pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.firstKey,
|
||||
pBlockInfo->record.lastKey, pBlockInfo->record.numRow, tstrerror(code), pReader->idStr);
|
||||
pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->firstKey,
|
||||
pBlockInfo->lastKey, pBlockInfo->numRow, tstrerror(code), pReader->idStr);
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -1120,10 +1139,10 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
|
|||
* This is an two rectangles overlap cases.
|
||||
*/
|
||||
static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SFileDataBlockInfo* pBlock) {
|
||||
return (pWindow->ekey < pBlock->record.lastKey && pWindow->ekey >= pBlock->record.firstKey) ||
|
||||
(pWindow->skey > pBlock->record.firstKey && pWindow->skey <= pBlock->record.lastKey) ||
|
||||
(pVerRange->minVer > pBlock->record.minVer && pVerRange->minVer <= pBlock->record.maxVer) ||
|
||||
(pVerRange->maxVer < pBlock->record.maxVer && pVerRange->maxVer >= pBlock->record.minVer);
|
||||
return (pWindow->ekey < pBlock->lastKey && pWindow->ekey >= pBlock->firstKey) ||
|
||||
(pWindow->skey > pBlock->firstKey && pWindow->skey <= pBlock->lastKey) ||
|
||||
(pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) ||
|
||||
(pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer);
|
||||
}
|
||||
|
||||
static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo,
|
||||
|
|
@ -1142,7 +1161,7 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo
|
|||
STableDataBlockIdx* pTableDataBlockIdx =
|
||||
taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step);
|
||||
SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex);
|
||||
memcpy(pRecord, &p->record, sizeof(SBrinRecord));
|
||||
blockInfoToRecord(pRecord, p);
|
||||
|
||||
*nextIndex = pBlockInfo->tbBlockIdx + step;
|
||||
return true;
|
||||
|
|
@ -1211,9 +1230,9 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte
|
|||
static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order) {
|
||||
// it is the last block in current file, no chance to overlap with neighbor blocks.
|
||||
if (ASCENDING_TRAVERSE(order)) {
|
||||
return pBlock->record.lastKey == pRec->firstKey;
|
||||
return pBlock->lastKey == pRec->firstKey;
|
||||
} else {
|
||||
return pBlock->record.firstKey == pRec->lastKey;
|
||||
return pBlock->firstKey == pRec->lastKey;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1223,9 +1242,9 @@ static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, STableBlockScanI
|
|||
int64_t key = 0;
|
||||
if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) {
|
||||
int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey;
|
||||
key = ascScan ? TMIN(pBlock->record.firstKey, keyInStt) : TMAX(pBlock->record.lastKey, keyInStt);
|
||||
key = ascScan ? TMIN(pBlock->firstKey, keyInStt) : TMAX(pBlock->lastKey, keyInStt);
|
||||
} else {
|
||||
key = ascScan ? pBlock->record.firstKey : pBlock->record.lastKey;
|
||||
key = ascScan ? pBlock->firstKey : pBlock->lastKey;
|
||||
}
|
||||
|
||||
return key;
|
||||
|
|
@ -1241,8 +1260,8 @@ static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBloc
|
|||
}
|
||||
|
||||
static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) {
|
||||
return (key.ts >= pBlock->record.firstKey && key.ts <= pBlock->record.lastKey) &&
|
||||
(pBlock->record.maxVer >= pVerRange->minVer) && (pBlock->record.minVer <= pVerRange->maxVer);
|
||||
return (key.ts >= pBlock->firstKey && key.ts <= pBlock->lastKey) &&
|
||||
(pBlock->maxVer >= pVerRange->minVer) && (pBlock->minVer <= pVerRange->maxVer);
|
||||
}
|
||||
|
||||
static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo,
|
||||
|
|
@ -1258,18 +1277,20 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo*
|
|||
pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock2(pBlockInfo, &rec, pReader->info.order);
|
||||
}
|
||||
|
||||
SBrinRecord pRecord;
|
||||
blockInfoToRecord(&pRecord, pBlockInfo);
|
||||
// has duplicated ts of different version in this block
|
||||
pInfo->hasDupTs = (pBlockInfo->record.numRow > pBlockInfo->record.count) || (pBlockInfo->record.count <= 0);
|
||||
pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pBlockInfo->record, pReader->info.order);
|
||||
pInfo->hasDupTs = (pBlockInfo->numRow > pBlockInfo->count) || (pBlockInfo->count <= 0);
|
||||
pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pRecord, pReader->info.order);
|
||||
|
||||
ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT);
|
||||
if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) {
|
||||
int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey;
|
||||
pInfo->overlapWithSttBlock =
|
||||
!(pBlockInfo->record.lastKey < nextProcKeyInStt || pBlockInfo->record.firstKey > nextProcKeyInStt);
|
||||
!(pBlockInfo->lastKey < nextProcKeyInStt || pBlockInfo->firstKey > nextProcKeyInStt);
|
||||
}
|
||||
|
||||
pInfo->moreThanCapcity = pBlockInfo->record.numRow > pReader->resBlockInfo.capacity;
|
||||
pInfo->moreThanCapcity = pBlockInfo->numRow > pReader->resBlockInfo.capacity;
|
||||
pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->info.window, &pReader->info.verRange, pBlockInfo);
|
||||
pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlockInfo, &pReader->info.verRange);
|
||||
}
|
||||
|
|
@ -2361,17 +2382,14 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
|
|||
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
|
||||
SSttBlockReader* pSttBlockReader = pReader->status.fileIter.pSttBlockReader;
|
||||
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
|
||||
SBrinRecord* pRecord = NULL;
|
||||
|
||||
STableBlockScanInfo* pBlockScanInfo = NULL;
|
||||
if (pBlockInfo == NULL) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
pRecord = &pBlockInfo->record;
|
||||
|
||||
if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) {
|
||||
setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order);
|
||||
setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order);
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -2380,13 +2398,12 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
|
|||
goto _end;
|
||||
}
|
||||
|
||||
pRecord = &pBlockInfo->record;
|
||||
TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader);
|
||||
|
||||
// it is a clean block, load it directly
|
||||
int64_t cap = pReader->resBlockInfo.capacity;
|
||||
if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pRecord->numRow <= cap)) {
|
||||
if (((asc && (pRecord->firstKey < keyInBuf.ts)) || (!asc && (pRecord->lastKey > keyInBuf.ts))) &&
|
||||
if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pBlockInfo->numRow <= cap)) {
|
||||
if (((asc && (pBlockInfo->firstKey < keyInBuf.ts)) || (!asc && (pBlockInfo->lastKey > keyInBuf.ts))) &&
|
||||
(pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA)) {
|
||||
code = copyBlockDataToSDataBlock(pReader);
|
||||
if (code) {
|
||||
|
|
@ -2394,7 +2411,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
|
|||
}
|
||||
|
||||
// record the last key value
|
||||
pBlockScanInfo->lastProcKey = asc ? pRecord->lastKey : pRecord->firstKey;
|
||||
pBlockScanInfo->lastProcKey = asc ? pBlockInfo->lastKey : pBlockInfo->firstKey;
|
||||
goto _end;
|
||||
}
|
||||
}
|
||||
|
|
@ -2422,7 +2439,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
|
|||
bool loadNeighbor = false;
|
||||
code = loadNeighborIfOverlap(pBlockInfo, pBlockScanInfo, pReader, &loadNeighbor);
|
||||
if ((!loadNeighbor) || (code != 0)) {
|
||||
setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order);
|
||||
setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -2441,7 +2458,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
|
|||
|
||||
// currently loaded file data block is consumed
|
||||
if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) {
|
||||
setBlockAllDumped(pDumpInfo, pRecord->lastKey, pReader->info.order);
|
||||
setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order);
|
||||
break;
|
||||
}
|
||||
|
||||
|
|
@ -2801,7 +2818,7 @@ static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, STableBlockSc
|
|||
return true;
|
||||
} else {
|
||||
int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey;
|
||||
return (asc && pBlockInfo->record.lastKey < keyInStt) || (!asc && pBlockInfo->record.firstKey > keyInStt);
|
||||
return (asc && pBlockInfo->lastKey < keyInStt) || (!asc && pBlockInfo->firstKey > keyInStt);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2815,7 +2832,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
|
|||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
||||
if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) {
|
||||
setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order);
|
||||
setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order);
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -2850,20 +2867,20 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
|
|||
if (notOverlapWithSttFiles(pBlockInfo, pScanInfo, asc)) {
|
||||
// whole block is required, return it directly
|
||||
SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info;
|
||||
pInfo->rows = pBlockInfo->record.numRow;
|
||||
pInfo->rows = pBlockInfo->numRow;
|
||||
pInfo->id.uid = pScanInfo->uid;
|
||||
pInfo->dataLoad = 0;
|
||||
pInfo->window = (STimeWindow){.skey = pBlockInfo->record.firstKey, .ekey = pBlockInfo->record.lastKey};
|
||||
pInfo->window = (STimeWindow){.skey = pBlockInfo->firstKey, .ekey = pBlockInfo->lastKey};
|
||||
setComposedBlockFlag(pReader, false);
|
||||
setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order);
|
||||
setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order);
|
||||
|
||||
// update the last key for the corresponding table
|
||||
pScanInfo->lastProcKey = asc ? pInfo->window.ekey : pInfo->window.skey;
|
||||
tsdbDebug("%p uid:%" PRIu64
|
||||
" clean file block retrieved from file, global index:%d, "
|
||||
"table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s",
|
||||
pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->record.numRow,
|
||||
pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr);
|
||||
pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->numRow,
|
||||
pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr);
|
||||
} else {
|
||||
SBlockData* pBData = &pReader->status.fileBlockData;
|
||||
tBlockDataReset(pBData);
|
||||
|
|
@ -2892,10 +2909,10 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
|
|||
|
||||
// data in stt now overlaps with current active file data block, need to composed with file data block.
|
||||
int64_t lastKeyInStt = getCurrentKeyInSttBlock(pSttBlockReader);
|
||||
if ((lastKeyInStt >= pBlockInfo->record.firstKey && asc) ||
|
||||
(lastKeyInStt <= pBlockInfo->record.lastKey && (!asc))) {
|
||||
if ((lastKeyInStt >= pBlockInfo->firstKey && asc) ||
|
||||
(lastKeyInStt <= pBlockInfo->lastKey && (!asc))) {
|
||||
tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader,
|
||||
lastKeyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr);
|
||||
lastKeyInStt, pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -3012,8 +3029,8 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter)
|
|||
lastKey = pScanInfo->lastProcKey;
|
||||
}
|
||||
|
||||
pDumpInfo->totalRows = pBlockInfo->record.numRow;
|
||||
pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->record.numRow - 1;
|
||||
pDumpInfo->totalRows = pBlockInfo->numRow;
|
||||
pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1;
|
||||
} else {
|
||||
pDumpInfo->totalRows = 0;
|
||||
pDumpInfo->rowIndex = 0;
|
||||
|
|
@ -4632,7 +4649,9 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
|
|||
// int64_t st = taosGetTimestampUs();
|
||||
TARRAY2_CLEAR(&pSup->colAggArray, 0);
|
||||
|
||||
code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pFBlock->record, &pSup->colAggArray);
|
||||
SBrinRecord pRecord;
|
||||
blockInfoToRecord(&pRecord, pFBlock);
|
||||
code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pRecord, &pSup->colAggArray);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code),
|
||||
pReader->idStr);
|
||||
|
|
@ -4663,7 +4682,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
|
|||
}
|
||||
|
||||
// do fill all null column value SMA info
|
||||
doFillNullColSMA(pSup, pFBlock->record.numRow, numOfCols, pTsAgg);
|
||||
doFillNullColSMA(pSup, pFBlock->numRow, numOfCols, pTsAgg);
|
||||
|
||||
size_t size = pSup->colAggArray.size;
|
||||
|
||||
|
|
@ -4884,7 +4903,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT
|
|||
while (true) {
|
||||
if (hasNext) {
|
||||
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter);
|
||||
int32_t numOfRows = pBlockInfo->record.numRow;
|
||||
int32_t numOfRows = pBlockInfo->numRow;
|
||||
|
||||
pTableBlockInfo->totalRows += numOfRows;
|
||||
|
||||
|
|
@ -4896,7 +4915,7 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT
|
|||
pTableBlockInfo->minRows = numOfRows;
|
||||
}
|
||||
|
||||
pTableBlockInfo->totalSize += pBlockInfo->record.blockSize;
|
||||
pTableBlockInfo->totalSize += pBlockInfo->blockSize;
|
||||
|
||||
int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows, numOfBuckets);
|
||||
pTableBlockInfo->blockRowsHisto[bucketIndex]++;
|
||||
|
|
|
|||
|
|
@ -359,6 +359,21 @@ static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, v
|
|||
return pLeftBlock->offset > pRightBlock->offset ? 1 : -1;
|
||||
}
|
||||
|
||||
static void recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record){
|
||||
pBlockInfo->uid = record->uid;
|
||||
pBlockInfo->firstKey = record->firstKey;
|
||||
pBlockInfo->lastKey = record->lastKey;
|
||||
pBlockInfo->minVer = record->minVer;
|
||||
pBlockInfo->maxVer = record->maxVer;
|
||||
pBlockInfo->blockOffset = record->blockOffset;
|
||||
pBlockInfo->smaOffset = record->smaOffset;
|
||||
pBlockInfo->blockSize = record->blockSize;
|
||||
pBlockInfo->blockKeySize = record->blockKeySize;
|
||||
pBlockInfo->smaSize = record->smaSize;
|
||||
pBlockInfo->numRow = record->numRow;
|
||||
pBlockInfo->count = record->count;
|
||||
}
|
||||
|
||||
int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) {
|
||||
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
|
||||
|
||||
|
|
@ -416,8 +431,9 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
|
|||
pTableScanInfo->pBlockIdxList = taosArrayInit(numOfBlocks, sizeof(STableDataBlockIdx));
|
||||
}
|
||||
for (int32_t i = 0; i < numOfBlocks; ++i) {
|
||||
SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[0][i].uid, .tbBlockIdx = i};
|
||||
blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i);
|
||||
SFileDataBlockInfo blockInfo = {.tbBlockIdx = i};
|
||||
SBrinRecord* record = (SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[0][i].pInfo->pBlockList, i);
|
||||
recordToBlockInfo(&blockInfo, record);
|
||||
|
||||
taosArrayPush(pBlockIter->blockList, &blockInfo);
|
||||
STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};
|
||||
|
|
@ -450,8 +466,9 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
|
|||
int32_t pos = tMergeTreeGetChosenIndex(pTree);
|
||||
int32_t index = sup.indexPerTable[pos]++;
|
||||
|
||||
SFileDataBlockInfo blockInfo = {.uid = sup.pDataBlockInfo[pos][index].uid, .tbBlockIdx = index};
|
||||
blockInfo.record = *(SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index);
|
||||
SFileDataBlockInfo blockInfo = {.tbBlockIdx = index};
|
||||
SBrinRecord* record = (SBrinRecord*)taosArrayGet(sup.pDataBlockInfo[pos][index].pInfo->pBlockList, index);
|
||||
recordToBlockInfo(&blockInfo, record);
|
||||
|
||||
taosArrayPush(pBlockIter->blockList, &blockInfo);
|
||||
STableBlockScanInfo* pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo;
|
||||
|
|
|
|||
|
|
@ -175,9 +175,22 @@ typedef struct SFilesetIter {
|
|||
|
||||
typedef struct SFileDataBlockInfo {
|
||||
// index position in STableBlockScanInfo in order to check whether neighbor block overlaps with it
|
||||
uint64_t uid;
|
||||
int32_t tbBlockIdx;
|
||||
SBrinRecord record;
|
||||
// int64_t suid;
|
||||
int64_t uid;
|
||||
int64_t firstKey;
|
||||
// int64_t firstKeyVer;
|
||||
int64_t lastKey;
|
||||
// int64_t lastKeyVer;
|
||||
int64_t minVer;
|
||||
int64_t maxVer;
|
||||
int64_t blockOffset;
|
||||
int64_t smaOffset;
|
||||
int32_t blockSize;
|
||||
int32_t blockKeySize;
|
||||
int32_t smaSize;
|
||||
int32_t numRow;
|
||||
int32_t count;
|
||||
int32_t tbBlockIdx;
|
||||
} SFileDataBlockInfo;
|
||||
|
||||
typedef struct SDataBlockIter {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@
|
|||
#include "tcoding.h"
|
||||
|
||||
// todo refactor API
|
||||
#define BOUNDARY_SIZE 1024*1024*1024 // 1G
|
||||
#define BOUNDARY_SMALL_FACTOR 1.2
|
||||
#define BOUNDARY_BIG_FACTOR 2
|
||||
|
||||
SArray* taosArrayInit(size_t size, size_t elemSize) {
|
||||
if (elemSize == 0) {
|
||||
|
|
@ -85,9 +88,13 @@ static int32_t taosArrayResize(SArray* pArray) {
|
|||
|
||||
int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) {
|
||||
if (newCap > pArray->capacity) {
|
||||
size_t tsize = (pArray->capacity << 1u);
|
||||
float factor = BOUNDARY_BIG_FACTOR;
|
||||
if(newCap * pArray->elemSize > BOUNDARY_SIZE){
|
||||
factor = BOUNDARY_SMALL_FACTOR;
|
||||
}
|
||||
size_t tsize = (pArray->capacity * factor);
|
||||
while (newCap > tsize) {
|
||||
tsize = (tsize << 1u);
|
||||
tsize = (tsize * factor);
|
||||
}
|
||||
|
||||
pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize);
|
||||
|
|
|
|||
Loading…
Reference in New Issue