xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enhance: refactor create initial sources

This commit is contained in:
slzhou 2023-07-18 10:38:40 +08:00
parent ba2b404295
commit 97a6e89d11
2 changed files with 150 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
source/libs/executor/src/scanoperator.c
View File

@ -2840,7 +2840,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
int32_t tableStartIdx = pInfo->tableStartIndex; int32_t tableStartIdx = pInfo->tableStartIndex;
int32_t tableEndIdx = pInfo->tableEndIndex; int32_t tableEndIdx = pInfo->tableEndIndex;
pInfo->sortBufSize = 2048 * pInfo->bufPageSize; pInfo->sortBufSize = 1024 * pInfo->bufPageSize;
int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_TABLE_MERGE_SCAN, pInfo->bufPageSize, numOfBufPage, pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_TABLE_MERGE_SCAN, pInfo->bufPageSize, numOfBufPage,
pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);

123
source/libs/executor/src/tsort.c
View File

@ -475,11 +475,11 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT
if (pHandle->type == SORT_SINGLESOURCE_SORT) { if (pHandle->type == SORT_SINGLESOURCE_SORT) {
pSource->pageIndex++; pSource->pageIndex++;
if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) { if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) {
qInfo("adjust merge tree. %d source completed %d", *numOfCompleted, pSource->pageIndex);
(*numOfCompleted) += 1; (*numOfCompleted) += 1;
pSource->src.rowIndex = -1; pSource->src.rowIndex = -1;
pSource->pageIndex = -1; pSource->pageIndex = -1;
pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock); pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock);
uInfo("adjust merge tree. %d source completed", *numOfCompleted);
} else { } else {
int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex); int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex);
@ -495,8 +495,6 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT
} }
releaseBufPage(pHandle->pBuf, pPage); releaseBufPage(pHandle->pBuf, pPage);
if (pSource->pageIndex % 256 == 0)
uInfo("got block from page %d from ext mem source %p", pSource->pageIndex, pSource);
} }
} else { } else {
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
@ -506,7 +504,7 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT
if (pSource->src.pBlock == NULL) { if (pSource->src.pBlock == NULL) {
(*numOfCompleted) += 1; (*numOfCompleted) += 1;
pSource->src.rowIndex = -1; pSource->src.rowIndex = -1;
uInfo("adjust merge tree. %d source completed", *numOfCompleted); qInfo("adjust merge tree. %d source completed", *numOfCompleted);
} }
} }
} }
@ -688,7 +686,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
// Only *numOfInputSources* can be loaded into buffer to perform the external sort. // Only *numOfInputSources* can be loaded into buffer to perform the external sort.
for (int32_t i = 0; i < sortGroup; ++i) { for (int32_t i = 0; i < sortGroup; ++i) {
uInfo("internal merge sort pass %d group %d. num input sources %d ", t, i, numOfInputSources); qInfo("internal merge sort pass %d group %d. num input sources %d ", t, i, numOfInputSources);
pHandle->sourceId += 1; pHandle->sourceId += 1;
int32_t end = (i + 1) * numOfInputSources - 1; int32_t end = (i + 1) * numOfInputSources - 1;
@ -884,11 +882,13 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO
int32_t minIdx = tMergeTreeGetChosenIndex(pTree); int32_t minIdx = tMergeTreeGetChosenIndex(pTree);
SSDataBlock* minBlk = taosArrayGetP(aBlk, minIdx); SSDataBlock* minBlk = taosArrayGetP(aBlk, minIdx);
int32_t minRow = sup.aRowIdx[minIdx]; int32_t minRow = sup.aRowIdx[minIdx];
appendOneRowToDataBlock(pHandle->pDataBlock, minBlk, &minRow); appendOneRowToDataBlock(pHandle->pDataBlock, minBlk, &minRow);
++nRows; ++nRows;
if (pHandle->pDataBlock->info.rows >= rowCap) { if (pHandle->pDataBlock->info.rows >= rowCap) {
appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId); appendDataBlockToPageBuf(pHandle, pHandle->pDataBlock, aPgId);
} }
if (sup.aRowIdx[minIdx] == minBlk->info.rows - 1) { if (sup.aRowIdx[minIdx] == minBlk->info.rows - 1) {
sup.aRowIdx[minIdx] = -1; sup.aRowIdx[minIdx] = -1;
++numEnded; ++numEnded;
@ -911,11 +911,56 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO
return 0; return 0;
} }
static int32_t createInitialSources(SSortHandle* pHandle) { static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0);
int32_t code = 0; size_t nSrc = taosArrayGetSize(pHandle->pOrderedSource);
SArray* aExtSrc = taosArrayInit(nSrc, POINTER_BYTES);
size_t maxBufSize = pHandle->numOfPages * pHandle->pageSize;
createPageBuf(pHandle);
SSortSource* pSrc = taosArrayGetP(pHandle->pOrderedSource, 0);
int32_t szSort = 0;
SArray* aBlkSort = taosArrayInit(8, POINTER_BYTES);
while (1) {
SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param);
if (pBlk != NULL) {
szSort += blockDataGetSize(pBlk);
SSDataBlock* blk = createOneDataBlock(pBlk, true);
taosArrayPush(aBlkSort, &blk);
}
if ((pBlk != NULL && szSort > maxBufSize) || (pBlk == NULL && szSort > 0)) {
int64_t p = taosGetTimestampUs();
sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc);
int64_t el = taosGetTimestampUs() - p;
pHandle->sortElapsed += el;
for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) {
blockDataDestroy(taosArrayGetP(aBlkSort, i));
}
taosArrayClear(aBlkSort);
szSort = 0;
}
if (pBlk == NULL) {
break;
};
}
taosArrayDestroy(aBlkSort);
tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL);
taosArrayAddAll(pHandle->pOrderedSource, aExtSrc);
taosArrayDestroy(aExtSrc);
pHandle->type = SORT_SINGLESOURCE_SORT;
return 0;
}
static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
int32_t code = 0;
size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize;
if (pHandle->type == SORT_SINGLESOURCE_SORT) {
SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0); SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0);
SSortSource* source = *pSource; SSortSource* source = *pSource;
*pSource = NULL; *pSource = NULL;
@ -1017,56 +1062,22 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
code = doAddToBuf(pHandle->pDataBlock, pHandle); code = doAddToBuf(pHandle->pDataBlock, pHandle);
} }
} }
return code;
}
static int32_t createInitialSources(SSortHandle* pHandle) {
int32_t code = 0;
if (pHandle->type == SORT_SINGLESOURCE_SORT) {
code = createBlocksQuickSortInitialSources(pHandle);
} else if (pHandle->type == SORT_TABLE_MERGE_SCAN) { } else if (pHandle->type == SORT_TABLE_MERGE_SCAN) {
SBlockOrderInfo* pOrder = taosArrayGet(pHandle->pSortInfo, 0); code = createBlocksMergeSortInitialSources(pHandle);
size_t nSrc = taosArrayGetSize(pHandle->pOrderedSource);
SArray* aExtSrc = taosArrayInit(nSrc, POINTER_BYTES);
size_t maxBufSize = pHandle->numOfPages * pHandle->pageSize;
createPageBuf(pHandle);
SSortSource* pSrc = taosArrayGetP(pHandle->pOrderedSource, 0);
int32_t szSort = 0;
SArray* aBlkSort = taosArrayInit(8, POINTER_BYTES);
while (1) {
SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param);
if (pBlk == NULL) {
break;
};
szSort += blockDataGetSize(pBlk);
SSDataBlock* blk = createOneDataBlock(pBlk, true);
taosArrayPush(aBlkSort, &blk);
if (szSort > maxBufSize) {
sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc);
uInfo("initial source %zu created for %zu blocks", taosArrayGetSize(aExtSrc), taosArrayGetSize(aBlkSort));
for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) {
blockDataDestroy(taosArrayGetP(aBlkSort, i));
} }
taosArrayClear(aBlkSort); qInfo("%zu sources created", taosArrayGetSize(pHandle->pOrderedSource));
szSort = 0; for (int i = 0; i < taosArrayGetSize(pHandle->pOrderedSource); ++i) {
SSortSource* pSrc = taosArrayGetP(pHandle->pOrderedSource, i);
qInfo("source %d, num of pages %zu", i,taosArrayGetSize(pSrc->pageIdList));
} }
}
if (szSort > 0) {
sortBlocksToExtSource(pHandle, aBlkSort, pOrder, aExtSrc);
for (int i = 0; i < taosArrayGetSize(aBlkSort); ++i) {
blockDataDestroy(taosArrayGetP(aBlkSort, i));
}
taosArrayClear(aBlkSort);
szSort = 0;
}
taosArrayDestroy(aBlkSort);
tsortClearOrderdSource(pHandle->pOrderedSource, NULL, NULL);
taosArrayAddAll(pHandle->pOrderedSource, aExtSrc);
taosArrayDestroy(aExtSrc);
pHandle->type = SORT_SINGLESOURCE_SORT;
uInfo("create initial sources for table merge scan ended");
}
return code; return code;
} }