xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

partition interval and limimt, dataload error

This commit is contained in:
factosea 2024-05-31 18:58:35 +08:00
parent c48db6b212
commit a6217eec03
2 changed files with 94 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
source/libs/executor/src/groupoperator.c
View File

@ -53,6 +53,8 @@ typedef struct SPartitionOperatorInfo {
int32_t rowCapacity; // maximum number of rows for each buffer page int32_t rowCapacity; // maximum number of rows for each buffer page
int32_t* columnOffset; // start position for each column data int32_t* columnOffset; // start position for each column data
SArray* sortedGroupArray; // SDataGroupInfo sorted by group id SArray* sortedGroupArray; // SDataGroupInfo sorted by group id
SArray* blockForNotLoaded; // SSDataBlock that data is not loaded
int32_t offsetForNotLoaded;// read offset for SSDataBlock that data is not loaded
int32_t groupIndex; // group index int32_t groupIndex; // group index
int32_t pageIndex; // page index of current group int32_t pageIndex; // page index of current group
SExprSupp scalarSup; SExprSupp scalarSup;
@ -584,6 +586,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
pGroupInfo->groupId = calcGroupId(pInfo->keyBuf, len); pGroupInfo->groupId = calcGroupId(pInfo->keyBuf, len);
} }
if (pBlock->info.dataLoad) {
// number of rows // number of rows
int32_t* rows = (int32_t*)pPage; int32_t* rows = (int32_t*)pPage;
@ -649,6 +652,20 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
setBufPageDirty(pPage, true); setBufPageDirty(pPage, true);
releaseBufPage(pInfo->pBuf, pPage); releaseBufPage(pInfo->pBuf, pPage);
} else {
SSDataBlock* dataNotLoadBlock = createOneDataBlock(pBlock, true);
if (dataNotLoadBlock == NULL) {
T_LONG_JMP(pTaskInfo->env, terrno);
}
if (pInfo->blockForNotLoaded == NULL) {
pInfo->blockForNotLoaded = taosArrayInit(1, sizeof(SSDataBlock));
pInfo->offsetForNotLoaded = 0;
}
dataNotLoadBlock->info.id.groupId = pGroupInfo->groupId;
dataNotLoadBlock->info.dataLoad = 0;
taosArrayInsert(pInfo->blockForNotLoaded, pInfo->blockForNotLoaded->size, &dataNotLoadBlock);
break;
}
} }
} }
@ -734,6 +751,14 @@ static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) {
} }
taosArrayClear(pInfo->sortedGroupArray); taosArrayClear(pInfo->sortedGroupArray);
clearDiskbasedBuf(pInfo->pBuf); clearDiskbasedBuf(pInfo->pBuf);
if (pInfo->blockForNotLoaded) {
for (int32_t i = 0; i < pInfo->blockForNotLoaded->size; i++) {
SSDataBlock** pBlock = taosArrayGet(pInfo->blockForNotLoaded, i);
blockDataDestroy(*pBlock);
}
taosArrayClear(pInfo->blockForNotLoaded);
pInfo->offsetForNotLoaded = 0;
}
} }
static int compareDataGroupInfo(const void* group1, const void* group2) { static int compareDataGroupInfo(const void* group1, const void* group2) {
@ -747,6 +772,21 @@ static int compareDataGroupInfo(const void* group1, const void* group2) {
return (pGroupInfo1->groupId < pGroupInfo2->groupId) ? -1 : 1; return (pGroupInfo1->groupId < pGroupInfo2->groupId) ? -1 : 1;
} }
static SSDataBlock* buildPartitionResultForNotLoadBlock(SOperatorInfo* pOperator) {
SPartitionOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
if (pInfo->blockForNotLoaded && pInfo->offsetForNotLoaded < pInfo->blockForNotLoaded->size) {
SSDataBlock** pBlock = taosArrayGet(pInfo->blockForNotLoaded, pInfo->offsetForNotLoaded);
pInfo->offsetForNotLoaded++;
return *pBlock;
} else {
setOperatorCompleted(pOperator);
clearPartitionOperator(pInfo);
return NULL;
}
}
static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
SPartitionOperatorInfo* pInfo = pOperator->info; SPartitionOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
@ -757,12 +797,10 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
(pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL;
if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) {
// try next group data // try next group data
++pInfo->groupIndex; if (pInfo->groupIndex + 1 >= taosArrayGetSize(pInfo->sortedGroupArray)) {
if (pInfo->groupIndex >= taosArrayGetSize(pInfo->sortedGroupArray)) { return buildPartitionResultForNotLoadBlock(pOperator);
setOperatorCompleted(pOperator);
clearPartitionOperator(pInfo);
return NULL;
} }
++pInfo->groupIndex;
pGroupInfo = taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex); pGroupInfo = taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex);
pInfo->pageIndex = 0; pInfo->pageIndex = 0;

1
source/libs/executor/src/scanoperator.c
View File

@ -769,6 +769,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
SSDataBlock* pBlock = pTableScanInfo->pResBlock; SSDataBlock* pBlock = pTableScanInfo->pResBlock;
bool hasNext = false; bool hasNext = false;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
pBlock->info.dataLoad = false;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();