xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh(query): retrieve in-memory data.

This commit is contained in:
Haojun Liao 2022-06-28 10:32:42 +08:00
parent aac1fb643f
commit c638c33a76
1 changed files with 412 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

609
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -63,8 +63,8 @@ typedef struct STableBlockScanInfo {
// int32_t numOfBlocks : 29; // number of qualified data blocks not the original blocks // int32_t numOfBlocks : 29; // number of qualified data blocks not the original blocks
uint8_t chosen : 2; // indicate which iterator should move forward uint8_t chosen : 2; // indicate which iterator should move forward
bool iterInit; // whether to initialize the in-memory skip list iterator or not bool iterInit; // whether to initialize the in-memory skip list iterator or not
STbDataIter iter; // mem buffer skip list iterator STbDataIter* iter; // mem buffer skip list iterator
STbDataIter iiter; // imem buffer skip list iterator STbDataIter* iiter; // imem buffer skip list iterator
bool memHasVal; bool memHasVal;
bool imemHasVal; bool imemHasVal;
} STableBlockScanInfo; } STableBlockScanInfo;
@ -125,18 +125,15 @@ typedef struct SComposedDataBlock {
} SComposedDataBlock; } SComposedDataBlock;
typedef struct SReaderStatus { typedef struct SReaderStatus {
SQueryFilePos cur; // current position SQueryFilePos cur; // current position
int32_t tableListIndex; bool loadFromFile; // check file stage
bool loadFromFile; // check file stage SHashObj* pTableMap; // SHash<STableBlockScanInfo>
bool initStartPos; STableBlockScanInfo* pTableIter; // table iterator used in building in-memory buffer data blocks.
SHashObj* pTableMap; // SHash<STableBlockScanInfo>
int32_t realNumOfRows;
SFileBlockDumpInfo fBlockDumpInfo; SFileBlockDumpInfo fBlockDumpInfo;
SBlockData fileBlockData; SBlockData fileBlockData;
SFileSetIter fileIter;
SFileSetIter fileIter; SDataBlockIter blockIter;
SDataBlockIter blockIter; bool composedDataBlock;// the returned data block is a composed block or not
} SReaderStatus; } SReaderStatus;
struct STsdbReader { struct STsdbReader {
@ -173,10 +170,13 @@ struct STsdbReader {
}; };
static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter);
static int tsdbReadRowsFromCache(STableBlockScanInfo* pScanInfo, TSDBKEY maxKey, int32_t capacity, STsdbReader* pReader); static int buildInmemDataBlockImpl(STableBlockScanInfo* pBlockScanInfo, TSDBKEY maxKey, int32_t capacity, STsdbReader* pReader);
static TSDBROW* getValidRow(STbDataIter* pIter, bool* hasVal, STsdbReader* pReader); static TSDBROW* getValidRow(STbDataIter* pIter, bool* hasVal, STsdbReader* pReader);
static int32_t doLoadRowsOfIdenticalTsInFileBlock(SBlockData* pData, SFileBlockDumpInfo* pDumpInfo, int64_t ts, SRowMerger *pMerger, static int32_t doLoadRowsOfIdenticalTsInFileBlock(SFileDataBlockInfo* pFBlock, SBlock* pBlock, SBlockData* pBlockData,
STsdbReader* pReader, STSRow** pRow); STableBlockScanInfo* pScanInfo, STsdbReader* pReader, SRowMerger* pMerger);
static int32_t doLoadRowsOfIdenticalTs(STbDataIter *pIter, bool* hasVal, int64_t ts, SRowMerger* pMerger, STsdbReader* pReader);
static int32_t doAppendOneRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow);
static void setComposedBlockFlag(STsdbReader* pReader, bool composed);
// static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { // static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) {
// pBlockLoadInfo->slot = -1; // pBlockLoadInfo->slot = -1;
@ -204,7 +204,7 @@ static int32_t setColumnIdList(STsdbReader* pReader, SSDataBlock* pBlock) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const uint64_t* idList, int32_t numOfTables) { static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableKeyInfo* idList, int32_t numOfTables) {
ASSERT(numOfTables >= 1); ASSERT(numOfTables >= 1);
// allocate buffer in order to load data blocks from file // allocate buffer in order to load data blocks from file
@ -216,7 +216,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const uint64_
// todo apply the lastkey of table check to avoid to load header file // todo apply the lastkey of table check to avoid to load header file
for (int32_t j = 0; j < numOfTables; ++j) { for (int32_t j = 0; j < numOfTables; ++j) {
STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j]}; STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid};
if (ASCENDING_TRAVERSE(pTsdbReader->order)) { if (ASCENDING_TRAVERSE(pTsdbReader->order)) {
if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) { if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) {
info.lastKey = pTsdbReader->window.skey; info.lastKey = pTsdbReader->window.skey;
@ -337,7 +337,7 @@ static void resetDataBlockIterator(SDataBlockIter* pIter) {
pIter->numOfBlocks = -1; pIter->numOfBlocks = -1;
} }
static bool nextFilesetIterator(SFileSetIter* pIter, int32_t order, STsdbReader* pReader) { static bool filesetIteratorNext(SFileSetIter* pIter, int32_t order, STsdbReader* pReader) {
if (pIter->index >= pIter->numOfFiles) { if (pIter->index >= pIter->numOfFiles) {
return false; return false;
} }
@ -370,8 +370,7 @@ static bool nextFilesetIterator(SFileSetIter* pIter, int32_t order, STsdbReader*
static void initReaderStatus(SReaderStatus* pStatus) { static void initReaderStatus(SReaderStatus* pStatus) {
pStatus->cur.fid = INT32_MIN; pStatus->cur.fid = INT32_MIN;
pStatus->cur.win = TSWINDOW_INITIALIZER; pStatus->cur.win = TSWINDOW_INITIALIZER;
pStatus->initStartPos = false; pStatus->pTableIter = NULL;
pStatus->tableListIndex = 0; // current active table index
pStatus->loadFromFile = true; pStatus->loadFromFile = true;
} }
@ -384,13 +383,18 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
} }
initReaderStatus(&pReader->status); initReaderStatus(&pReader->status);
pReader->pTsdb = pVnode->pTsdb; pReader->pTsdb = pVnode->pTsdb;
pReader->suid = pCond->suid; pReader->suid = pCond->suid;
pReader->order = pCond->order; pReader->order = pCond->order;
pReader->capacity = 4096; pReader->capacity = 4096;
pReader->idStr = strdup(idstr); pReader->idStr = strdup(idstr);
pReader->startVersion= pCond->startVersion; pReader->startVersion= pCond->startVersion;
pReader->endVersion = pCond->endVersion; pReader->endVersion = 100000;//pCond->endVersion; // todo for test purpose
pReader->type = pCond->type;
pReader->window = *pCond->twindows;
pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1);
// todo remove this // todo remove this
setQueryTimewindow(pReader, pCond, 0); setQueryTimewindow(pReader, pCond, 0);
@ -405,7 +409,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
goto _end; goto _end;
} }
// todo use new api refactor this // todo use new api refactor this after merge with 3.0
pReader->pResBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pReader->pResBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
if (pReader->pResBlock == NULL) { if (pReader->pResBlock == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -418,13 +422,15 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
colInfo.info = pCond->colList[i]; colInfo.info = pCond->colList[i];
taosArrayPush(pReader->pResBlock->pDataBlock, &colInfo); taosArrayPush(pReader->pResBlock->pDataBlock, &colInfo);
} }
pReader->pResBlock->info.numOfCols = taosArrayGetSize(pReader->pResBlock->pDataBlock);
blockDataEnsureCapacity(pReader->pResBlock, pReader->capacity);
setColumnIdList(pReader, pReader->pResBlock); setColumnIdList(pReader, pReader->pResBlock);
pReader->suppInfo.slotIds = taosMemoryCalloc(pCond->numOfCols, sizeof(int32_t)); pReader->suppInfo.slotIds = taosMemoryCalloc(pCond->numOfCols, sizeof(int32_t));
pReader->suppInfo.plist = taosMemoryCalloc(pCond->numOfCols, POINTER_BYTES); pReader->suppInfo.plist = taosMemoryCalloc(pCond->numOfCols, POINTER_BYTES);
} }
// todo refactor
STsdbFSState* pFState = pReader->pTsdb->fs->cState; STsdbFSState* pFState = pReader->pTsdb->fs->cState;
initFileIterator(&pReader->status.fileIter, pFState); initFileIterator(&pReader->status.fileIter, pFState);
resetDataBlockIterator(&pReader->status.blockIter); resetDataBlockIterator(&pReader->status.blockIter);
@ -443,39 +449,6 @@ _end:
return code; return code;
} }
// static int32_t setCurrentSchema(SVnode* pVnode, STsdbReader* pTsdbReadHandle) {
// STableBlockScanInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, 0);
// int32_t sversion = 1;
// SMetaReader mr = {0};
// metaReaderInit(&mr, pVnode->pMeta, 0);
// int32_t code = metaGetTableEntryByUid(&mr, pCheckInfo->tableId);
// if (code != TSDB_CODE_SUCCESS) {
// terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
// metaReaderClear(&mr);
// return terrno;
// }
// if (mr.me.type == TSDB_CHILD_TABLE) {
// tb_uid_t suid = mr.me.ctbEntry.suid;
// code = metaGetTableEntryByUid(&mr, suid);
// if (code != TSDB_CODE_SUCCESS) {
// terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
// metaReaderClear(&mr);
// return terrno;
// }
// sversion = mr.me.stbEntry.schemaRow.version;
// } else {
// ASSERT(mr.me.type == TSDB_NORMAL_TABLE);
// sversion = mr.me.ntbEntry.schemaRow.version;
// }
// metaReaderClear(&mr);
// pTsdbReadHandle->pSchema = metaGetTbTSchema(pVnode->pMeta, pCheckInfo->tableId, sversion);
// return TSDB_CODE_SUCCESS;
// }
// void tsdbResetQueryHandleForNewTable(STsdbReader* queryHandle, SQueryTableDataCond* pCond, STableListInfo* tableList, // void tsdbResetQueryHandleForNewTable(STsdbReader* queryHandle, SQueryTableDataCond* pCond, STableListInfo* tableList,
// int32_t tWinIdx) { // int32_t tWinIdx) {
// STsdbReader* pTsdbReadHandle = queryHandle; // STsdbReader* pTsdbReadHandle = queryHandle;
@ -796,7 +769,7 @@ _end:
// int32_t step = ASCENDING_TRAVERSE(pHandle->order) ? 1 : -1; // int32_t step = ASCENDING_TRAVERSE(pHandle->order) ? 1 : -1;
// STimeWindow* win = &pHandle->cur.win; // STimeWindow* win = &pHandle->cur.win;
// pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo, pHandle->window.ekey, pHandle->outputCapacity, win, pHandle); // pHandle->cur.rows = buildInmemDataBlockImpl(pCheckInfo, pHandle->window.ekey, pHandle->outputCapacity, win, pHandle);
// // update the last key value // // update the last key value
// pCheckInfo->lastKey = win->ekey + step; // pCheckInfo->lastKey = win->ekey + step;
@ -1036,7 +1009,7 @@ _error:
// TSKEY maxKey = ascScan ? (binfo.window.skey - step) : (binfo.window.ekey - step); // TSKEY maxKey = ascScan ? (binfo.window.skey - step) : (binfo.window.ekey - step);
// cur->rows = // cur->rows =
// tsdbReadRowsFromCache(pCheckInfo, maxKey, pTsdbReadHandle->outputCapacity, &cur->win, pTsdbReadHandle); // buildInmemDataBlockImpl(pCheckInfo, maxKey, pTsdbReadHandle->outputCapacity, &cur->win, pTsdbReadHandle);
// pTsdbReadHandle->realNumOfRows = cur->rows; // pTsdbReadHandle->realNumOfRows = cur->rows;
// // update the last key value // // update the last key value
@ -2083,6 +2056,15 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool blockIteratorNext(STsdbReader* pReader, SDataBlockIter* pBlockIter) {
if (pBlockIter->index >= pBlockIter->numOfBlocks - 1) {
return false;
}
pBlockIter->index += 1;
return true;
}
// static int32_t getFirstFileDataBlock(STsdbReader* pTsdbReadHandle, bool* exists); // static int32_t getFirstFileDataBlock(STsdbReader* pTsdbReadHandle, bool* exists);
//static int32_t getDataBlock(STsdbReader* pTsdbReadHandle, SFileBlockInfo* pNext, bool* exists) { //static int32_t getDataBlock(STsdbReader* pTsdbReadHandle, SFileBlockInfo* pNext, bool* exists) {
@ -2228,77 +2210,271 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) {
return pFBlockInfo; return pFBlockInfo;
} }
static bool overlapWithNeighborBlock(SBlock* pBlock, int32_t blockIndex) { static bool overlapWithNeighborBlock(SFileDataBlockInfo *pFBlockInfo, SBlock* pBlock, STableBlockScanInfo* pTableBlockScanInfo) {
return false; // it is the last block in current file, no chance to overlap with neighbor blocks.
if(pFBlockInfo->tbBlockIdx == taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) { // last block in current file,
return false;
}
SBlock* pNext = taosArrayGet(pTableBlockScanInfo->pBlockList, pFBlockInfo->tbBlockIdx + 1);
return (pNext->minKey.ts == pBlock->maxKey.ts);
} }
static bool bufferDataInFileBlockGap(int32_t order, int64_t key, SBlock* pBlock) { static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SBlock* pBlock) {
bool ascScan = ASCENDING_TRAVERSE(order); bool ascScan = ASCENDING_TRAVERSE(order);
return (ascScan && (key != TSKEY_INITIAL_VAL && key <= pBlock->minKey.ts)) || return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->minKey.ts)) ||
(!ascScan && (key != TSKEY_INITIAL_VAL && key >= pBlock->maxKey.ts)); (!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->maxKey.ts));
} }
static int32_t buildInmemDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlock* pBlock, TSDBKEY *key) { static int32_t buildInmemDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBKEY *key) {
int32_t code = TSDB_CODE_SUCCESS; if (pBlockScanInfo->iter != NULL) {
pBlockScanInfo->memHasVal = tsdbTbDataIterNext(pBlockScanInfo->iter);
} else {
pBlockScanInfo->memHasVal = false;
}
bool ascScan = ASCENDING_TRAVERSE(pReader->order); if (pBlockScanInfo->iiter != NULL) {
bool cacheDataInFileBlockHole = (ascScan && (key->ts != TSKEY_INITIAL_VAL && key->ts < pBlock->minKey.ts)) || pBlockScanInfo->imemHasVal = tsdbTbDataIterNext(pBlockScanInfo->iiter);
(!ascScan && (key->ts != TSKEY_INITIAL_VAL && key->ts > pBlock->maxKey.ts)); } else {
ASSERT(cacheDataInFileBlockHole); pBlockScanInfo->imemHasVal = false;
}
// do not load file block into buffer if (!(pBlockScanInfo->imemHasVal || pBlockScanInfo->memHasVal)) {
int32_t step = ascScan ? 1 : -1; return TSDB_CODE_SUCCESS;
}
TSDBKEY maxKey = {.version = pReader->endVersion}; int32_t code = buildInmemDataBlockImpl(pBlockScanInfo, *key, pReader->capacity, pReader);
maxKey.ts = ascScan ? (pBlock->minKey.ts - step) : (pBlock->maxKey.ts - step); setComposedBlockFlag(pReader, true);
pBlockScanInfo->memHasVal = tsdbTbDataIterNext(&pBlockScanInfo->iter); // set the correct block data info
pBlockScanInfo->imemHasVal = tsdbTbDataIterNext(&pBlockScanInfo->iiter);
code = tsdbReadRowsFromCache(pBlockScanInfo, maxKey, pReader->capacity, pReader);
return code; return code;
} }
static int32_t buildComposedDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SBlock* pBlock, STableBlockScanInfo* pBlockScanInfo) {
SFileBlockDumpInfo *pDumpInfo = &pReader->status.fBlockDumpInfo; SFileBlockDumpInfo *pDumpInfo = &pReader->status.fBlockDumpInfo;
SBlockData* pData = &pReader->status.fileBlockData; SBlockData* pBlockData = &pReader->status.fileBlockData;
STSchema* pSchema = NULL;
SRowMerger merge = {0}; SRowMerger merge = {0};
STSRow* pTSRow = NULL;
TSKEY mergeTs = TSKEY_INITIAL_VAL;
int64_t key = pData->aTSKEY[0]; int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex];
TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, pReader); TSDBROW* pRow = getValidRow(pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, pReader);
TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, pReader); TSDBROW* piRow = getValidRow(pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, pReader);
if (pBlockScanInfo->memHasVal && pBlockScanInfo->imemHasVal) { if (pBlockScanInfo->memHasVal && pBlockScanInfo->imemHasVal) {
TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY k = TSDBROW_KEY(pRow);
TSDBKEY ik = TSDBROW_KEY(piRow); TSDBKEY ik = TSDBROW_KEY(piRow);
// todo check version in file // [1&2] key <= [k.ts|ik.ts]
if (key < k.ts || key < ik.ts) { if (key <= k.ts || key <= ik.ts) {
tRowMergerInit(&merge, NULL, pSchema); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
STSRow* pTsRow = NULL; tRowMergerInit(&merge, &fRow, pReader->pSchema);
doLoadRowsOfIdenticalTsInFileBlock(pData, pDumpInfo, key, &merge, pReader, &pTsRow);
doLoadRowsOfIdenticalTsInFileBlock(pFBlock, pBlock, pBlockData, pBlockScanInfo, pReader, &merge);
if (ik.ts == mergeTs) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader);
}
if (k.ts == mergeTs) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
}
tRowMergerGetRow(&merge, &pTSRow);
doAppendOneRow(pReader->pResBlock, pReader, pTSRow);
} else {
// [3] ik.ts < key <= k.ts
if (ik.ts < k.ts) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader);
tRowMergerGetRow(&merge, &pTSRow);
doAppendOneRow(pReader->pResBlock, pReader, pTSRow);
return TSDB_CODE_SUCCESS;
}
// [4] k.ts < key <= ik.ts
if (k.ts < ik.ts) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
tRowMergerGetRow(&merge, &pTSRow);
doAppendOneRow(pReader->pResBlock, pReader, pTSRow);
return TSDB_CODE_SUCCESS;
}
// [5] k.ts == ik.ts < key
if (k.ts == ik.ts) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader);
tRowMergerGetRow(&merge, &pTSRow);
if (k.ts == mergeTs) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
}
tRowMergerGetRow(&merge, &pTSRow);
doAppendOneRow(pReader->pResBlock, pReader, pTSRow);
return TSDB_CODE_SUCCESS;
}
// [6] k.ts < ik.ts < key
if (k.ts < ik.ts) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
tRowMergerGetRow(&merge, &pTSRow);
doAppendOneRow(pReader->pResBlock, pReader, pTSRow);
return TSDB_CODE_SUCCESS;
}
// [6] ik.ts < k.ts < key
if (ik.ts < k.ts) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader);
tRowMergerGetRow(&merge, &pTSRow);
doAppendOneRow(pReader->pResBlock, pReader, pTSRow);
return TSDB_CODE_SUCCESS;
}
}
} else if (pBlockScanInfo->imemHasVal) {
TSDBKEY ik = TSDBROW_KEY(piRow);
if (key <= ik.ts) {
TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
tRowMergerInit(&merge, &fRow, pReader->pSchema);
doLoadRowsOfIdenticalTsInFileBlock(pFBlock, pBlock, pBlockData, pBlockScanInfo, pReader, &merge);
if (ik.ts == mergeTs) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader);
}
tRowMergerGetRow(&merge, &pTSRow);
doAppendOneRow(pReader->pResBlock, pReader, pTSRow);
return TSDB_CODE_SUCCESS;
}
if (ik.ts < key) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader);
tRowMergerGetRow(&merge, &pTSRow);
doAppendOneRow(pReader->pResBlock, pReader, pTSRow);
return TSDB_CODE_SUCCESS;
}
} else { // pBlockScanInfo->memHasVal != NULL
TSDBKEY k = TSDBROW_KEY(pRow);
if (key <= k.ts) {
TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
tRowMergerInit(&merge, &fRow, pReader->pSchema);
doLoadRowsOfIdenticalTsInFileBlock(pFBlock, pBlock, pBlockData, pBlockScanInfo, pReader, &merge);
if (k.ts == mergeTs) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
}
tRowMergerGetRow(&merge, &pTSRow);
doAppendOneRow(pReader->pResBlock, pReader, pTSRow);
return TSDB_CODE_SUCCESS;
}
if (k.ts < key) {
doLoadRowsOfIdenticalTs(pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
tRowMergerGetRow(&merge, &pTSRow);
doAppendOneRow(pReader->pResBlock, pReader, pTSRow);
return TSDB_CODE_SUCCESS;
} }
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t buildComposedDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SBlock* pBlock, STableBlockScanInfo* pBlockScanInfo) {
SSDataBlock* pResBlock = pReader->pResBlock;
while(1) {
buildComposedDataBlockImpl(pReader, pFBlock, pBlock, pBlockScanInfo);
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter);
if (pBlockInfo->tbBlockIdx == pFBlock->tbBlockIdx) { // still in the same file block now
if (pDumpInfo->rowIndex >= pBlock->nRow) {
break;
}
if (pResBlock->info.rows >= pReader->capacity) {
break;
}
} else { // todo traverse to next file due to time window overlap
if (pResBlock->info.rows >= pReader->capacity) {
ASSERT(0);
return TSDB_CODE_SUCCESS;
}
}
}
pResBlock->info.uid = pBlockScanInfo->uid;
setComposedBlockFlag(pReader, true);
return TSDB_CODE_SUCCESS;
}
void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; }
static int32_t initMemIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
if (pBlockScanInfo->iterInit) {
return TSDB_CODE_SUCCESS;
}
TSDBKEY startKey = {.ts = pReader->window.skey, .version = pReader->startVersion};
STbData* d = NULL;
if (pReader->pTsdb->mem != NULL) {
tsdbGetTbDataFromMemTable(pReader->pTsdb->mem, pReader->suid, pBlockScanInfo->uid, &d);
tsdbTbDataIterCreate(d, &startKey, 0, &pBlockScanInfo->iter);
}
STbData* di = NULL;
if (pReader->pTsdb->imem != NULL) {
tsdbGetTbDataFromMemTable(pReader->pTsdb->imem, pReader->suid, pBlockScanInfo->uid, &di);
tsdbTbDataIterCreate(di, &startKey, 0, &pBlockScanInfo->iiter);
}
pBlockScanInfo->iterInit = true;
return TSDB_CODE_SUCCESS;
}
static TSDBKEY getCurrentKeyInBuf(SDataBlockIter* pBlockIter, STsdbReader* pReader) {
TSDBKEY key = {.ts = TSKEY_INITIAL_VAL};
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter);
STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid));
SBlock* pBlock = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx);
initMemIterator(pScanInfo, pReader);
if (pScanInfo->memHasVal) {
TSDBROW* pRow = getValidRow(pScanInfo->iter, &pScanInfo->memHasVal, pReader);
key = TSDBROW_KEY(pRow);
}
if (pScanInfo->imemHasVal) {
TSDBROW* pRow = getValidRow(pScanInfo->iiter, &pScanInfo->imemHasVal, pReader);
TSDBKEY k = TSDBROW_KEY(pRow);
if (key.ts > k.ts) {
key = k;
}
}
return key;
}
static int32_t loadDataInFiles(STsdbReader* pReader, bool* exists) { static int32_t loadDataInFiles(STsdbReader* pReader, bool* exists) {
SReaderStatus* pStatus = &pReader->status; SReaderStatus* pStatus = &pReader->status;
SFileSetIter* pFIter = &pStatus->fileIter; SFileSetIter* pFIter = &pStatus->fileIter;
SDataBlockIter* pBlockIter = &pReader->status.blockIter;
if (pFIter->index < pFIter->numOfFiles) { if (pFIter->index < pFIter->numOfFiles) {
if (pReader->status.blockIter.index == -1) { if (pReader->status.blockIter.index == -1) {
int32_t numOfBlocks = 0; int32_t numOfBlocks = 0;
while (1) { while (1) {
bool hasNext = nextFilesetIterator(&pStatus->fileIter, pReader->order, pReader); bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader->order, pReader);
if (!hasNext) { if (!hasNext) { // no data files on disk
// no data files on disk
break; break;
} }
@ -2323,51 +2499,61 @@ static int32_t loadDataInFiles(STsdbReader* pReader, bool* exists) {
// no blocks in current file, try next files // no blocks in current file, try next files
} }
SDataBlockIter* pBlockIter = &pReader->status.blockIter;
int32_t code = initBlockIterator(pReader, pBlockIter, numOfBlocks); int32_t code = initBlockIterator(pReader, pBlockIter, numOfBlocks);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
int64_t key = 0; // todo get the first qualified key in buffer // todo extract method: getCurrentKeyInBuf()
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter);
STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid));
SBlock* pBlock = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx); SBlock* pBlock = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx);
if (pScanInfo->iterInit == false) { TSDBKEY key = getCurrentKeyInBuf(pBlockIter, pReader);
STbData* d = NULL;
tsdbGetTbDataFromMemTable(pReader->pTsdb->mem, pReader->suid, pScanInfo->uid, &d);
TSDBKEY startKey = {.ts = pReader->window.skey, .version = pReader->startVersion}; if (dataBlockPartialRequired(&pReader->window, pBlock) || overlapWithNeighborBlock(pFBlock, pBlock, pScanInfo) /*|| points overlaps with data block*/) {
tsdbTbDataIterOpen(d, &startKey, 0, &pScanInfo->iter);
STbData* di = NULL;
tsdbGetTbDataFromMemTable(pReader->pTsdb->imem, pReader->suid, pScanInfo->uid, &di);
tsdbTbDataIterOpen(di, &startKey, 0, &pScanInfo->iiter);
pScanInfo->iterInit = true;
}
if (dataBlockPartialRequired(&pReader->window, pBlock) || overlapWithNeighborBlock(pBlock, pFBlock->tbBlockIdx)) {
SBlockData data = {0}; SBlockData data = {0};
doLoadFileBlockData(pReader, pBlockIter, pScanInfo, &data); doLoadFileBlockData(pReader, pBlockIter, pScanInfo, &data);
// build composed data block // build composed data block
buildComposedDataBlock(pReader, pFBlock, pBlock, pScanInfo);
} else if (bufferDataInFileBlockGap(pReader->order, key, pBlock)) { } else if (bufferDataInFileBlockGap(pReader->order, key, pBlock)) {
// data in memory that are earlier than current file block // data in memory that are earlier than current file block
TSDBKEY maxKey = {.ts = pReader->window.ekey, .version = pReader->endVersion}; TSDBKEY maxKey = {.ts = pReader->window.ekey, .version = pReader->endVersion};
buildInmemDataBlock(pReader, pScanInfo, pBlock, &maxKey); buildInmemDataBlock(pReader, pScanInfo, &maxKey);
// build data block from in-memory buffer data completed. // build data block from in-memory buffer data completed.
} else { // whole block is required, return it directly } else { // whole block is required, return it directly
SDataBlockInfo info = {0}; // todo check the data version
info.rows = pBlock->nRow; SDataBlockInfo* pInfo = &pReader->pResBlock->info;
info.uid = pScanInfo->uid; pInfo->rows = pBlock->nRow;
info.window.skey = pBlock->minKey.ts; pInfo->uid = pScanInfo->uid;
info.window.ekey = pBlock->maxKey.ts; pInfo->window.skey = pBlock->minKey.ts;
pInfo->window.ekey = pBlock->maxKey.ts;
setComposedBlockFlag(pReader, false);
}
} else {
SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter);
STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid));
SBlock* pBlock = taosArrayGet(pScanInfo->pBlockList, pFBlock->tbBlockIdx);
// current block are exhausted, try the next file block
if (pDumpInfo->rowIndex >= pBlock->nRow) {
bool hasNext = blockIteratorNext(pReader, &pReader->status.blockIter);
if (!hasNext) {
// current file is exhausted, let's try the next file
} else { // try next data block in current file
// 1. check if ts in buffer is overlap with current file data block
TSDBKEY key1 = getCurrentKeyInBuf(pBlockIter, pReader);
}
} else {
buildComposedDataBlock(pReader, pFBlock, pBlock, pScanInfo);
return TSDB_CODE_SUCCESS;
} }
} else { // repeat the previous procedure.
} }
} }
@ -2375,20 +2561,6 @@ static int32_t loadDataInFiles(STsdbReader* pReader, bool* exists) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// static bool doHasDataInBuffer(STsdbReader* pTsdbReadHandle) {
// size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo);
// while (pTsdbReadHandle->activeIndex < numOfTables) {
// if (hasMoreDataInCache(pTsdbReadHandle)) {
// return true;
// }
// pTsdbReadHandle->activeIndex += 1;
// }
// return false;
// }
// // todo not unref yet, since it is not support multi-group interpolation query // // todo not unref yet, since it is not support multi-group interpolation query
// static UNUSED_FUNC void changeQueryHandleForInterpQuery(STsdbReader* pHandle) { // static UNUSED_FUNC void changeQueryHandleForInterpQuery(STsdbReader* pHandle) {
// // filter the queried time stamp in the first place // // filter the queried time stamp in the first place
@ -2423,7 +2595,7 @@ static int32_t loadDataInFiles(STsdbReader* pReader, bool* exists) {
// } // }
TSDBROW* getValidRow(STbDataIter* pIter, bool* hasVal, STsdbReader* pReader) { TSDBROW* getValidRow(STbDataIter* pIter, bool* hasVal, STsdbReader* pReader) {
if (!hasVal) { if (!(*hasVal)) {
return NULL; return NULL;
} }
@ -2462,7 +2634,7 @@ TSDBROW* getValidRow(STbDataIter* pIter, bool* hasVal, STsdbReader* pReader) {
int32_t doLoadRowsOfIdenticalTs(STbDataIter *pIter, bool* hasVal, int64_t ts, SRowMerger* pMerger, STsdbReader* pReader) { int32_t doLoadRowsOfIdenticalTs(STbDataIter *pIter, bool* hasVal, int64_t ts, SRowMerger* pMerger, STsdbReader* pReader) {
while (1) { while (1) {
*hasVal = tsdbTbDataIterNext(pIter); *hasVal = tsdbTbDataIterNext(pIter);
if (!*hasVal) { if (!(*hasVal)) {
break; break;
} }
@ -2478,33 +2650,47 @@ int32_t doLoadRowsOfIdenticalTs(STbDataIter *pIter, bool* hasVal, int64_t ts, SR
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t doLoadRowsOfIdenticalTsInFileBlock(SBlockData* pData, SFileBlockDumpInfo* pDumpInfo, int64_t ts, SRowMerger *pMerger, int32_t doLoadRowsOfIdenticalTsInFileBlock(SFileDataBlockInfo* pFBlock, SBlock* pBlock, SBlockData* pBlockData,
STsdbReader* pReader, STSRow** pRow) { STableBlockScanInfo* pScanInfo, STsdbReader* pReader, SRowMerger* pMerger) {
int64_t key = pData->aTSKEY[pDumpInfo->rowIndex]; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
if ((pDumpInfo->rowIndex < pData->nRow - 1)) {
if (pData->aTSKEY[pDumpInfo->rowIndex + 1] < key) { int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex];
SRowMerger merger = {0}; if (pDumpInfo->rowIndex < pBlockData->nRow - 1) {
tRowMergerInit(&merger, NULL, NULL); if (pBlockData->aTSKEY[pDumpInfo->rowIndex + 1] == key) {
int32_t rowIndex = pDumpInfo->rowIndex + 1; int32_t rowIndex = pDumpInfo->rowIndex + 1;
while (pData->aTSKEY[rowIndex] == key) {
tRowMerge(&merger, NULL); while (pBlockData->aTSKEY[rowIndex] == key) {
if (pBlockData->aVersion[rowIndex] > pReader->endVersion) {
continue;
}
TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex);
tRowMerge(pMerger, &fRow);
rowIndex += 1;
} }
tRowMergerGetRow(&merger, pRow);
tRowMergerClear(&merger); pDumpInfo->rowIndex = rowIndex;
} }
} else { } else { // last row of current block, check if current block is overlapped with neighbor block
pDumpInfo->rowIndex += 1;
bool overlap = overlapWithNeighborBlock(pFBlock, pBlock, pScanInfo);
if (overlap) {
// load next block
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow) { int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow) {
STSchema* pSchema = NULL; // todo set the correct schema
TSKEY mergeTs = TSKEY_INITIAL_VAL; TSKEY mergeTs = TSKEY_INITIAL_VAL;
SRowMerger merge = {0}; SRowMerger merge = {0};
TSDBROW* pRow = getValidRow(&pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, pReader); TSDBROW* pRow = getValidRow(pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, pReader);
TSDBROW* piRow = getValidRow(&pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, pReader); TSDBROW* piRow = getValidRow(pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, pReader);
TSDBKEY k = {.ts = TSKEY_INITIAL_VAL}; TSDBKEY k = {.ts = TSKEY_INITIAL_VAL};
TSDBKEY ik = {.ts = TSKEY_INITIAL_VAL}; TSDBKEY ik = {.ts = TSKEY_INITIAL_VAL};
@ -2514,33 +2700,35 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
ik = TSDBROW_KEY(piRow); ik = TSDBROW_KEY(piRow);
if (ik.ts <= k.ts) { if (ik.ts <= k.ts) {
tRowMergerInit(&merge, piRow, pSchema); tRowMergerInit(&merge, piRow, pReader->pSchema);
doLoadRowsOfIdenticalTs(&pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader); doLoadRowsOfIdenticalTs(pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader);
if (k.ts == mergeTs) { if (k.ts == mergeTs) {
doLoadRowsOfIdenticalTs(&pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader); doLoadRowsOfIdenticalTs(pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
} }
tRowMergerGetRow(&merge, pTSRow); tRowMergerGetRow(&merge, pTSRow);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { // k.ts < ik.ts } else { // k.ts < ik.ts
tRowMergerInit(&merge, pRow, pSchema); tRowMergerInit(&merge, pRow, pReader->pSchema);
doLoadRowsOfIdenticalTs(&pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader); doLoadRowsOfIdenticalTs(pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
tRowMergerGetRow(&merge, pTSRow); tRowMergerGetRow(&merge, pTSRow);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} }
if (pBlockScanInfo->memHasVal) { if (pBlockScanInfo->memHasVal) {
tRowMergerInit(&merge, pRow, pSchema); k = TSDBROW_KEY(pRow);
doLoadRowsOfIdenticalTs(&pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader); tRowMergerInit(&merge, pRow, pReader->pSchema);
doLoadRowsOfIdenticalTs(pBlockScanInfo->iter, &pBlockScanInfo->memHasVal, k.ts, &merge, pReader);
tRowMergerGetRow(&merge, pTSRow); tRowMergerGetRow(&merge, pTSRow);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (pBlockScanInfo->imemHasVal) { if (pBlockScanInfo->imemHasVal) {
tRowMergerInit(&merge, piRow, pSchema); ik = TSDBROW_KEY(piRow);
doLoadRowsOfIdenticalTs(&pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader); tRowMergerInit(&merge, piRow, pReader->pSchema);
doLoadRowsOfIdenticalTs(pBlockScanInfo->iiter, &pBlockScanInfo->imemHasVal, ik.ts, &merge, pReader);
tRowMergerGetRow(&merge, pTSRow); tRowMergerGetRow(&merge, pTSRow);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2548,22 +2736,33 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t tsdbReadRowsFromCache(STableBlockScanInfo* pBlockScanInfo, TSDBKEY maxKey, int32_t capacity, STsdbReader* pReader) { int32_t doAppendOneRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow) {
int32_t numOfRows = 0; int32_t numOfRows = pBlock->info.rows;
int32_t numOfCols = (int32_t)taosArrayGetSize(pReader->pResBlock->pDataBlock); int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
SColVal colVal = {0};
for(int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
tTSRowGetVal(pTSRow, pReader->pSchema, pColInfoData->info.colId, &colVal);
colDataAppend(pColInfoData, numOfRows, (const char*)&colVal.value, colVal.isNull);
}
pBlock->info.rows += 1;
return TSDB_CODE_SUCCESS;
}
int32_t buildInmemDataBlockImpl(STableBlockScanInfo* pBlockScanInfo, TSDBKEY maxKey, int32_t capacity, STsdbReader* pReader) {
SSDataBlock* pBlock = pReader->pResBlock; SSDataBlock* pBlock = pReader->pResBlock;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
STSchema* pSchema = NULL;
do { do {
STSRow* pTSRow = NULL; STSRow* pTSRow = NULL;
tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow); tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow);
// todo assign to ssdatablock doAppendOneRow(pBlock, pReader, pTSRow);
if (pBlockScanInfo->memHasVal) { if (pBlockScanInfo->memHasVal) {
TSDBROW* pRow = tsdbTbDataIterGet(&pBlockScanInfo->iter); TSDBROW* pRow = tsdbTbDataIterGet(pBlockScanInfo->iter);
TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY k = TSDBROW_KEY(pRow);
if (k.ts >= maxKey.ts) { if (k.ts >= maxKey.ts) {
break; break;
@ -2571,7 +2770,7 @@ int32_t tsdbReadRowsFromCache(STableBlockScanInfo* pBlockScanInfo, TSDBKEY maxKe
} }
if (pBlockScanInfo->imemHasVal) { if (pBlockScanInfo->imemHasVal) {
TSDBROW* pRow = tsdbTbDataIterGet(&pBlockScanInfo->iiter); TSDBROW* pRow = tsdbTbDataIterGet(pBlockScanInfo->iiter);
TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY k = TSDBROW_KEY(pRow);
if (k.ts >= maxKey.ts) { if (k.ts >= maxKey.ts) {
break; break;
@ -2583,13 +2782,15 @@ int32_t tsdbReadRowsFromCache(STableBlockScanInfo* pBlockScanInfo, TSDBKEY maxKe
break; break;
} }
if (numOfRows >= capacity) { if (pBlock->info.rows >= capacity) {
break; break;
} }
} while (1); } while (1);
taosMemoryFreeClear(pSchema); ASSERT(pBlock->info.rows <= capacity);
assert(numOfRows <= capacity); pBlock->info.uid = pBlockScanInfo->uid;
int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock);
int64_t elapsedTime = taosGetTimestampUs() - st; int64_t elapsedTime = taosGetTimestampUs() - st;
tsdbDebug("%p build data block from cache completed, elapsed time:%" PRId64 " us, numOfRows:%d, numOfCols:%d, %s", tsdbDebug("%p build data block from cache completed, elapsed time:%" PRId64 " us, numOfRows:%d, numOfCols:%d, %s",
@ -2810,7 +3011,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInf
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
pReader->status.pTableMap = createDataBlockScanInfo(pReader, pCond->uidList, pCond->numOfTables); pReader->status.pTableMap = createDataBlockScanInfo(pReader, pTableList->pTableList->pData, taosArrayGetSize(pTableList->pTableList));
if (pReader->status.pTableMap == NULL) { if (pReader->status.pTableMap == NULL) {
tsdbReaderClose(pReader); tsdbReaderClose(pReader);
*ppReader = NULL; *ppReader = NULL;
@ -2908,19 +3109,44 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
int64_t stime = taosGetTimestampUs(); int64_t stime = taosGetTimestampUs();
int64_t elapsedTime = stime; int64_t elapsedTime = stime;
SReaderStatus* pStatus = &pReader->status;
if (pReader->type == BLOCK_LOAD_OFFSET_ORDER) { if (pReader->type == BLOCK_LOAD_OFFSET_ORDER) {
if (pReader->status.loadFromFile) { if (pStatus->loadFromFile) {
bool exists = true; bool exists = true;
int32_t code = loadDataInFiles(pReader, &exists); int32_t code = loadDataInFiles(pReader, &exists);
} else { // no data in files, let's try in-memory buffer } else { // no data in files, let's try the buffer
while(1) {
if (pStatus->pTableIter == NULL) {
pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL);
if (pStatus->pTableIter == NULL) {
return false;
}
}
STableBlockScanInfo* pBlockScanInfo = pStatus->pTableIter;
initMemIterator(pBlockScanInfo, pReader);
TSDBKEY maxKey = {.ts = pReader->window.ekey, .version = pReader->endVersion};
buildInmemDataBlock(pReader, pBlockScanInfo, &maxKey);
if (pReader->pResBlock->info.rows > 0) {
return true;
}
// current table is exhausted, let's try the next table
pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, &pStatus->pTableIter);
if (pStatus->pTableIter == NULL) {
return false;
}
}
} }
} else if (pReader->type == BLOCK_LOAD_TABLESEQ_ORDER) { } else if (pReader->type == BLOCK_LOAD_TABLESEQ_ORDER) {
} else if (pReader->type == BLOCK_LOAD_EXTERN_ORDER) { } else if (pReader->type == BLOCK_LOAD_EXTERN_ORDER) {
} else {
ASSERT(0);
} }
// if (pReader->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { // if (pReader->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) {
// return loadDataBlockFromTableSeq(pReader); // return loadDataBlockFromTableSeq(pReader);
@ -2957,32 +3183,10 @@ bool tsdbNextDataBlock(STsdbReader* pReader) {
} }
void tsdbRetrieveDataBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockInfo) { void tsdbRetrieveDataBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockInfo) {
// SQueryFilePos* cur = &pReader->cur; ASSERT(pDataBlockInfo != NULL && pReader != NULL);
pDataBlockInfo->rows = pReader->pResBlock->info.rows;
// uint64_t uid = 0; pDataBlockInfo->uid = pReader->pResBlock->info.uid;
pDataBlockInfo->window = pReader->pResBlock->info.window;
// // there are data in file
// if (pReader->cur.fid != INT32_MIN) {
// SFileBlockInfo* pBlockInfo = &pReader->pDataBlockInfo[cur->slot];
// uid = pBlockInfo->pTableCheckInfo->tableId;
// } else {
// STableBlockScanInfo* pCheckInfo = taosArrayGet(pReader->pTableCheckInfo, pReader->activeIndex);
// uid = pCheckInfo->tableId;
// }
// tsdbDebug("data block generated, uid:%" PRIu64 " numOfRows:%d, tsrange:%" PRId64 " - %" PRId64 " %s", uid,
// cur->rows,
// cur->win.skey, cur->win.ekey, pReader->idStr);
// pDataBlockInfo->uid = uid;
// #if 0
// // for multi-group data query processing test purpose
// pDataBlockInfo->groupId = uid;
// #endif
// pDataBlockInfo->rows = cur->rows;
// pDataBlockInfo->window = cur->win;
} }
int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) { int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) {
@ -3061,6 +3265,17 @@ int32_t tsdbRetrieveDataBlockStatisInfo(STsdbReader* pReader, SColumnDataAgg***
} }
SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) {
if (pReader->status.composedDataBlock) {
return pReader->pResBlock->pDataBlock;
} else {
SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter);
STableBlockScanInfo* pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pFBlock->uid, sizeof(pFBlock->uid));
SBlockData data = {0};
doLoadFileBlockData(pReader, &pReader->status.blockIter, pBlockScanInfo, &data);
// todo convert blockData to ssdatablock
}
// /** // /**
// * In the following two cases, the data has been loaded to SColumnInfoData. // * In the following two cases, the data has been loaded to SColumnInfoData.
// * 1. data is from cache, 2. data block is not completed qualified to query time range // * 1. data is from cache, 2. data block is not completed qualified to query time range